diff --git a/.buildkite/packer_cache.sh b/.buildkite/packer_cache.sh new file mode 100755 index 0000000000000..752914ba55c23 --- /dev/null +++ b/.buildkite/packer_cache.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +ROOT_DIR=$(cd "$(dirname "$0")/.." && pwd) + +branches=($(cat "$ROOT_DIR/branches.json" | jq -r '.branches[].branch')) +for branch in "${branches[@]}"; do + echo "Resolving dependencies for ${branch} branch" + rm -rf "checkout/$branch" + git clone /opt/git-mirrors/elastic-elasticsearch --branch "$branch" --single-branch "checkout/$branch" + + CHECKOUT_DIR=$(cd "./checkout/${branch}" && pwd) + CI_DIR="$CHECKOUT_DIR/.ci" + + if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then + ## On ARM we use a different properties file for setting java home + ## Also, we don't bother attempting to resolve dependencies for the 6.8 branch + source "$CI_DIR/java-versions-aarch64.properties" + export JAVA16_HOME="$HOME/.java/jdk16" + else + source "$CI_DIR/java-versions.properties" + ## We are caching BWC versions too, need these so we can build those + export JAVA8_HOME="$HOME/.java/java8" + export JAVA11_HOME="$HOME/.java/java11" + export JAVA12_HOME="$HOME/.java/openjdk12" + export JAVA13_HOME="$HOME/.java/openjdk13" + export JAVA14_HOME="$HOME/.java/openjdk14" + export JAVA15_HOME="$HOME/.java/openjdk15" + export JAVA16_HOME="$HOME/.java/openjdk16" + fi + + export JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA" + "checkout/${branch}/gradlew" --project-dir "$CHECKOUT_DIR" --parallel -s resolveAllDependencies -Dorg.gradle.warning.mode=none -DisCI + rm -rf "checkout/${branch}" +done diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index 762790e8816c0..3271007a00077 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -48,7 +48,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.16", "8.11.4", "8.12.0", "8.13.0"] + BWC_VERSION: ["7.17.17", "8.11.4", "8.12.0", "8.13.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 414f06dda49e6..66eb1fc79e3ca 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -1089,6 +1089,22 @@ steps: env: BWC_VERSION: 7.17.16 + - label: "{{matrix.image}} / 7.17.17 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.17 + timeout_in_minutes: 300 + matrix: + setup: + image: + - rocky-8 + - ubuntu-2004 + agents: + provider: gcp + image: family/elasticsearch-{{matrix.image}} + machineType: custom-16-32768 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 7.17.17 + - label: "{{matrix.image}} / 8.0.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.0 timeout_in_minutes: 300 diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml index b52f8506885c9..faf904f2f8b04 100644 --- a/.buildkite/pipelines/periodic-platform-support.yml +++ b/.buildkite/pipelines/periodic-platform-support.yml @@ -80,3 +80,19 @@ steps: diskName: /dev/sda1 env: GRADLE_TASK: "{{matrix.GRADLE_TASK}}" + - group: platform-support-unix-aws + steps: + - label: "{{matrix.image}} / platform-support-aws" + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true functionalTests + timeout_in_minutes: 420 + matrix: + setup: + image: + - amazonlinux-2023 + agents: + provider: aws + imagePrefix: elasticsearch-{{matrix.image}} + instanceType: m6a.8xlarge + diskSizeGb: 350 + diskType: gp3 + diskName: /dev/sda1 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 5d5fd6be4a6a5..3ce048533d131 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -662,6 +662,16 @@ steps: buildDirectory: /dev/shm/bk env: BWC_VERSION: 7.17.16 + - label: 7.17.17 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.17#bwcTest + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n1-standard-32 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 7.17.17 - label: 8.0.0 / bwc command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.0.0#bwcTest timeout_in_minutes: 300 diff --git a/.buildkite/scripts/periodic.trigger.sh b/.buildkite/scripts/periodic.trigger.sh index 3571d112c5b6d..cc10a5ae41861 100755 --- a/.buildkite/scripts/periodic.trigger.sh +++ b/.buildkite/scripts/periodic.trigger.sh @@ -6,11 +6,26 @@ echo "steps:" source .buildkite/scripts/branches.sh +IS_FIRST=true +SKIP_DELAY="${SKIP_DELAY:-false}" + for BRANCH in "${BRANCHES[@]}"; do INTAKE_PIPELINE_SLUG="elasticsearch-intake" BUILD_JSON=$(curl -sH "Authorization: Bearer ${BUILDKITE_API_TOKEN}" "https://api.buildkite.com/v2/organizations/elastic/pipelines/${INTAKE_PIPELINE_SLUG}/builds?branch=${BRANCH}&state=passed&per_page=1" | jq '.[0] | {commit: .commit, url: .web_url}') LAST_GOOD_COMMIT=$(echo "${BUILD_JSON}" | jq -r '.commit') + # Put a delay between each branch's set of pipelines by prepending each non-first branch with a sleep + # This is to smooth out the spike in agent requests + if [[ "$IS_FIRST" != "true" && "$SKIP_DELAY" != "true" ]]; then + cat < createRunBwcGradleTask( loggedExec.args("-Dorg.elasticsearch.build.cache.url=" + buildCacheUrl); } + if (System.getProperty("isCI") != null) { + loggedExec.args("-DisCI"); + } + loggedExec.args("-Dbuild.snapshot=true", "-Dscan.tag.NESTED"); final LogLevel logLevel = project.getGradle().getStartParameter().getLogLevel(); List nonDefaultLogLevels = Arrays.asList(LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheCacheableTestFixtures.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheCacheableTestFixtures.java index a01b1c28a851f..bfc52adcdecfd 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheCacheableTestFixtures.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheCacheableTestFixtures.java @@ -25,7 +25,9 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.Constructor; import java.lang.reflect.Method; +import java.lang.reflect.Modifier; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; @@ -74,9 +76,14 @@ public void execute() { Set> classes = (Set>) reflections.getSubTypesOf(ifClass); for (Class cacheableTestFixtureClazz : classes) { - Object o = cacheableTestFixtureClazz.getDeclaredConstructor().newInstance(); - Method cacheMethod = cacheableTestFixtureClazz.getMethod("cache"); - cacheMethod.invoke(o); + if (Modifier.isAbstract(cacheableTestFixtureClazz.getModifiers()) == false) { + Constructor declaredConstructor = cacheableTestFixtureClazz.getDeclaredConstructor(); + declaredConstructor.setAccessible(true); + Object o = declaredConstructor.newInstance(); + Method cacheMethod = cacheableTestFixtureClazz.getMethod("cache"); + System.out.println("Caching resources from " + cacheableTestFixtureClazz.getName()); + cacheMethod.invoke(o); + } } } catch (Exception e) { throw new RuntimeException(e); diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java index 29650e4b74114..d312fae4456f1 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java @@ -10,6 +10,7 @@ import org.elasticsearch.bootstrap.ServerArgs; import org.elasticsearch.cli.ExitCodes; +import org.elasticsearch.cli.ProcessInfo; import org.elasticsearch.cli.UserException; import java.io.BufferedReader; @@ -39,7 +40,7 @@ /** * Parses JVM options from a file and prints a single line with all JVM options to standard output. */ -final class JvmOptionsParser { +public final class JvmOptionsParser { static class JvmOptionsFileParserException extends Exception { @@ -59,7 +60,6 @@ SortedMap invalidLines() { this.jvmOptionsFile = jvmOptionsFile; this.invalidLines = invalidLines; } - } /** @@ -70,25 +70,27 @@ SortedMap invalidLines() { * variable. * * @param args the start-up arguments - * @param configDir the ES config dir + * @param processInfo information about the CLI process. * @param tmpDir the directory that should be passed to {@code -Djava.io.tmpdir} - * @param envOptions the options passed through the ES_JAVA_OPTS env var * @return the list of options to put on the Java command line * @throws InterruptedException if the java subprocess is interrupted * @throws IOException if there is a problem reading any of the files * @throws UserException if there is a problem parsing the `jvm.options` file or `jvm.options.d` files */ - static List determineJvmOptions(ServerArgs args, Path configDir, Path tmpDir, String envOptions) throws InterruptedException, + public static List determineJvmOptions(ServerArgs args, ProcessInfo processInfo, Path tmpDir) throws InterruptedException, IOException, UserException { - final JvmOptionsParser parser = new JvmOptionsParser(); final Map substitutions = new HashMap<>(); substitutions.put("ES_TMPDIR", tmpDir.toString()); - substitutions.put("ES_PATH_CONF", configDir.toString()); + substitutions.put("ES_PATH_CONF", args.configDir().toString()); + + final String envOptions = processInfo.envVars().get("ES_JAVA_OPTS"); try { - return parser.jvmOptions(args, configDir, tmpDir, envOptions, substitutions); + return Collections.unmodifiableList( + parser.jvmOptions(args, args.configDir(), tmpDir, envOptions, substitutions, processInfo.sysprops()) + ); } catch (final JvmOptionsFileParserException e) { final String errorMessage = String.format( Locale.ROOT, @@ -122,7 +124,8 @@ private List jvmOptions( final Path config, Path tmpDir, final String esJavaOpts, - final Map substitutions + final Map substitutions, + final Map cliSysprops ) throws InterruptedException, IOException, JvmOptionsFileParserException, UserException { final List jvmOptions = readJvmOptionsFiles(config); @@ -137,7 +140,7 @@ private List jvmOptions( ); substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(config, substitutedJvmOptions)); final List ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions); - final List systemJvmOptions = SystemJvmOptions.systemJvmOptions(args.nodeSettings()); + final List systemJvmOptions = SystemJvmOptions.systemJvmOptions(args.nodeSettings(), cliSysprops); final List apmOptions = APMJvmOptions.apmJvmOptions(args.nodeSettings(), args.secrets(), args.logsDir(), tmpDir); diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java index ea2df72fb2c0b..aac5f718081b4 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java @@ -243,8 +243,15 @@ protected Command loadTool(String toolname, String libs) { } // protected to allow tests to override - protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) throws UserException { - return ServerProcess.start(terminal, processInfo, args); + protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) throws Exception { + var tempDir = ServerProcessUtils.setupTempDir(processInfo); + var jvmOptions = JvmOptionsParser.determineJvmOptions(args, processInfo, tempDir); + var serverProcessBuilder = new ServerProcessBuilder().withTerminal(terminal) + .withProcessInfo(processInfo) + .withServerArgs(args) + .withTempDir(tempDir) + .withJvmOptions(jvmOptions); + return serverProcessBuilder.start(); } // protected to allow tests to override diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java index d4b4d57977f5d..3972095a3a5c0 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java @@ -9,34 +9,17 @@ package org.elasticsearch.server.cli; import org.elasticsearch.bootstrap.BootstrapInfo; -import org.elasticsearch.bootstrap.ServerArgs; -import org.elasticsearch.cli.ExitCodes; -import org.elasticsearch.cli.ProcessInfo; -import org.elasticsearch.cli.Terminal; -import org.elasticsearch.cli.UserException; -import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; import org.elasticsearch.core.IOUtils; -import org.elasticsearch.core.PathUtils; -import org.elasticsearch.core.SuppressForbidden; import java.io.IOException; import java.io.OutputStream; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.FileAttribute; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import static org.elasticsearch.server.cli.ProcessUtil.nonInterruptible; /** * A helper to control a {@link Process} running the main Elasticsearch server. * - *

The process can be started by calling {@link #start(Terminal, ProcessInfo, ServerArgs)}. + *

The process can be started by calling {@link ServerProcessBuilder#start()}. * The process is controlled by internally sending arguments and control signals on stdin, * and receiving control signals on stderr. The start method does not return until the * server is ready to process requests and has exited the bootstrap thread. @@ -64,68 +47,6 @@ public class ServerProcess { this.errorPump = errorPump; } - // this allows mocking the process building by tests - interface OptionsBuilder { - List getJvmOptions(ServerArgs args, Path configDir, Path tmpDir, String envOptions) throws InterruptedException, - IOException, UserException; - } - - // this allows mocking the process building by tests - interface ProcessStarter { - Process start(ProcessBuilder pb) throws IOException; - } - - /** - * Start a server in a new process. - * - * @param terminal A terminal to connect the standard inputs and outputs to for the new process. - * @param processInfo Info about the current process, for passing through to the subprocess. - * @param args Arguments to the server process. - * @return A running server process that is ready for requests - * @throws UserException If the process failed during bootstrap - */ - public static ServerProcess start(Terminal terminal, ProcessInfo processInfo, ServerArgs args) throws UserException { - return start(terminal, processInfo, args, JvmOptionsParser::determineJvmOptions, ProcessBuilder::start); - } - - // package private so tests can mock options building and process starting - static ServerProcess start( - Terminal terminal, - ProcessInfo processInfo, - ServerArgs args, - OptionsBuilder optionsBuilder, - ProcessStarter processStarter - ) throws UserException { - Process jvmProcess = null; - ErrorPumpThread errorPump; - - boolean success = false; - try { - jvmProcess = createProcess(args, processInfo, args.configDir(), optionsBuilder, processStarter); - errorPump = new ErrorPumpThread(terminal.getErrorWriter(), jvmProcess.getErrorStream()); - errorPump.start(); - sendArgs(args, jvmProcess.getOutputStream()); - - String errorMsg = errorPump.waitUntilReady(); - if (errorMsg != null) { - // something bad happened, wait for the process to exit then rethrow - int exitCode = jvmProcess.waitFor(); - throw new UserException(exitCode, errorMsg); - } - success = true; - } catch (InterruptedException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new UncheckedIOException(e); - } finally { - if (success == false && jvmProcess != null && jvmProcess.isAlive()) { - jvmProcess.destroyForcibly(); - } - } - - return new ServerProcess(jvmProcess, errorPump); - } - /** * Return the process id of the server. */ @@ -169,19 +90,6 @@ public synchronized void stop() { waitFor(); // ignore exit code, we are already shutting down } - private static void sendArgs(ServerArgs args, OutputStream processStdin) { - // DO NOT close the underlying process stdin, since we need to be able to write to it to signal exit - var out = new OutputStreamStreamOutput(processStdin); - try { - args.writeTo(out); - out.flush(); - } catch (IOException ignore) { - // A failure to write here means the process has problems, and it will die anyway. We let this fall through - // so the pump thread can complete, writing out the actual error. All we get here is the failure to write to - // the process pipe, which isn't helpful to print. - } - } - private void sendShutdownMarker() { try { OutputStream os = jvmProcess.getOutputStream(); @@ -191,80 +99,4 @@ private void sendShutdownMarker() { // process is already effectively dead, fall through to wait for it, or should we SIGKILL? } } - - private static Process createProcess( - ServerArgs args, - ProcessInfo processInfo, - Path configDir, - OptionsBuilder optionsBuilder, - ProcessStarter processStarter - ) throws InterruptedException, IOException, UserException { - Map envVars = new HashMap<>(processInfo.envVars()); - Path tempDir = setupTempDir(processInfo, envVars.remove("ES_TMPDIR")); - if (envVars.containsKey("LIBFFI_TMPDIR") == false) { - envVars.put("LIBFFI_TMPDIR", tempDir.toString()); - } - - List jvmOptions = optionsBuilder.getJvmOptions(args, configDir, tempDir, envVars.remove("ES_JAVA_OPTS")); - // also pass through distribution type - jvmOptions.add("-Des.distribution.type=" + processInfo.sysprops().get("es.distribution.type")); - - Path esHome = processInfo.workingDir(); - Path javaHome = PathUtils.get(processInfo.sysprops().get("java.home")); - List command = new ArrayList<>(); - boolean isWindows = processInfo.sysprops().get("os.name").startsWith("Windows"); - command.add(javaHome.resolve("bin").resolve("java" + (isWindows ? ".exe" : "")).toString()); - command.addAll(jvmOptions); - command.add("--module-path"); - command.add(esHome.resolve("lib").toString()); - // Special circumstances require some modules (not depended on by the main server module) to be explicitly added: - command.add("--add-modules=jdk.net"); // needed to reflectively set extended socket options - // we control the module path, which may have additional modules not required by server - command.add("--add-modules=ALL-MODULE-PATH"); - command.add("-m"); - command.add("org.elasticsearch.server/org.elasticsearch.bootstrap.Elasticsearch"); - - var builder = new ProcessBuilder(command); - builder.environment().putAll(envVars); - builder.redirectOutput(ProcessBuilder.Redirect.INHERIT); - - return processStarter.start(builder); - } - - /** - * Returns the java.io.tmpdir Elasticsearch should use, creating it if necessary. - * - *

On non-Windows OS, this will be created as a subdirectory of the default temporary directory. - * Note that this causes the created temporary directory to be a private temporary directory. - */ - private static Path setupTempDir(ProcessInfo processInfo, String tmpDirOverride) throws UserException, IOException { - final Path path; - if (tmpDirOverride != null) { - path = Paths.get(tmpDirOverride); - if (Files.exists(path) == false) { - throw new UserException(ExitCodes.CONFIG, "Temporary directory [" + path + "] does not exist or is not accessible"); - } - if (Files.isDirectory(path) == false) { - throw new UserException(ExitCodes.CONFIG, "Temporary directory [" + path + "] is not a directory"); - } - } else { - if (processInfo.sysprops().get("os.name").startsWith("Windows")) { - /* - * On Windows, we avoid creating a unique temporary directory per invocation lest - * we pollute the temporary directory. On other operating systems, temporary directories - * will be cleaned automatically via various mechanisms (e.g., systemd, or restarts). - */ - path = Paths.get(processInfo.sysprops().get("java.io.tmpdir"), "elasticsearch"); - Files.createDirectories(path); - } else { - path = createTempDirectory("elasticsearch-"); - } - } - return path; - } - - @SuppressForbidden(reason = "Files#createTempDirectory(String, FileAttribute...)") - private static Path createTempDirectory(final String prefix, final FileAttribute... attrs) throws IOException { - return Files.createTempDirectory(prefix, attrs); - } } diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java new file mode 100644 index 0000000000000..4ef1e2bfd4737 --- /dev/null +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java @@ -0,0 +1,208 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.server.cli; + +import org.elasticsearch.bootstrap.ServerArgs; +import org.elasticsearch.cli.ProcessInfo; +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.cli.UserException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; +import org.elasticsearch.core.PathUtils; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +/** + * This class is used to create a {@link ServerProcess}. + * Each ServerProcessBuilder instance manages a collection of process attributes. The {@link ServerProcessBuilder#start()} method creates + * a new {@link ServerProcess} instance with those attributes. + * + * Each process builder manages these process attributes: + * - a temporary directory + * - process info to pass through to the new Java subprocess + * - the command line arguments to run Elasticsearch + * - a list of JVM options to be passed to the Elasticsearch Java process + * - a {@link Terminal} to read input and write output from/to the cli console + */ +public class ServerProcessBuilder { + private Path tempDir; + private ServerArgs serverArgs; + private ProcessInfo processInfo; + private List jvmOptions; + private Terminal terminal; + + // this allows mocking the process building by tests + interface ProcessStarter { + Process start(ProcessBuilder pb) throws IOException; + } + + /** + * Specifies the temporary directory to be used by the server process + */ + public ServerProcessBuilder withTempDir(Path tempDir) { + this.tempDir = tempDir; + return this; + } + + /** + * Specifies the process info to pass through to the new Java subprocess + */ + public ServerProcessBuilder withProcessInfo(ProcessInfo processInfo) { + this.processInfo = processInfo; + return this; + } + + /** + * Specifies the command line arguments to run Elasticsearch + */ + public ServerProcessBuilder withServerArgs(ServerArgs serverArgs) { + this.serverArgs = serverArgs; + return this; + } + + /** + * Specifies the JVM options to be passed to the Elasticsearch Java process + */ + public ServerProcessBuilder withJvmOptions(List jvmOptions) { + this.jvmOptions = jvmOptions; + return this; + } + + /** + * Specifies the {@link Terminal} to use for reading input and writing output from/to the cli console + */ + public ServerProcessBuilder withTerminal(Terminal terminal) { + this.terminal = terminal; + return this; + } + + private Map getEnvironment() { + Map envVars = new HashMap<>(processInfo.envVars()); + + envVars.remove("ES_TMPDIR"); + if (envVars.containsKey("LIBFFI_TMPDIR") == false) { + envVars.put("LIBFFI_TMPDIR", tempDir.toString()); + } + envVars.remove("ES_JAVA_OPTS"); + + return envVars; + } + + private List getJvmArgs() { + Path esHome = processInfo.workingDir(); + return List.of( + "--module-path", + esHome.resolve("lib").toString(), + // Special circumstances require some modules (not depended on by the main server module) to be explicitly added: + "--add-modules=jdk.net", // needed to reflectively set extended socket options + // we control the module path, which may have additional modules not required by server + "--add-modules=ALL-MODULE-PATH", + "-m", + "org.elasticsearch.server/org.elasticsearch.bootstrap.Elasticsearch" + ); + } + + private String getCommand() { + Path javaHome = PathUtils.get(processInfo.sysprops().get("java.home")); + + boolean isWindows = processInfo.sysprops().get("os.name").startsWith("Windows"); + return javaHome.resolve("bin").resolve("java" + (isWindows ? ".exe" : "")).toString(); + } + + /** + * Start a server in a new process. + * + * @return A running server process that is ready for requests + * @throws UserException If the process failed during bootstrap + */ + public ServerProcess start() throws UserException { + return start(ProcessBuilder::start); + } + + private static void checkRequiredArgument(Object argument, String argumentName) { + if (argument == null) { + throw new IllegalStateException( + Strings.format("'%s' is a required argument and needs to be specified before calling start()", argumentName) + ); + } + } + + // package private for testing + ServerProcess start(ProcessStarter processStarter) throws UserException { + checkRequiredArgument(tempDir, "tempDir"); + checkRequiredArgument(serverArgs, "serverArgs"); + checkRequiredArgument(processInfo, "processInfo"); + checkRequiredArgument(jvmOptions, "jvmOptions"); + checkRequiredArgument(terminal, "terminal"); + + Process jvmProcess = null; + ErrorPumpThread errorPump; + + boolean success = false; + try { + jvmProcess = createProcess(getCommand(), getJvmArgs(), jvmOptions, getEnvironment(), processStarter); + errorPump = new ErrorPumpThread(terminal.getErrorWriter(), jvmProcess.getErrorStream()); + errorPump.start(); + sendArgs(serverArgs, jvmProcess.getOutputStream()); + + String errorMsg = errorPump.waitUntilReady(); + if (errorMsg != null) { + // something bad happened, wait for the process to exit then rethrow + int exitCode = jvmProcess.waitFor(); + throw new UserException(exitCode, errorMsg); + } + success = true; + } catch (InterruptedException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new UncheckedIOException(e); + } finally { + if (success == false && jvmProcess != null && jvmProcess.isAlive()) { + jvmProcess.destroyForcibly(); + } + } + + return new ServerProcess(jvmProcess, errorPump); + } + + private static Process createProcess( + String command, + List jvmArgs, + List jvmOptions, + Map environment, + ProcessStarter processStarter + ) throws InterruptedException, IOException { + + var builder = new ProcessBuilder(Stream.concat(Stream.of(command), Stream.concat(jvmOptions.stream(), jvmArgs.stream())).toList()); + builder.environment().putAll(environment); + builder.redirectOutput(ProcessBuilder.Redirect.INHERIT); + + return processStarter.start(builder); + } + + private static void sendArgs(ServerArgs args, OutputStream processStdin) { + // DO NOT close the underlying process stdin, since we need to be able to write to it to signal exit + var out = new OutputStreamStreamOutput(processStdin); + try { + args.writeTo(out); + out.flush(); + } catch (IOException ignore) { + // A failure to write here means the process has problems, and it will die anyway. We let this fall through + // so the pump thread can complete, writing out the actual error. All we get here is the failure to write to + // the process pipe, which isn't helpful to print. + } + } +} diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessUtils.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessUtils.java new file mode 100644 index 0000000000000..ebbc68b1be90b --- /dev/null +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessUtils.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.server.cli; + +import org.elasticsearch.cli.ExitCodes; +import org.elasticsearch.cli.ProcessInfo; +import org.elasticsearch.cli.UserException; +import org.elasticsearch.core.SuppressForbidden; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileAttribute; + +public class ServerProcessUtils { + + /** + * Returns the java.io.tmpdir Elasticsearch should use, creating it if necessary. + * + *

On non-Windows OS, this will be created as a subdirectory of the default temporary directory. + * Note that this causes the created temporary directory to be a private temporary directory. + */ + public static Path setupTempDir(ProcessInfo processInfo) throws UserException { + final Path path; + String tmpDirOverride = processInfo.envVars().get("ES_TMPDIR"); + if (tmpDirOverride != null) { + path = Paths.get(tmpDirOverride); + if (Files.exists(path) == false) { + throw new UserException(ExitCodes.CONFIG, "Temporary directory [" + path + "] does not exist or is not accessible"); + } + if (Files.isDirectory(path) == false) { + throw new UserException(ExitCodes.CONFIG, "Temporary directory [" + path + "] is not a directory"); + } + } else { + try { + if (processInfo.sysprops().get("os.name").startsWith("Windows")) { + /* + * On Windows, we avoid creating a unique temporary directory per invocation lest + * we pollute the temporary directory. On other operating systems, temporary directories + * will be cleaned automatically via various mechanisms (e.g., systemd, or restarts). + */ + path = Paths.get(processInfo.sysprops().get("java.io.tmpdir"), "elasticsearch"); + Files.createDirectories(path); + } else { + path = createTempDirectory("elasticsearch-"); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + return path; + } + + @SuppressForbidden(reason = "Files#createTempDirectory(String, FileAttribute...)") + private static Path createTempDirectory(final String prefix, final FileAttribute... attrs) throws IOException { + return Files.createTempDirectory(prefix, attrs); + } +} diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java index 6e250075f7747..4a8b3da4777a0 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java @@ -12,12 +12,13 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; final class SystemJvmOptions { - static List systemJvmOptions(Settings nodeSettings) { + static List systemJvmOptions(Settings nodeSettings, final Map sysprops) { return Stream.of( /* * Cache ttl in seconds for positive DNS lookups noting that this overrides the JDK security property networkaddress.cache.ttl; @@ -65,7 +66,9 @@ static List systemJvmOptions(Settings nodeSettings) { */ "--add-opens=java.base/java.io=org.elasticsearch.preallocate", maybeOverrideDockerCgroup(), - maybeSetActiveProcessorCount(nodeSettings) + maybeSetActiveProcessorCount(nodeSettings), + // Pass through distribution type + "-Des.distribution.type=" + sysprops.get("es.distribution.type") ).filter(e -> e.isEmpty() == false).collect(Collectors.toList()); } diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java index 03856b1024992..101be4301b522 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java @@ -53,7 +53,6 @@ public void testUnversionedOptions() throws IOException { try (StringReader sr = new StringReader("-Xms1g\n-Xmx1g"); BufferedReader br = new BufferedReader(sr)) { assertExpectedJvmOptions(randomIntBetween(8, Integer.MAX_VALUE), br, Arrays.asList("-Xms1g", "-Xmx1g")); } - } public void testSingleVersionOption() throws IOException { @@ -351,25 +350,30 @@ public void accept(final int lineNumber, final String line) { public void testNodeProcessorsActiveCount() { { - final List jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY); + final List jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, Map.of()); assertThat(jvmOptions, not(hasItem(containsString("-XX:ActiveProcessorCount=")))); } { Settings nodeSettings = Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 1).build(); - final List jvmOptions = SystemJvmOptions.systemJvmOptions(nodeSettings); + final List jvmOptions = SystemJvmOptions.systemJvmOptions(nodeSettings, Map.of()); assertThat(jvmOptions, hasItem("-XX:ActiveProcessorCount=1")); } { // check rounding Settings nodeSettings = Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 0.2).build(); - final List jvmOptions = SystemJvmOptions.systemJvmOptions(nodeSettings); + final List jvmOptions = SystemJvmOptions.systemJvmOptions(nodeSettings, Map.of()); assertThat(jvmOptions, hasItem("-XX:ActiveProcessorCount=1")); } { // check validation Settings nodeSettings = Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 10000).build(); - var e = expectThrows(IllegalArgumentException.class, () -> SystemJvmOptions.systemJvmOptions(nodeSettings)); + var e = expectThrows(IllegalArgumentException.class, () -> SystemJvmOptions.systemJvmOptions(nodeSettings, Map.of())); assertThat(e.getMessage(), containsString("setting [node.processors] must be <=")); } } + + public void testCommandLineDistributionType() { + final List jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, Map.of("es.distribution.type", "testdistro")); + assertThat(jvmOptions, hasItem("-Des.distribution.type=testdistro")); + } } diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java index da2c0104dd08e..e469764590bd6 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java @@ -314,6 +314,21 @@ public void testIgnoreNullExceptionOutput() throws Exception { assertThat(terminal.getErrorOutput(), not(containsString("null"))); } + public void testOptionsBuildingInterrupted() throws IOException { + Command command = new TestServerCli() { + @Override + protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) throws Exception { + throw new InterruptedException("interrupted while get jvm options"); + } + }; + var e = expectThrows( + InterruptedException.class, + () -> command.main(new String[0], terminal, new ProcessInfo(sysprops, envVars, esHomeDir)) + ); + assertThat(e.getMessage(), equalTo("interrupted while get jvm options")); + command.close(); + } + public void testServerExitsNonZero() throws Exception { mockServerExitCode = 140; int exitCode = executeMain(); @@ -480,63 +495,65 @@ void reset() { } } - @Override - protected Command newCommand() { - return new ServerCli() { - @Override - protected Command loadTool(String toolname, String libs) { - if (toolname.equals("auto-configure-node")) { - assertThat(libs, equalTo("modules/x-pack-core,modules/x-pack-security,lib/tools/security-cli")); - return AUTO_CONFIG_CLI; - } else if (toolname.equals("sync-plugins")) { - assertThat(libs, equalTo("lib/tools/plugin-cli")); - return SYNC_PLUGINS_CLI; - } - throw new AssertionError("Unknown tool: " + toolname); + private class TestServerCli extends ServerCli { + @Override + protected Command loadTool(String toolname, String libs) { + if (toolname.equals("auto-configure-node")) { + assertThat(libs, equalTo("modules/x-pack-core,modules/x-pack-security,lib/tools/security-cli")); + return AUTO_CONFIG_CLI; + } else if (toolname.equals("sync-plugins")) { + assertThat(libs, equalTo("lib/tools/plugin-cli")); + return SYNC_PLUGINS_CLI; } + throw new AssertionError("Unknown tool: " + toolname); + } - @Override - Environment autoConfigureSecurity( - Terminal terminal, - OptionSet options, - ProcessInfo processInfo, - Environment env, - SecureString keystorePassword - ) throws Exception { - if (mockSecureSettingsLoader != null && mockSecureSettingsLoader.supportsSecurityAutoConfiguration() == false) { - fail("We shouldn't be calling auto configure on loaders that don't support it"); - } - return super.autoConfigureSecurity(terminal, options, processInfo, env, keystorePassword); + @Override + Environment autoConfigureSecurity( + Terminal terminal, + OptionSet options, + ProcessInfo processInfo, + Environment env, + SecureString keystorePassword + ) throws Exception { + if (mockSecureSettingsLoader != null && mockSecureSettingsLoader.supportsSecurityAutoConfiguration() == false) { + fail("We shouldn't be calling auto configure on loaders that don't support it"); } + return super.autoConfigureSecurity(terminal, options, processInfo, env, keystorePassword); + } - @Override - protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) { - if (argsValidator != null) { - argsValidator.accept(args); - } - mockServer.reset(); - return mockServer; + @Override + void syncPlugins(Terminal terminal, Environment env, ProcessInfo processInfo) throws Exception { + if (mockSecureSettingsLoader != null && mockSecureSettingsLoader instanceof MockSecureSettingsLoader mock) { + mock.verifiedEnv = true; + // equals as a pointer, environment shouldn't be changed if autoconfigure is not supported + assertFalse(mockSecureSettingsLoader.supportsSecurityAutoConfiguration()); + assertTrue(mock.environment == env); } - @Override - void syncPlugins(Terminal terminal, Environment env, ProcessInfo processInfo) throws Exception { - if (mockSecureSettingsLoader != null && mockSecureSettingsLoader instanceof MockSecureSettingsLoader mock) { - mock.verifiedEnv = true; - // equals as a pointer, environment shouldn't be changed if autoconfigure is not supported - assertFalse(mockSecureSettingsLoader.supportsSecurityAutoConfiguration()); - assertTrue(mock.environment == env); - } + super.syncPlugins(terminal, env, processInfo); + } - super.syncPlugins(terminal, env, processInfo); + @Override + protected SecureSettingsLoader secureSettingsLoader(Environment env) { + if (mockSecureSettingsLoader != null) { + return mockSecureSettingsLoader; } + return new KeystoreSecureSettingsLoader(); + } + } + + @Override + protected Command newCommand() { + return new TestServerCli() { @Override - protected SecureSettingsLoader secureSettingsLoader(Environment env) { - if (mockSecureSettingsLoader != null) { - return mockSecureSettingsLoader; + protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) { + if (argsValidator != null) { + argsValidator.accept(args); } - - return new KeystoreSecureSettingsLoader(); + mockServer.reset(); + return mockServer; } }; } diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessTests.java index 57993d40391ac..fa36007b40af7 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.cli.ExitCodes; import org.elasticsearch.cli.MockTerminal; import org.elasticsearch.cli.ProcessInfo; -import org.elasticsearch.cli.UserException; import org.elasticsearch.common.io.stream.InputStreamStreamInput; import org.elasticsearch.common.settings.KeyStoreWrapper; import org.elasticsearch.common.settings.SecureSettings; @@ -34,7 +33,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -47,7 +45,6 @@ import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.server.cli.ProcessUtil.nonInterruptibleVoid; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; @@ -56,7 +53,6 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; -import static org.hamcrest.Matchers.startsWith; public class ServerProcessTests extends ESTestCase { @@ -66,7 +62,6 @@ public class ServerProcessTests extends ESTestCase { protected final Map envVars = new HashMap<>(); Path esHomeDir; Settings.Builder nodeSettings; - ServerProcess.OptionsBuilder optionsBuilder; ProcessValidator processValidator; MainMethod mainCallback; MockElasticsearchProcess process; @@ -81,7 +76,7 @@ interface ProcessValidator { } int runForeground() throws Exception { - var server = startProcess(false, false, ""); + var server = startProcess(false, false); return server.waitFor(); } @@ -94,7 +89,6 @@ public void resetEnv() { envVars.clear(); esHomeDir = createTempDir(); nodeSettings = Settings.builder(); - optionsBuilder = (args, configDir, tmpDir, envOptions) -> new ArrayList<>(); processValidator = null; mainCallback = null; secrets = KeyStoreWrapper.create(); @@ -193,9 +187,12 @@ public Process destroyForcibly() { } } - ServerProcess startProcess(boolean daemonize, boolean quiet, String keystorePassword) throws Exception { - var pinfo = new ProcessInfo(Map.copyOf(sysprops), Map.copyOf(envVars), esHomeDir); - var args = new ServerArgs( + ProcessInfo createProcessInfo() { + return new ProcessInfo(Map.copyOf(sysprops), Map.copyOf(envVars), esHomeDir); + } + + ServerArgs createServerArgs(boolean daemonize, boolean quiet) { + return new ServerArgs( daemonize, quiet, null, @@ -204,14 +201,23 @@ ServerProcess startProcess(boolean daemonize, boolean quiet, String keystorePass esHomeDir.resolve("config"), esHomeDir.resolve("logs") ); - ServerProcess.ProcessStarter starter = pb -> { + } + + ServerProcess startProcess(boolean daemonize, boolean quiet) throws Exception { + var pinfo = createProcessInfo(); + ServerProcessBuilder.ProcessStarter starter = pb -> { if (processValidator != null) { processValidator.validate(pb); } process = new MockElasticsearchProcess(); return process; }; - return ServerProcess.start(terminal, pinfo, args, optionsBuilder, starter); + var serverProcessBuilder = new ServerProcessBuilder().withTerminal(terminal) + .withProcessInfo(pinfo) + .withServerArgs(createServerArgs(daemonize, quiet)) + .withJvmOptions(List.of()) + .withTempDir(ServerProcessUtils.setupTempDir(pinfo)); + return serverProcessBuilder.start(starter); } public void testProcessBuilder() throws Exception { @@ -231,7 +237,7 @@ public void testProcessBuilder() throws Exception { } public void testPid() throws Exception { - var server = startProcess(true, false, ""); + var server = startProcess(true, false); assertThat(server.pid(), equalTo(12345L)); server.stop(); } @@ -246,18 +252,12 @@ public void testBootstrapError() throws Exception { assertThat(terminal.getErrorOutput(), containsString("a bootstrap exception")); } - public void testStartError() throws Exception { + public void testStartError() { processValidator = pb -> { throw new IOException("something went wrong"); }; - var e = expectThrows(UncheckedIOException.class, () -> runForeground()); + var e = expectThrows(UncheckedIOException.class, this::runForeground); assertThat(e.getCause().getMessage(), equalTo("something went wrong")); } - public void testOptionsBuildingInterrupted() throws Exception { - optionsBuilder = (args, configDir, tmpDir, envOptions) -> { throw new InterruptedException("interrupted while get jvm options"); }; - var e = expectThrows(RuntimeException.class, () -> runForeground()); - assertThat(e.getCause().getMessage(), equalTo("interrupted while get jvm options")); - } - public void testEnvPassthrough() throws Exception { envVars.put("MY_ENV", "foo"); processValidator = pb -> { assertThat(pb.environment(), hasEntry(equalTo("MY_ENV"), equalTo("foo"))); }; @@ -276,83 +276,48 @@ public void testLibffiEnv() throws Exception { runForeground(); } - public void testTempDir() throws Exception { - optionsBuilder = (args, configDir, tmpDir, envOptions) -> { - assertThat(tmpDir.toString(), Files.exists(tmpDir), is(true)); - assertThat(tmpDir.getFileName().toString(), startsWith("elasticsearch-")); - return new ArrayList<>(); - }; - runForeground(); - } - - public void testTempDirWindows() throws Exception { - Path baseTmpDir = createTempDir(); - sysprops.put("os.name", "Windows 10"); - sysprops.put("java.io.tmpdir", baseTmpDir.toString()); - optionsBuilder = (args, configDir, tmpDir, envOptions) -> { - assertThat(tmpDir.toString(), Files.exists(tmpDir), is(true)); - assertThat(tmpDir.getFileName().toString(), equalTo("elasticsearch")); - assertThat(tmpDir.getParent().toString(), equalTo(baseTmpDir.toString())); - return new ArrayList<>(); - }; - runForeground(); - } - - public void testTempDirOverride() throws Exception { + public void testEnvCleared() throws Exception { Path customTmpDir = createTempDir(); envVars.put("ES_TMPDIR", customTmpDir.toString()); - optionsBuilder = (args, configDir, tmpDir, envOptions) -> { - assertThat(tmpDir.toString(), equalTo(customTmpDir.toString())); - return new ArrayList<>(); - }; - processValidator = pb -> assertThat(pb.environment(), not(hasKey("ES_TMPDIR"))); - runForeground(); - } - - public void testTempDirOverrideMissing() throws Exception { - Path baseDir = createTempDir(); - envVars.put("ES_TMPDIR", baseDir.resolve("dne").toString()); - var e = expectThrows(UserException.class, () -> runForeground()); - assertThat(e.exitCode, equalTo(ExitCodes.CONFIG)); - assertThat(e.getMessage(), containsString("dne] does not exist")); - } - - public void testTempDirOverrideNotADirectory() throws Exception { - Path tmpFile = createTempFile(); - envVars.put("ES_TMPDIR", tmpFile.toString()); - var e = expectThrows(UserException.class, () -> runForeground()); - assertThat(e.exitCode, equalTo(ExitCodes.CONFIG)); - assertThat(e.getMessage(), containsString("is not a directory")); - } - - public void testCustomJvmOptions() throws Exception { envVars.put("ES_JAVA_OPTS", "-Dmyoption=foo"); - optionsBuilder = (args, configDir, tmpDir, envOptions) -> { - assertThat(envOptions, equalTo("-Dmyoption=foo")); - return new ArrayList<>(); + + processValidator = pb -> { + assertThat(pb.environment(), not(hasKey("ES_TMPDIR"))); + assertThat(pb.environment(), not(hasKey("ES_JAVA_OPTS"))); }; - processValidator = pb -> assertThat(pb.environment(), not(hasKey("ES_JAVA_OPTS"))); runForeground(); } public void testCommandLineSysprops() throws Exception { - optionsBuilder = (args, configDir, tmpDir, envOptions) -> List.of("-Dfoo1=bar", "-Dfoo2=baz"); - processValidator = pb -> { - assertThat(pb.command(), contains("-Dfoo1=bar")); - assertThat(pb.command(), contains("-Dfoo2=bar")); + ServerProcessBuilder.ProcessStarter starter = pb -> { + assertThat(pb.command(), hasItems("-Dfoo1=bar", "-Dfoo2=baz")); + process = new MockElasticsearchProcess(); + return process; }; + var serverProcessBuilder = new ServerProcessBuilder().withTerminal(terminal) + .withProcessInfo(createProcessInfo()) + .withServerArgs(createServerArgs(false, false)) + .withJvmOptions(List.of("-Dfoo1=bar", "-Dfoo2=baz")) + .withTempDir(Path.of(".")); + serverProcessBuilder.start(starter).waitFor(); + } + + public void testServerProcessBuilderMissingArgumentError() throws Exception { + ServerProcessBuilder.ProcessStarter starter = pb -> new MockElasticsearchProcess(); + var serverProcessBuilder = new ServerProcessBuilder().withTerminal(terminal) + .withProcessInfo(createProcessInfo()) + .withServerArgs(createServerArgs(false, false)) + .withTempDir(Path.of(".")); + var ex = expectThrows(IllegalStateException.class, () -> serverProcessBuilder.start(starter).waitFor()); + assertThat(ex.getMessage(), equalTo("'jvmOptions' is a required argument and needs to be specified before calling start()")); } public void testCommandLine() throws Exception { String mainClass = "org.elasticsearch.server/org.elasticsearch.bootstrap.Elasticsearch"; - String distroSysprop = "-Des.distribution.type=testdistro"; String modulePath = esHomeDir.resolve("lib").toString(); Path javaBin = Paths.get("javahome").resolve("bin"); - sysprops.put("es.distribution.type", "testdistro"); AtomicReference expectedJava = new AtomicReference<>(javaBin.resolve("java").toString()); - processValidator = pb -> { - assertThat(pb.command(), hasItems(expectedJava.get(), distroSysprop, "--module-path", modulePath, "-m", mainClass)); - }; + processValidator = pb -> { assertThat(pb.command(), hasItems(expectedJava.get(), "--module-path", modulePath, "-m", mainClass)); }; runForeground(); sysprops.put("os.name", "Windows 10"); @@ -370,7 +335,7 @@ public void testDetach() throws Exception { // will block until stdin closed manually after test assertThat(stdin.read(), equalTo(-1)); }; - var server = startProcess(true, false, ""); + var server = startProcess(true, false); server.detach(); assertThat(terminal.getErrorOutput(), containsString("final message")); server.stop(); // this should be a noop, and will fail the stdin read assert above if shutdown sent @@ -384,7 +349,7 @@ public void testStop() throws Exception { nonInterruptibleVoid(mainReady::await); stderr.println("final message"); }; - var server = startProcess(false, false, ""); + var server = startProcess(false, false); mainReady.countDown(); server.stop(); assertThat(process.main.isDone(), is(true)); // stop should have waited @@ -399,7 +364,7 @@ public void testWaitFor() throws Exception { assertThat(stdin.read(), equalTo((int) BootstrapInfo.SERVER_SHUTDOWN_MARKER)); stderr.println("final message"); }; - var server = startProcess(false, false, ""); + var server = startProcess(false, false); new Thread(() -> { // simulate stop run as shutdown hook in another thread, eg from Ctrl-C nonInterruptibleVoid(mainReady::await); @@ -420,7 +385,7 @@ public void testProcessDies() throws Exception { nonInterruptibleVoid(mainExit::await); exitCode.set(-9); }; - var server = startProcess(false, false, ""); + var server = startProcess(false, false); mainExit.countDown(); int exitCode = server.waitFor(); assertThat(exitCode, equalTo(-9)); diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessUtilsTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessUtilsTests.java new file mode 100644 index 0000000000000..8cd1b63e41b03 --- /dev/null +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessUtilsTests.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.server.cli; + +import org.elasticsearch.cli.ExitCodes; +import org.elasticsearch.cli.ProcessInfo; +import org.elasticsearch.cli.UserException; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.startsWith; + +public class ServerProcessUtilsTests extends ESTestCase { + + protected final Map sysprops = new HashMap<>(); + protected final Map envVars = new HashMap<>(); + + @Before + public void resetEnv() { + sysprops.clear(); + sysprops.put("os.name", "Linux"); + sysprops.put("java.home", "javahome"); + envVars.clear(); + } + + private ProcessInfo createProcessInfo() { + return new ProcessInfo(Map.copyOf(sysprops), Map.copyOf(envVars), Path.of(".")); + } + + public void testTempDir() throws Exception { + var tmpDir = ServerProcessUtils.setupTempDir(createProcessInfo()); + assertThat(tmpDir.toString(), Files.exists(tmpDir), is(true)); + assertThat(tmpDir.getFileName().toString(), startsWith("elasticsearch-")); + } + + public void testTempDirWindows() throws Exception { + Path baseTmpDir = createTempDir(); + sysprops.put("os.name", "Windows 10"); + sysprops.put("java.io.tmpdir", baseTmpDir.toString()); + var tmpDir = ServerProcessUtils.setupTempDir(createProcessInfo()); + assertThat(tmpDir.toString(), Files.exists(tmpDir), is(true)); + assertThat(tmpDir.getFileName().toString(), equalTo("elasticsearch")); + assertThat(tmpDir.getParent().toString(), equalTo(baseTmpDir.toString())); + } + + public void testTempDirOverride() throws Exception { + Path customTmpDir = createTempDir(); + envVars.put("ES_TMPDIR", customTmpDir.toString()); + var tmpDir = ServerProcessUtils.setupTempDir(createProcessInfo()); + assertThat(tmpDir.toString(), equalTo(customTmpDir.toString())); + } + + public void testTempDirOverrideMissing() { + Path baseDir = createTempDir(); + envVars.put("ES_TMPDIR", baseDir.resolve("dne").toString()); + var e = expectThrows(UserException.class, () -> ServerProcessUtils.setupTempDir(createProcessInfo())); + assertThat(e.exitCode, equalTo(ExitCodes.CONFIG)); + assertThat(e.getMessage(), containsString("dne] does not exist")); + } + + public void testTempDirOverrideNotADirectory() throws Exception { + Path tmpFile = createTempFile(); + envVars.put("ES_TMPDIR", tmpFile.toString()); + var e = expectThrows(UserException.class, () -> ServerProcessUtils.setupTempDir(createProcessInfo())); + assertThat(e.exitCode, equalTo(ExitCodes.CONFIG)); + assertThat(e.getMessage(), containsString("is not a directory")); + } +} diff --git a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java index 858787b361654..2c42dcf5cb2f5 100644 --- a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java +++ b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java @@ -17,7 +17,10 @@ import org.elasticsearch.common.settings.KeyStoreWrapper; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.env.Environment; +import org.elasticsearch.server.cli.JvmOptionsParser; import org.elasticsearch.server.cli.ServerProcess; +import org.elasticsearch.server.cli.ServerProcessBuilder; +import org.elasticsearch.server.cli.ServerProcessUtils; /** * Starts an Elasticsearch process, but does not wait for it to exit. @@ -38,7 +41,14 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce // the Windows service daemon doesn't support secure settings implementations other than the keystore try (var loadedSecrets = KeyStoreWrapper.bootstrap(env.configFile(), () -> new SecureString(new char[0]))) { var args = new ServerArgs(false, true, null, loadedSecrets, env.settings(), env.configFile(), env.logsFile()); - this.server = ServerProcess.start(terminal, processInfo, args); + var tempDir = ServerProcessUtils.setupTempDir(processInfo); + var jvmOptions = JvmOptionsParser.determineJvmOptions(args, processInfo, tempDir); + var serverProcessBuilder = new ServerProcessBuilder().withTerminal(terminal) + .withProcessInfo(processInfo) + .withServerArgs(args) + .withTempDir(tempDir) + .withJvmOptions(jvmOptions); + this.server = serverProcessBuilder.start(); // start does not return until the server is ready, and we do not wait for the process } } diff --git a/docs/build.gradle b/docs/build.gradle index ddd2a38b5160b..b6f696f0aae6a 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -113,8 +113,9 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach { requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.12.0") - extraConfigFile 'op-jwks.json', project(':x-pack:test:idp-fixture').file("oidc/op-jwks.json") - extraConfigFile 'idp-docs-metadata.xml', project(':x-pack:test:idp-fixture').file("idp/shibboleth-idp/metadata/idp-docs-metadata.xml") + // TODO Rene: clean up this kind of cross project file references + extraConfigFile 'op-jwks.json', project(':x-pack:test:idp-fixture').file("src/main/resources/oidc/op-jwks.json") + extraConfigFile 'idp-docs-metadata.xml', project(':x-pack:test:idp-fixture').file("src/main/resources/idp/shibboleth-idp/metadata/idp-docs-metadata.xml") extraConfigFile 'testClient.crt', project(':x-pack:plugin:security').file("src/test/resources/org/elasticsearch/xpack/security/action/pki_delegation/testClient.crt") setting 'xpack.security.enabled', 'true' setting 'xpack.security.authc.api_key.enabled', 'true' diff --git a/docs/changelog/100740.yaml b/docs/changelog/100740.yaml new file mode 100644 index 0000000000000..c93fbf676ef81 --- /dev/null +++ b/docs/changelog/100740.yaml @@ -0,0 +1,6 @@ +pr: 100740 +summary: "ESQL: Referencing expressions that contain backticks requires <>." +area: ES|QL +type: enhancement +issues: + - 100312 diff --git a/docs/changelog/102862.yaml b/docs/changelog/102862.yaml new file mode 100644 index 0000000000000..bb453163009d5 --- /dev/null +++ b/docs/changelog/102862.yaml @@ -0,0 +1,5 @@ +pr: 102862 +summary: Add optional pruning configuration (weighted terms scoring) to text expansion query +area: "Machine Learning" +type: enhancement +issues: [] diff --git a/docs/changelog/102879.yaml b/docs/changelog/102879.yaml new file mode 100644 index 0000000000000..b35d36dd0a3a9 --- /dev/null +++ b/docs/changelog/102879.yaml @@ -0,0 +1,5 @@ +pr: 102879 +summary: Fix disk computation when initializing new shards +area: Allocation +type: bug +issues: [] diff --git a/docs/changelog/103099.yaml b/docs/changelog/103099.yaml new file mode 100644 index 0000000000000..c3fd3f9d7b8e4 --- /dev/null +++ b/docs/changelog/103099.yaml @@ -0,0 +1,6 @@ +pr: 103099 +summary: "ESQL: Simpify IS NULL/IS NOT NULL evaluation" +area: ES|QL +type: enhancement +issues: + - 103097 diff --git a/docs/changelog/103176.yaml b/docs/changelog/103176.yaml new file mode 100644 index 0000000000000..a0f46c1462f62 --- /dev/null +++ b/docs/changelog/103176.yaml @@ -0,0 +1,5 @@ +pr: 103176 +summary: Validate settings in `ReloadSecureSettings` API +area: Client +type: bug +issues: [] diff --git a/docs/changelog/103212.yaml b/docs/changelog/103212.yaml new file mode 100644 index 0000000000000..3cbbddc8f2229 --- /dev/null +++ b/docs/changelog/103212.yaml @@ -0,0 +1,5 @@ +pr: 103212 +summary: Use the eql query filter for the open-pit request +area: EQL +type: enhancement +issues: [] diff --git a/docs/changelog/103223.yaml b/docs/changelog/103223.yaml new file mode 100644 index 0000000000000..c2f4c1b6a2cf4 --- /dev/null +++ b/docs/changelog/103223.yaml @@ -0,0 +1,10 @@ +pr: 103223 +summary: "[Synonyms] Mark Synonyms as GA" +area: "Search" +type: feature +issues: [] +highlight: + title: "GA Release of Synonyms API" + body: |- + Removes the beta label for the Synonyms API to make it GA. + notable: true diff --git a/docs/changelog/103316.yaml b/docs/changelog/103316.yaml new file mode 100644 index 0000000000000..47eddcc34d924 --- /dev/null +++ b/docs/changelog/103316.yaml @@ -0,0 +1,5 @@ +pr: 103316 +summary: Review KEEP logic to prevent duplicate column names +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/103339.yaml b/docs/changelog/103339.yaml new file mode 100644 index 0000000000000..6ea1ab0cf799a --- /dev/null +++ b/docs/changelog/103339.yaml @@ -0,0 +1,6 @@ +pr: 103339 +summary: "ESQL: Fix resolution of MV_EXPAND after KEEP *" +area: ES|QL +type: bug +issues: + - 103331 diff --git a/docs/changelog/103361.yaml b/docs/changelog/103361.yaml new file mode 100644 index 0000000000000..441acc09895ef --- /dev/null +++ b/docs/changelog/103361.yaml @@ -0,0 +1,5 @@ +pr: 103361 +summary: Prevent attempts to access non-existent node information during rebalancing +area: Machine Learning +type: bug +issues: [ ] diff --git a/docs/changelog/103408.yaml b/docs/changelog/103408.yaml new file mode 100644 index 0000000000000..bf5081b854f08 --- /dev/null +++ b/docs/changelog/103408.yaml @@ -0,0 +1,6 @@ +pr: 103408 +summary: Cache component versions +area: Infra/Core +type: bug +issues: + - 102103 diff --git a/docs/changelog/103427.yaml b/docs/changelog/103427.yaml new file mode 100644 index 0000000000000..57a27aa687ab7 --- /dev/null +++ b/docs/changelog/103427.yaml @@ -0,0 +1,5 @@ +pr: 103427 +summary: "[Connector API] Fix bug with nullable tooltip field in parser" +area: Application +type: bug +issues: [] diff --git a/docs/changelog/103430.yaml b/docs/changelog/103430.yaml new file mode 100644 index 0000000000000..cd2444270849d --- /dev/null +++ b/docs/changelog/103430.yaml @@ -0,0 +1,5 @@ +pr: 103430 +summary: "[Connectors API] Fix bug with missing TEXT `DisplayType` enum" +area: Application +type: bug +issues: [] diff --git a/docs/changelog/103435.yaml b/docs/changelog/103435.yaml new file mode 100644 index 0000000000000..95e3c7169ada9 --- /dev/null +++ b/docs/changelog/103435.yaml @@ -0,0 +1,5 @@ +pr: 103435 +summary: Dispatch `ClusterStateAction#buildResponse` to executor +area: Distributed +type: bug +issues: [] diff --git a/docs/changelog/103474.yaml b/docs/changelog/103474.yaml new file mode 100644 index 0000000000000..a1da15a6bfbe5 --- /dev/null +++ b/docs/changelog/103474.yaml @@ -0,0 +1,6 @@ +pr: 103474 +summary: Fix now in millis for ESQL search contexts +area: ES|QL +type: bug +issues: + - 103455 diff --git a/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc new file mode 100644 index 0000000000000..6123b7eb5511d --- /dev/null +++ b/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc @@ -0,0 +1,50 @@ +[[cancel-connector-sync-job-api]] +=== Cancel connector sync job API +++++ +Cancel connector sync job +++++ + +Cancels a connector sync job. + +[[cancel-connector-sync-job-api-request]] +==== {api-request-title} +`PUT _connector/_sync_job//_cancel` + +[[cancel-connector-sync-job-api-prereqs]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. +* The `connector_sync_job_id` parameter should reference an existing connector sync job. + +[[cancel-connector-sync-job-api-desc]] +==== {api-description-title} + +Cancels a connector sync job, which sets the `status` to `cancelling` and updates `cancellation_requested_at` to the current time. +The connector service is then responsible for setting the `status` of connector sync jobs to `cancelled`. + +[[cancel-connector-sync-job-api-path-params]] +==== {api-path-parms-title} + +`connector_sync_job_id`:: +(Required, string) + +[[cancel-connector-sync-job-api-response-codes]] +==== {api-response-codes-title} + +`200`:: +Connector sync job cancellation was successfully requested. + +`404`:: +No connector sync job matching `connector_sync_job_id` could be found. + +[[cancel-connector-sync-job-api-example]] +==== {api-examples-title} + +The following example cancels the connector sync job with ID `my-connector-sync-job-id`: + +[source,console] +---- +PUT _connector/_sync_job/my-connector-sync-job-id/_cancel +---- +// TEST[skip:there's no way to clean up after creating a connector sync job, as we don't know the id ahead of time. Therefore, skip this test.] + diff --git a/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc new file mode 100644 index 0000000000000..04c8057e2c115 --- /dev/null +++ b/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc @@ -0,0 +1,48 @@ +[[check-in-connector-sync-job-api]] +=== Check in connector sync job API +++++ +Check in connector sync job +++++ + +Checks in a connector sync job (updates `last_seen` to the current time). + +[[check-in-connector-sync-job-api-request]] +==== {api-request-title} +`PUT _connector/_sync_job//_check_in/` + +[[check-in-connector-sync-job-api-prereqs]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. +* The `connector_sync_job_id` parameter should reference an existing connector sync job. + +[[check-in-connector-sync-job-api-desc]] +==== {api-description-title} + +Checks in a connector sync job and sets `last_seen` to the time right before updating it in the internal index. + +[[check-in-connector-sync-job-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[[check-in-connector-sync-job-api-response-codes]] +==== {api-response-codes-title} + +`200`:: +Connector sync job was successfully checked in. + +`404`:: +No connector sync job matching `connector_sync_job_id` could be found. + +[[check-in-connector-sync-job-api-example]] +==== {api-examples-title} + +The following example checks in the connector sync job `my-connector-sync-job`: + +[source,console] +---- +PUT _connector/_sync_job/my-connector-sync-job/_check_in +---- +// TEST[skip:there's no way to clean up after creating a connector sync job, as we don't know the id ahead of time. Therefore, skip this test.] diff --git a/docs/reference/connector/apis/connector-apis.asciidoc b/docs/reference/connector/apis/connector-apis.asciidoc index a777d5919f71a..e31eb7899f5d9 100644 --- a/docs/reference/connector/apis/connector-apis.asciidoc +++ b/docs/reference/connector/apis/connector-apis.asciidoc @@ -23,6 +23,9 @@ You can use these APIs to create, get, delete and update connectors. Use the following APIs to manage connectors: * <> +* <> +* <> +* <> [discrete] @@ -34,4 +37,23 @@ You can use these APIs to create, cancel, delete and update sync jobs. Use the following APIs to manage sync jobs: +* <> +* <> +* <> +* <> +* <> +* <> +* <> + + +include::cancel-connector-sync-job-api.asciidoc[] +include::check-in-connector-sync-job-api.asciidoc[] include::create-connector-api.asciidoc[] +include::create-connector-sync-job-api.asciidoc[] +include::delete-connector-api.asciidoc[] +include::delete-connector-sync-job-api.asciidoc[] +include::get-connector-api.asciidoc[] +include::get-connector-sync-job-api.asciidoc[] +include::list-connectors-api.asciidoc[] +include::list-connector-sync-jobs-api.asciidoc[] +include::set-connector-sync-job-error-api.asciidoc[] diff --git a/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc new file mode 100644 index 0000000000000..e8c2c364797c4 --- /dev/null +++ b/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc @@ -0,0 +1,69 @@ +[[create-connector-sync-job-api]] +=== Create connector sync job API +++++ +Create connector sync job +++++ + +Creates a connector sync job. + +[source, console] +-------------------------------------------------- +POST _connector/_sync_job +{ + "id": "connector-id", + "job_type": "full", + "trigger_method": "on_demand" +} +-------------------------------------------------- +// TEST[skip:there's no way to clean up after this code snippet, as we don't know the id ahead of time] + + +[[create-connector-sync-job-api-request]] +==== {api-request-title} +`POST _connector/_sync_job` + + +[[create-connector-sync-job-api-prereqs]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. +* The `id` parameter should reference an existing connector. + +[[create-connector-sync-job-api-desc]] +==== {api-description-title} + +Creates a connector sync job document in the internal index and initializes its counters and timestamps with default values. +Certain values can be updated via the API. + +[role="child_attributes"] +[[create-connector-sync-job-api-request-body]] +==== {api-request-body-title} + +`id`:: +(Required, string) The id of the connector to create the sync job for. + +`job_type`:: +(Optional, string) The job type of the created sync job. Defaults to `full`. + +`trigger_method`:: +(Optional, string) The trigger method of the created sync job. Defaults to `on_demand`. + + +[role="child_attributes"] +[[create-connector-sync-job-api-response-body]] +==== {api-response-body-title} + +`id`:: +(string) The ID associated with the connector sync job document. + +[[create-connector-sync-job-api-response-codes]] +==== {api-response-codes-title} + +`201`:: +Indicates that the connector sync job was created successfully. + +`400`:: +Indicates that the request was malformed. + +`404`:: +Indicates that either the index or the referenced connector is missing. diff --git a/docs/reference/connector/apis/delete-connector-api.asciidoc b/docs/reference/connector/apis/delete-connector-api.asciidoc new file mode 100644 index 0000000000000..2bda7da72cb72 --- /dev/null +++ b/docs/reference/connector/apis/delete-connector-api.asciidoc @@ -0,0 +1,66 @@ +[[delete-connector-api]] +=== Delete connector API + +preview::[] + +++++ +Delete connector +++++ + +Removes a connector and its associated data. +This is a destructive action that is not recoverable. + +[[delete-connector-api-request]] +==== {api-request-title} + +`DELETE _connector/` + +[[delete-connector-api-prereq]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. + +[[delete-connector-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[[delete-connector-api-response-codes]] +==== {api-response-codes-title} + +`400`:: +The `connector_id` was not provided. + +`404` (Missing resources):: +No connector matching `connector_id` could be found. + +[[delete-connector-api-example]] +==== {api-examples-title} + +The following example deletes the connector with ID `my-connector`: + +//// +[source, console] +-------------------------------------------------- +PUT _connector/my-connector +{ + "index_name": "search-google-drive", + "name": "My Connector", + "service_type": "google_drive" +} +-------------------------------------------------- +// TESTSETUP +//// + +[source,console] +---- +DELETE _connector/my-connector +---- + +[source,console-result] +---- +{ + "acknowledged": true +} +---- diff --git a/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc new file mode 100644 index 0000000000000..8641794576bf1 --- /dev/null +++ b/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc @@ -0,0 +1,54 @@ +[[delete-connector-sync-job-api]] +=== Delete connector sync job API + +preview::[] + +++++ +Delete connector sync job +++++ + +Removes a connector sync job and its associated data. +This is a destructive action that is not recoverable. + +[[delete-connector-sync-job-api-request]] +==== {api-request-title} + +`DELETE _connector/_sync_job/` + +[[delete-connector-sync-job-api-prereq]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. + +[[delete-connector-sync-job-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[[delete-connector-sync-job-api-response-codes]] +==== {api-response-codes-title} + +`400`:: +The `connector_sync_job_id` was not provided. + +`404`:: +No connector sync job matching `connector_sync_job_id` could be found. + +[[delete-connector-sync-job-api-example]] +==== {api-examples-title} + +The following example deletes the connector sync job with ID `my-connector-sync-job-id`: + +[source,console] +---- +DELETE _connector/_sync_job/my-connector-sync-job-id +---- +// TEST[skip:there's no way to clean up after this code snippet, as we don't know the ids of sync jobs ahead of time] + +[source,console-result] +---- +{ + "acknowledged": true +} +---- diff --git a/docs/reference/connector/apis/get-connector-api.asciidoc b/docs/reference/connector/apis/get-connector-api.asciidoc new file mode 100644 index 0000000000000..ab4a2758ce4f1 --- /dev/null +++ b/docs/reference/connector/apis/get-connector-api.asciidoc @@ -0,0 +1,63 @@ +[[get-connector-api]] +=== Get connector API +preview::[] +++++ +Get connector +++++ + +Retrieves the details about a connector. + +[[get-connector-api-request]] +==== {api-request-title} + +`GET _connector/` + +[[get-connector-api-prereq]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. + +[[get-connector-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[[get-connector-api-response-codes]] +==== {api-response-codes-title} + +`400`:: +The `connector_id` was not provided. + +`404` (Missing resources):: +No connector matching `connector_id` could be found. + +[[get-connector-api-example]] +==== {api-examples-title} + +The following example gets the connector `my-connector`: + +//// +[source,console] +-------------------------------------------------- +PUT _connector/my-connector +{ + "index_name": "search-google-drive", + "name": "Google Drive Connector", + "service_type": "google_drive" +} + +-------------------------------------------------- +// TESTSETUP + +[source,console] +-------------------------------------------------- +DELETE _connector/my-connector +-------------------------------------------------- +// TEARDOWN +//// + +[source,console] +---- +GET _connector/my-connector +---- diff --git a/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc new file mode 100644 index 0000000000000..b33aec8c55e60 --- /dev/null +++ b/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc @@ -0,0 +1,44 @@ +[[get-connector-sync-job-api]] +=== Get connector sync job API +preview::[] +++++ +Get connector sync job +++++ + +Retrieves the details about a connector sync job. + +[[get-connector-sync-job-api-request]] +==== {api-request-title} + +`GET _connector/_sync_job/` + +[[get-connector-sync-job-api-prereq]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. + +[[get-connector-sync-job-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[[get-connector-sync-job-api-response-codes]] +==== {api-response-codes-title} + +`400`:: +The `connector_sync_job_id` was not provided. + +`404` (Missing resources):: +No connector sync job matching `connector_sync_job_id` could be found. + +[[get-connector-sync-job-api-example]] +==== {api-examples-title} + +The following example gets the connector sync job `my-connector-sync-job`: + +[source,console] +---- +GET _connector/_sync_job/my-connector-sync-job +---- +// TEST[skip:there's no way to clean up after creating a connector sync job, as we don't know the id ahead of time. Therefore, skip this test.] diff --git a/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc b/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc new file mode 100644 index 0000000000000..8b88f318f5304 --- /dev/null +++ b/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc @@ -0,0 +1,80 @@ +[role="xpack"] +[[list-connector-sync-jobs-api]] +=== List connector sync jobs API + +preview::[] + +++++ +List connector sync jobs +++++ + +Returns information about all stored connector sync jobs ordered by their creation date in ascending order. + + +[[list-connector-sync-jobs-api-request]] +==== {api-request-title} + +`GET _connector/_sync_job` + +[[list-connector-sync-jobs-api-prereq]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. + +[[list-connector-sync-jobs-api-path-params]] +==== {api-path-parms-title} + +`size`:: +(Optional, integer) Maximum number of results to retrieve. Defaults to `100`. + +`from`:: +(Optional, integer) The offset from the first result to fetch. Defaults to `0`. + +`status`:: +(Optional, job status) The job status the fetched sync jobs need to have. + +`connector_id`:: +(Optional, string) The connector id the fetched sync jobs need to have. + +[[list-connector-sync-jobs-api-example]] +==== {api-examples-title} + +The following example lists all connector sync jobs: + + +[source,console] +---- +GET _connector/_sync_job +---- +// TEST[skip:there's no way to clean up after this code snippet, as we don't know the ids of sync jobs ahead of time] + +The following example lists the first two connector sync jobs: + +[source,console] +---- +GET _connector/_sync_job?from=0&size=2 +---- +// TEST[skip:there's no way to clean up after this code snippet, as we don't know the ids of sync jobs ahead of time] + +The following example lists pending connector sync jobs (the first 100 per default): +[source,console] +---- +GET _connector/_sync_job?status=pending +---- +// TEST[skip:there's no way to clean up after this code snippet, as we don't know the ids of sync jobs ahead of time] + +The following example lists connector sync jobs (the first 100 per default) for the connector with id `connector-1`: +[source,console] +---- +GET _connector/_sync_job?connector_id=connector-1 +---- +// TEST[skip:there's no way to clean up after this code snippet, as we don't know the ids of sync jobs ahead of time] + +[[list-connector-sync-jobs-api-response-codes]] +==== {api-response-codes-title} + +`200`: +Indicates that results were successfully returned (results can also be empty). + +`400`: +Indicates that the request was malformed. diff --git a/docs/reference/connector/apis/list-connectors-api.asciidoc b/docs/reference/connector/apis/list-connectors-api.asciidoc new file mode 100644 index 0000000000000..57d3cc47aeb7a --- /dev/null +++ b/docs/reference/connector/apis/list-connectors-api.asciidoc @@ -0,0 +1,77 @@ +[role="xpack"] +[[list-connector-api]] +=== List connectors API + +preview::[] + +++++ +List connectors +++++ + +Returns information about all stored connectors. + + +[[list-connector-api-request]] +==== {api-request-title} + +`GET _connector` + +[[list-connector-api-prereq]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. + +[[list-connector-api-path-params]] +==== {api-path-parms-title} + +`size`:: +(Optional, integer) Maximum number of results to retrieve. + +`from`:: +(Optional, integer) The offset from the first result to fetch. + +[[list-connector-api-example]] +==== {api-examples-title} + +The following example lists all connectors: + +//// +[source,console] +-------------------------------------------------- +PUT _connector/connector-1 +{ + "index_name": "search-google-drive", + "name": "Google Drive Connector", + "service_type": "google_drive" +} + +PUT _connector/connector-2 +{ + "index_name": "search-sharepoint-online", + "name": "Sharepoint Online Connector", + "service_type": "sharepoint_online" +} + +-------------------------------------------------- +// TESTSETUP + +[source,console] +-------------------------------------------------- +DELETE _connector/connector-1 + +DELETE _connector/connector-2 +-------------------------------------------------- +// TEARDOWN +//// + +[source,console] +---- +GET _connector +---- + +The following example lists the first two connectors: + +[source,console] +---- +GET _connector/?from=0&size=2 +---- diff --git a/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc b/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc new file mode 100644 index 0000000000000..935fcccc77fcf --- /dev/null +++ b/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc @@ -0,0 +1,58 @@ +[[set-connector-sync-job-error-api]] +=== Set connector sync job error API +++++ +Set connector sync job error +++++ + +Sets a connector sync job error. + +[[set-connector-sync-job-error-api-request]] +==== {api-request-title} +`PUT _connector/_sync_job//_error` + +[[set-connector-sync-job-error-api-prereqs]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. +* The `connector_sync_job_id` parameter should reference an existing connector sync job. + +[[set-connector-sync-job-error-api-desc]] +==== {api-description-title} + +Sets the `error` field for the specified connector sync job and sets its `status` to `error`. + +[[set-connector-sync-job-error-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[role="child_attributes"] +[[set-connector-sync-job-error-api-request-body]] +==== {api-request-body-title} + +`error`:: +(Required, string) The error to set the connector sync job `error` field to. + +[[set-connector-sync-job-api-response-codes]] +==== {api-response-codes-title} + +`200`:: +Indicates that the connector sync job error was set successfully. + +`404`:: +No connector sync job matching `connector_sync_job_id` could be found. + +[[set-connector-sync-job-error-api-example]] +==== {api-examples-title} + +The following example sets the error `some-error` in the connector sync job `my-connector-sync-job`: + +[source,console] +---- +PUT _connector/_sync_job/my-connector-sync-job/_error +{ + "error": "some-error" +} +---- +// TEST[skip:there's no way to clean up after creating a connector sync job, as we don't know the id ahead of time. Therefore, skip this test.] diff --git a/docs/reference/mapping/types/flattened.asciidoc b/docs/reference/mapping/types/flattened.asciidoc index 87f5cebe21993..0a72ebc98ecef 100644 --- a/docs/reference/mapping/types/flattened.asciidoc +++ b/docs/reference/mapping/types/flattened.asciidoc @@ -294,8 +294,8 @@ The following mapping parameters are accepted: <>:: A string value which is substituted for any explicit `null` values within - the flattened object field. Defaults to `null`, which means null sields are - treated as if it were missing. + the flattened object field. Defaults to `null`, which means null fields are + treated as if they were missing. <>:: diff --git a/docs/reference/query-dsl/text-expansion-query.asciidoc b/docs/reference/query-dsl/text-expansion-query.asciidoc index e924cc05376d9..46a9aafdd1af8 100644 --- a/docs/reference/query-dsl/text-expansion-query.asciidoc +++ b/docs/reference/query-dsl/text-expansion-query.asciidoc @@ -52,10 +52,42 @@ text. (Required, string) The query text you want to use for search. +`pruning_config` :::: +(Optional, object) +preview:[] +Optional pruning configuration. If enabled, this will omit non-significant tokens from the query in order to improve query performance. +Default: Disabled. ++ +-- +Parameters for `` are: + +`tokens_freq_ratio_threshold`:: +(Optional, float) +preview:[] +Tokens whose frequency is more than `tokens_freq_ratio_threshold` times the average frequency of all tokens in the specified field are considered outliers and pruned. +This value must between 1 and 100. +Default: `5`. + +`tokens_weight_threshold`:: +(Optional, float) +preview:[] +Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned. +This value must be between 0 and 1. +Default: `0.4`. + +`only_score_pruned_tokens`:: +(Optional, boolean) +preview:[] +If `true` we only input pruned tokens into scoring, and discard non-pruned tokens. +It is strongly recommended to set this to `false` for the main query, but this can be set to `true` for a rescore query to get more relevant results. +Default: `false`. + +NOTE: The default values for `tokens_freq_ratio_threshold` and `tokens_weight_threshold` were chosen based on tests using ELSER that provided the most optimal results. +-- [discrete] [[text-expansion-query-example]] -=== Example +=== Example ELSER query The following is an example of the `text_expansion` query that references the ELSER model to perform semantic search. For a more detailed description of how @@ -69,7 +101,7 @@ GET my-index/_search "query":{ "text_expansion":{ "ml.tokens":{ - "model_id":".elser_model_1", + "model_id":".elser_model_2", "model_text":"How is the weather in Jamaica?" } } @@ -78,7 +110,108 @@ GET my-index/_search ---- // TEST[skip: TBD] +[discrete] +[[text-expansion-query-with-pruning-config-example]] +=== Example ELSER query with pruning configuration + +The following is an extension to the above example that adds a preview:[] pruning configuration to the `text_expansion` query. +The pruning configuration identifies non-significant tokens to prune from the query in order to improve query performance. +[source,console] +---- +GET my-index/_search +{ + "query":{ + "text_expansion":{ + "ml.tokens":{ + "model_id":".elser_model_2", + "model_text":"How is the weather in Jamaica?" + }, + "pruning_config": { + "tokens_freq_ratio_threshold": 5, + "tokens_weight_threshold": 0.4, + "only_score_pruned_tokens": false + } + } + } +} +---- +// TEST[skip: TBD] + +[discrete] +[[text-expansion-query-with-pruning-config-and-rescore-example]] +=== Example ELSER query with pruning configuration and rescore + +The following is an extension to the above example that adds a <> function on top of the preview:[] pruning configuration to the `text_expansion` query. +The pruning configuration identifies non-significant tokens to prune from the query in order to improve query performance. +Rescoring the query with the tokens that were originally pruned from the query may improve overall search relevance when using this pruning strategy. + +[source,console] +---- +GET my-index/_search +{ + "query":{ + "text_expansion":{ + "ml.tokens":{ + "model_id":".elser_model_2", + "model_text":"How is the weather in Jamaica?" + }, + "pruning_config": { + "tokens_freq_ratio_threshold": 5, + "tokens_weight_threshold": 0.4, + "only_score_pruned_tokens": false + } + } + }, + "rescore": { + "window_size": 100, + "query": { + "rescore_query": { + "text_expansion": { + "ml.tokens": { + "model_id": ".elser_model_2", + "model_text": "How is the weather in Jamaica?" + }, + "pruning_config": { + "tokens_freq_ratio_threshold": 5, + "tokens_weight_threshold": 0.4, + "only_score_pruned_tokens": false + } + } + } + } + } +} +---- +//TEST[skip: TBD] + [NOTE] ==== Depending on your data, the text expansion query may be faster with `track_total_hits: false`. ==== + +[discrete] +[[weighted-tokens-query-example]] +=== Example Weighted token query + +In order to quickly iterate during tests, we exposed a new preview:[] `weighted_tokens` query for evaluation of tokenized datasets. +While this is not a query that is intended for production use, it can be used to quickly evaluate relevance using various pruning configurations. + +[source,console] +---- +POST /docs/_search +{ + "query": { + "weighted_tokens": { + "query_expansion": { + "tokens": {"2161": 0.4679, "2621": 0.307, "2782": 0.1299, "2851": 0.1056, "3088": 0.3041, "3376": 0.1038, "3467": 0.4873, "3684": 0.8958, "4380": 0.334, "4542": 0.4636, "4633": 2.2805, "4785": 1.2628, "4860": 1.0655, "5133": 1.0709, "7139": 1.0016, "7224": 0.2486, "7387": 0.0985, "7394": 0.0542, "8915": 0.369, "9156": 2.8947, "10505": 0.2771, "11464": 0.3996, "13525": 0.0088, "14178": 0.8161, "16893": 0.1376, "17851": 1.5348, "19939": 0.6012}, + "pruning_config": { + "tokens_freq_ratio_threshold": 5, + "tokens_weight_threshold": 0.4, + "only_score_pruned_tokens": false + } + } + } + } +} +---- +//TEST[skip: TBD] diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc index 340ef3a5c57c4..068cb3d2f127b 100644 --- a/docs/reference/release-notes.asciidoc +++ b/docs/reference/release-notes.asciidoc @@ -8,6 +8,7 @@ This section summarizes the changes in each release. * <> * <> +* <> * <> * <> * <> @@ -59,6 +60,7 @@ This section summarizes the changes in each release. include::release-notes/8.13.0.asciidoc[] include::release-notes/8.12.0.asciidoc[] +include::release-notes/8.11.3.asciidoc[] include::release-notes/8.11.2.asciidoc[] include::release-notes/8.11.1.asciidoc[] include::release-notes/8.11.0.asciidoc[] diff --git a/docs/reference/release-notes/8.11.3.asciidoc b/docs/reference/release-notes/8.11.3.asciidoc index 48ab82d0d4391..ddeb50dad1f75 100644 --- a/docs/reference/release-notes/8.11.3.asciidoc +++ b/docs/reference/release-notes/8.11.3.asciidoc @@ -1,6 +1,8 @@ [[release-notes-8.11.3]] == {es} version 8.11.3 +coming[8.11.3] + Also see <>. [[bug-8.11.3]] diff --git a/docs/reference/synonyms/apis/delete-synonym-rule.asciidoc b/docs/reference/synonyms/apis/delete-synonym-rule.asciidoc index eaff47f5d7909..74cbab8c0b4a2 100644 --- a/docs/reference/synonyms/apis/delete-synonym-rule.asciidoc +++ b/docs/reference/synonyms/apis/delete-synonym-rule.asciidoc @@ -1,8 +1,6 @@ [[delete-synonym-rule]] === Delete synonym rule -beta::[] - ++++ Delete synonym rule ++++ diff --git a/docs/reference/synonyms/apis/delete-synonyms-set.asciidoc b/docs/reference/synonyms/apis/delete-synonyms-set.asciidoc index 6ba4dcdc8f7be..9ba33ff3a5c75 100644 --- a/docs/reference/synonyms/apis/delete-synonyms-set.asciidoc +++ b/docs/reference/synonyms/apis/delete-synonyms-set.asciidoc @@ -1,8 +1,6 @@ [[delete-synonyms-set]] === Delete synonyms set -beta::[] - ++++ Delete synonyms set ++++ diff --git a/docs/reference/synonyms/apis/get-synonym-rule.asciidoc b/docs/reference/synonyms/apis/get-synonym-rule.asciidoc index 6ce978ae68ac6..c6c35e0efecca 100644 --- a/docs/reference/synonyms/apis/get-synonym-rule.asciidoc +++ b/docs/reference/synonyms/apis/get-synonym-rule.asciidoc @@ -1,8 +1,6 @@ [[get-synonym-rule]] === Get synonym rule -beta::[] - ++++ Get synonym rule ++++ diff --git a/docs/reference/synonyms/apis/get-synonyms-set.asciidoc b/docs/reference/synonyms/apis/get-synonyms-set.asciidoc index ddd7d2079dbf5..70bb5fb69526d 100644 --- a/docs/reference/synonyms/apis/get-synonyms-set.asciidoc +++ b/docs/reference/synonyms/apis/get-synonyms-set.asciidoc @@ -1,8 +1,6 @@ [[get-synonyms-set]] === Get synonyms set -beta::[] - ++++ Get synonyms set ++++ diff --git a/docs/reference/synonyms/apis/list-synonyms-sets.asciidoc b/docs/reference/synonyms/apis/list-synonyms-sets.asciidoc index 2522542886d9e..705a24c809e99 100644 --- a/docs/reference/synonyms/apis/list-synonyms-sets.asciidoc +++ b/docs/reference/synonyms/apis/list-synonyms-sets.asciidoc @@ -1,8 +1,6 @@ [[list-synonyms-sets]] === List synonyms sets -beta::[] - ++++ List synonyms sets ++++ diff --git a/docs/reference/synonyms/apis/put-synonym-rule.asciidoc b/docs/reference/synonyms/apis/put-synonym-rule.asciidoc index 95492c95d36fe..de2865632d55e 100644 --- a/docs/reference/synonyms/apis/put-synonym-rule.asciidoc +++ b/docs/reference/synonyms/apis/put-synonym-rule.asciidoc @@ -1,8 +1,6 @@ [[put-synonym-rule]] === Create or update synonym rule -beta::[] - ++++ Create or update synonym rule ++++ diff --git a/docs/reference/synonyms/apis/put-synonyms-set.asciidoc b/docs/reference/synonyms/apis/put-synonyms-set.asciidoc index a3c06c70db17b..5651c4c99adcd 100644 --- a/docs/reference/synonyms/apis/put-synonyms-set.asciidoc +++ b/docs/reference/synonyms/apis/put-synonyms-set.asciidoc @@ -1,8 +1,6 @@ [[put-synonyms-set]] === Create or update synonyms set -beta::[] - ++++ Create or update synonyms set ++++ diff --git a/docs/reference/synonyms/apis/synonyms-apis.asciidoc b/docs/reference/synonyms/apis/synonyms-apis.asciidoc index 6849477177dcf..9b92ba8e8579d 100644 --- a/docs/reference/synonyms/apis/synonyms-apis.asciidoc +++ b/docs/reference/synonyms/apis/synonyms-apis.asciidoc @@ -1,8 +1,6 @@ [[synonyms-apis]] == Synonyms APIs -beta::[] - ++++ Synonyms APIs ++++ diff --git a/docs/reference/troubleshooting/network-timeouts.asciidoc b/docs/reference/troubleshooting/network-timeouts.asciidoc index ab60eeff1b1a9..1920dafe62210 100644 --- a/docs/reference/troubleshooting/network-timeouts.asciidoc +++ b/docs/reference/troubleshooting/network-timeouts.asciidoc @@ -34,9 +34,9 @@ end::troubleshooting-network-timeouts-packet-capture-fault-detection[] tag::troubleshooting-network-timeouts-threads[] * Long waits for particular threads to be available can be identified by taking -stack dumps (for example, using `jstack`) or a profiling trace (for example, -using Java Flight Recorder) in the few seconds leading up to the relevant log -message. +stack dumps of the main {es} process (for example, using `jstack`) or a +profiling trace (for example, using Java Flight Recorder) in the few seconds +leading up to the relevant log message. + The <> API sometimes yields useful information, but bear in mind that this API also requires a number of `transport_worker` and diff --git a/modules/apm/METERING.md b/modules/apm/METERING.md new file mode 100644 index 0000000000000..0f5fcc977295d --- /dev/null +++ b/modules/apm/METERING.md @@ -0,0 +1,142 @@ +# Metrics in Elasticsearch + +Elasticsearch has the metrics API available in server's package +`org.elasticsearch.telemetry.metric`. +This package contains base classes/interfaces for creating and working with metrics. +Please refer to the javadocs provided in these classes in that package for more details. +The entry point for working with metrics is `MeterRegistry`. + +## Implementation +We use elastic's apm-java-agent as an implementation of the API we expose. +the implementation can be found in `:modules:apm` +The apm-java-agent is responsible for buffering metrics and upon metrics_interval +send them over to apm server. +Metrics_interval is configured via a `tracing.apm.agent.metrics_interval` setting +The agent also collects a number of JVM metrics. +see https://www.elastic.co/guide/en/apm/agent/java/current/metrics.html#metrics-jvm + + +## How to choose an instrument + +The choice of the right instrument is not always easy as often differences are subtle. +The simplified algorithm could be as follows: + +1. You want to measure something (absolute value) + 1. values are non-additive + 1. use a gauge + 2. Example: a cpu temperature + 2. values are additive + 1. use asynchronous counter + 2. Example: total number of requests +2. You want to count something + 1. values are monotonously increasing + 1. use a counter + 2. Example: Recording a failed authentication count + 2. values can be decreased + 1. use UpDownCounter + 2. Example: Number of orders in a queue +3. You want to record a statistics + 1. use a histogram + 1. Example: A statistics about how long it took to access a value from cache + +refer to https://opentelemetry.io/docs/specs/otel/metrics/supplementary-guidelines/#instrument-selection +for more details + +## How to name an instrument +See the naming guidelines for metrics: +[NAMING GUIDE](NAMING.md) + +### Restarts and overflows +if the instrument is correctly chosen, the apm server will be able to determine if the metrics +were restarted (i.e. node was restarted) or there was a counter overflow +(the metric in ES might use an int internally, but apm backend might have a long ) + +## How to use an instrument +There are 2 types of usages of an instrument depending on a type. +- For synchronous instrument (counter/UpDownCounter) we need to register an instrument with + `MeterRegistry` and use the returned value to increment a value of that instrument +```java + MeterRegistry registry; + LongCounter longCounter = registry.registerLongCounter("es.test.requests.count", "a test counter", "count"); + longCounter.increment(); + longCounter.incrementBy(1, Map.of("name", "Alice")); + longCounter.incrementBy(1, Map.of("name", "Bob")); +``` + +- For asynchronous instrument (gauge/AsynchronousCounter) we register an instrument + and have to provide a callback that will report the absolute measured value. + This callback has to be provided upon registration and cannot be changed. +```java +MeterRegistry registry; +long someValue = 1; +registry.registerLongGauge("es.test.cpu.temperature", "the current CPU temperature as measured by psensor", "degrees Celsius", +() -> new LongWithAttributes(someValue, Map.of("cpuNumber", 1))); +``` + +If we don’t have access to ‘state’ that will be fetched on metric event (when callback is executed) +we can use a utility LongGaugeMetric or LongGaugeMetric +```java +MeterRegistry meterRegistry ; +LongGaugeMetric longGaugeMetric = LongGaugeMetric.create(meterRegistry, "es.test.gauge", "a test gauge", "total value"); +longGaugeMetric.set(123L); +``` +### The use of attributes aka dimensions +Each instrument can attach attributes to a reported value. This helps drilling down into the details +of value that was reported during the metric event + + +## Development + +### Mock http server + +The quickest way to verify that your metrics are working is to run `./gradlew run --with-apm-server`. +This will run ES node (or nodes in serverless) and also start a mock http server that will act +as an apm server. This fake http server will log all the http messages it receives from apm-agent + +### With APM server in cloud +You can also run local ES node with an apm server in cloud. +Create a new deployment in cloud, then click the 'hamburger' on the left, scroll to Observability and click APM under it. +At the upper right corner there is `Add data` link, then scroll down to `ApmAgents` section and pick Java +There you should be able to see `elastic.apm.secret_token` and `elastic.apm.server_url. You will use them in the next step. + +edit your `~/.gradle/init.d/apm.gradle` and replace the secret_token and the server_url. +```groovy +rootProject { + if (project.name == 'elasticsearch' && Boolean.getBoolean('metrics.enabled')) { + afterEvaluate { + testClusters.matching { it.name == "runTask" }.configureEach { + setting 'xpack.security.audit.enabled', 'true' + keystore 'tracing.apm.secret_token', 'TODO-REPLACE' + setting 'telemetry.metrics.enabled', 'true' + setting 'tracing.apm.agent.server_url', 'https://TODO-REPLACE-URL.apm.eastus2.staging.azure.foundit.no:443' + } + } + } +} +``` + +The example use: +``` +./gradlew :run -Dmetrics.enabled=true +``` + +#### Logging +with any approach you took to run your ES with APM you will find apm-agent.json file +in ES's logs directory. If there are any problems with connecting to APM you will see WARN/ERROR messages. +We run apm-agent with logs at WARN level, so normally you should not see any logs there. + +When running ES in cloud, logs are being also indexed in a logging cluster, so you will be able to find them +in kibana. The `datastream.dataset` is `elasticsearch.apm_agent` + + +### Testing +We currently provide a base `TestTelemetryPlugin` which should help you write an integration test. +See an example `S3BlobStoreRepositoryTests` + + + + +# Links and further reading +https://opentelemetry.io/docs/specs/otel/metrics/supplementary-guidelines/ + +https://www.elastic.co/guide/en/apm/guide/current/data-model-metrics.html diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java index 61ef5f1973854..3c869a89cfaa9 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java @@ -11,6 +11,8 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.elasticsearch.ESNetty4IntegTestCase; +import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest; +import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.MockLogAppender; @@ -19,6 +21,7 @@ import org.elasticsearch.transport.TransportLogger; import java.io.IOException; +import java.util.concurrent.TimeUnit; @ESIntegTestCase.ClusterScope(numDataNodes = 2, scope = ESIntegTestCase.Scope.TEST) public class ESLoggingHandlerIT extends ESNetty4IntegTestCase { @@ -84,7 +87,7 @@ public void testLoggingHandler() { appender.addExpectation(writeExpectation); appender.addExpectation(flushExpectation); appender.addExpectation(readExpectation); - clusterAdmin().prepareNodesHotThreads().get(); + client().execute(TransportNodesHotThreadsAction.TYPE, new NodesHotThreadsRequest()).actionGet(10, TimeUnit.SECONDS); appender.assertAllExpectationsMatched(); } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.post.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.post.json index e76124bbecf7d..aadb59e99af7a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.post.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.post.json @@ -26,7 +26,7 @@ }, "body": { "description": "The connector configuration.", - "required": false + "required": true } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.delete_synonym.json b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.delete_synonym.json index 00142ebcf00fc..9273a8dea87c3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.delete_synonym.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.delete_synonym.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/delete-synonyms-set.html", "description": "Deletes a synonym set" }, - "stability": "experimental", + "stability": "stable", "visibility": "public", "headers": { "accept": [ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.delete_synonym_rule.json b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.delete_synonym_rule.json index 11fb113d6b629..5a0de4ab94a7c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.delete_synonym_rule.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.delete_synonym_rule.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/delete-synonym-rule.html", "description": "Deletes a synonym rule in a synonym set" }, - "stability": "experimental", + "stability": "stable", "visibility": "public", "headers": { "accept": [ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonym.json b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonym.json index 6cb4fcc46f26b..25c177cabbdf1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonym.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonym.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/get-synonyms-set.html", "description": "Retrieves a synonym set" }, - "stability": "experimental", + "stability": "stable", "visibility": "public", "headers": { "accept": [ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonym_rule.json b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonym_rule.json index 5a718f1a48e46..ff9e7eb57b8a7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonym_rule.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonym_rule.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/get-synonym-rule.html", "description": "Retrieves a synonym rule from a synonym set" }, - "stability": "experimental", + "stability": "stable", "visibility": "public", "headers": { "accept": [ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonyms_sets.json b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonyms_sets.json index 66bd8df92e1e7..d94bef32cddcd 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonyms_sets.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonyms_sets.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/list-synonyms-sets.html", "description": "Retrieves a summary of all defined synonym sets" }, - "stability": "experimental", + "stability": "stable", "visibility": "public", "headers": { "accept": [ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.put_synonym.json b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.put_synonym.json index 6c412d174434b..e09bbb7e428a1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.put_synonym.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.put_synonym.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/put-synonyms-set.html", "description": "Creates or updates a synonyms set" }, - "stability": "experimental", + "stability": "stable", "visibility": "public", "headers": { "accept": [ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.put_synonym_rule.json b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.put_synonym_rule.json index 082432ae662f0..51503b5819862 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.put_synonym_rule.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.put_synonym_rule.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/put-synonym-rule.html", "description": "Creates or updates a synonym rule in a synonym set" }, - "stability": "experimental", + "stability": "stable", "visibility": "public", "headers": { "accept": [ diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java index 45906abd29ff8..5fa63aaed0508 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java @@ -11,8 +11,9 @@ import org.apache.lucene.util.Constants; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequestBuilder; +import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest; import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse; +import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.logging.ChunkedLoggingStreamTests; import org.elasticsearch.core.TimeValue; @@ -23,7 +24,7 @@ import java.util.Map; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; @@ -41,38 +42,26 @@ public class HotThreadsIT extends ESIntegTestCase { - public void testHotThreadsDontFail() throws ExecutionException, InterruptedException { - /** - * This test just checks if nothing crashes or gets stuck etc. - */ + public void testHotThreadsDontFail() throws InterruptedException { + // This test just checks if nothing crashes or gets stuck etc. createIndex("test"); final int iters = scaledRandomIntBetween(2, 20); final AtomicBoolean hasErrors = new AtomicBoolean(false); for (int i = 0; i < iters; i++) { - final String type; - NodesHotThreadsRequestBuilder nodesHotThreadsRequestBuilder = clusterAdmin().prepareNodesHotThreads(); + final NodesHotThreadsRequest request = new NodesHotThreadsRequest(); if (randomBoolean()) { TimeValue timeValue = new TimeValue(rarely() ? randomIntBetween(500, 5000) : randomIntBetween(20, 500)); - nodesHotThreadsRequestBuilder.setInterval(timeValue); + request.interval(timeValue); } if (randomBoolean()) { - nodesHotThreadsRequestBuilder.setThreads(rarely() ? randomIntBetween(500, 5000) : randomIntBetween(1, 500)); + request.threads(rarely() ? randomIntBetween(500, 5000) : randomIntBetween(1, 500)); } - nodesHotThreadsRequestBuilder.setIgnoreIdleThreads(randomBoolean()); + request.ignoreIdleThreads(randomBoolean()); if (randomBoolean()) { - type = switch (randomIntBetween(0, 3)) { - case 3 -> "mem"; - case 2 -> "cpu"; - case 1 -> "wait"; - default -> "block"; - }; - assertThat(type, notNullValue()); - nodesHotThreadsRequestBuilder.setType(HotThreads.ReportType.of(type)); - } else { - type = null; + request.type(HotThreads.ReportType.of(randomFrom("block", "mem", "cpu", "wait"))); } final CountDownLatch latch = new CountDownLatch(1); - nodesHotThreadsRequestBuilder.execute(new ActionListener() { + client().execute(TransportNodesHotThreadsAction.TYPE, request, new ActionListener<>() { @Override public void onResponse(NodesHotThreadsResponse nodeHotThreads) { boolean success = false; @@ -83,7 +72,6 @@ public void onResponse(NodesHotThreadsResponse nodeHotThreads) { assertThat(nodesMap.size(), equalTo(cluster().size())); for (NodeHotThreads ht : nodeHotThreads.getNodes()) { assertNotNull(ht.getHotThreads()); - // logger.info(ht.getHotThreads()); } success = true; } finally { @@ -120,40 +108,39 @@ public void onFailure(Exception e) { 3L ); } - latch.await(); + safeAwait(latch); assertThat(hasErrors.get(), is(false)); } } - public void testIgnoreIdleThreads() throws ExecutionException, InterruptedException { + public void testIgnoreIdleThreads() { assumeTrue("no support for hot_threads on FreeBSD", Constants.FREE_BSD == false); // First time, don't ignore idle threads: - NodesHotThreadsRequestBuilder builder = clusterAdmin().prepareNodesHotThreads(); - builder.setIgnoreIdleThreads(false); - builder.setThreads(Integer.MAX_VALUE); - NodesHotThreadsResponse response = builder.execute().get(); + final NodesHotThreadsResponse firstResponse = client().execute( + TransportNodesHotThreadsAction.TYPE, + new NodesHotThreadsRequest().ignoreIdleThreads(false).threads(Integer.MAX_VALUE) + ).actionGet(10, TimeUnit.SECONDS); final Matcher containsCachedTimeThreadRunMethod = containsString( "org.elasticsearch.threadpool.ThreadPool$CachedTimeThread.run" ); int totSizeAll = 0; - for (NodeHotThreads node : response.getNodesMap().values()) { + for (NodeHotThreads node : firstResponse.getNodesMap().values()) { totSizeAll += node.getHotThreads().length(); assertThat(node.getHotThreads(), containsCachedTimeThreadRunMethod); } // Second time, do ignore idle threads: - builder = clusterAdmin().prepareNodesHotThreads(); - builder.setThreads(Integer.MAX_VALUE); - + final var request = new NodesHotThreadsRequest().threads(Integer.MAX_VALUE); // Make sure default is true: - assertEquals(true, builder.request().ignoreIdleThreads()); - response = builder.execute().get(); + assertTrue(request.ignoreIdleThreads()); + final NodesHotThreadsResponse secondResponse = client().execute(TransportNodesHotThreadsAction.TYPE, request) + .actionGet(10, TimeUnit.SECONDS); int totSizeIgnoreIdle = 0; - for (NodeHotThreads node : response.getNodesMap().values()) { + for (NodeHotThreads node : secondResponse.getNodesMap().values()) { totSizeIgnoreIdle += node.getHotThreads().length(); assertThat(node.getHotThreads(), not(containsCachedTimeThreadRunMethod)); } @@ -162,22 +149,26 @@ public void testIgnoreIdleThreads() throws ExecutionException, InterruptedExcept assertThat(totSizeIgnoreIdle, lessThan(totSizeAll)); } - public void testTimestampAndParams() throws ExecutionException, InterruptedException { + public void testTimestampAndParams() { - NodesHotThreadsResponse response = clusterAdmin().prepareNodesHotThreads().execute().get(); + final NodesHotThreadsResponse response = client().execute(TransportNodesHotThreadsAction.TYPE, new NodesHotThreadsRequest()) + .actionGet(10, TimeUnit.SECONDS); if (Constants.FREE_BSD) { for (NodeHotThreads node : response.getNodesMap().values()) { - String result = node.getHotThreads(); - assertTrue(result.indexOf("hot_threads is not supported") != -1); + assertThat(node.getHotThreads(), containsString("hot_threads is not supported")); } } else { for (NodeHotThreads node : response.getNodesMap().values()) { - String result = node.getHotThreads(); - assertTrue(result.indexOf("Hot threads at") != -1); - assertTrue(result.indexOf("interval=500ms") != -1); - assertTrue(result.indexOf("busiestThreads=3") != -1); - assertTrue(result.indexOf("ignoreIdleThreads=true") != -1); + assertThat( + node.getHotThreads(), + allOf( + containsString("Hot threads at"), + containsString("interval=500ms"), + containsString("busiestThreads=3"), + containsString("ignoreIdleThreads=true") + ) + ); } } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java index ad17e4f0d49dd..4aa3598608fb6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java @@ -11,10 +11,13 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.reload.NodesReloadSecureSettingsResponse; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.KeyStoreWrapper; +import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.SecureSettings; import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Plugin; @@ -43,6 +46,8 @@ @ESIntegTestCase.ClusterScope(minNumDataNodes = 2) public class ReloadSecureSettingsIT extends ESIntegTestCase { + private static final String VALID_SECURE_SETTING_NAME = "some.setting.that.exists"; + @BeforeClass public static void disableInFips() { // Reload secure settings with a password protected keystore is tested in ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT @@ -350,9 +355,46 @@ public void testReloadWhileKeystoreChanged() throws Exception { } } + public void testInvalidKeyInSettings() throws Exception { + final Environment environment = internalCluster().getInstance(Environment.class); + + try (KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create()) { + keyStoreWrapper.setString(VALID_SECURE_SETTING_NAME, new char[0]); + keyStoreWrapper.save(environment.configFile(), new char[0], false); + } + + PlainActionFuture actionFuture = new PlainActionFuture<>(); + clusterAdmin().prepareReloadSecureSettings() + .setSecureStorePassword(new SecureString(new char[0])) + .setNodesIds(Strings.EMPTY_ARRAY) + .execute(actionFuture); + + actionFuture.get().getNodes().forEach(nodeResponse -> assertThat(nodeResponse.reloadException(), nullValue())); + + try (KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create()) { + assertThat(keyStoreWrapper, notNullValue()); + keyStoreWrapper.setString("some.setting.that.does.not.exist", new char[0]); + keyStoreWrapper.save(environment.configFile(), new char[0], false); + } + + actionFuture = new PlainActionFuture<>(); + clusterAdmin().prepareReloadSecureSettings() + .setSecureStorePassword(new SecureString(new char[0])) + .setNodesIds(Strings.EMPTY_ARRAY) + .execute(actionFuture); + + actionFuture.get() + .getNodes() + .forEach(nodeResponse -> assertThat(nodeResponse.reloadException(), instanceOf(IllegalArgumentException.class))); + } + @Override protected Collection> nodePlugins() { - final List> plugins = Arrays.asList(MockReloadablePlugin.class, MisbehavingReloadablePlugin.class); + final List> plugins = Arrays.asList( + MockWithSecureSettingPlugin.class, + MockReloadablePlugin.class, + MisbehavingReloadablePlugin.class + ); // shuffle as reload is called in order Collections.shuffle(plugins, random()); return plugins; @@ -455,4 +497,10 @@ public synchronized void setShouldThrow(boolean shouldThrow) { } } + public static class MockWithSecureSettingPlugin extends Plugin { + public List> getSettings() { + return List.of(SecureSetting.secureString(VALID_SECURE_SETTING_NAME, null)); + } + }; + } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java index 595788b1eb9f5..eaf8948348684 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java @@ -37,7 +37,7 @@ public void testPendingTasksWithIndexBlocks() { )) { try { enableIndexBlock("test", blockSetting); - PendingClusterTasksResponse response = clusterAdmin().preparePendingClusterTasks().get(); + PendingClusterTasksResponse response = getClusterPendingTasks(); assertNotNull(response.pendingTasks()); } finally { disableIndexBlock("test", blockSetting); @@ -53,7 +53,7 @@ public void testPendingTasksWithClusterReadOnlyBlock() { try { setClusterReadOnly(true); - PendingClusterTasksResponse response = clusterAdmin().preparePendingClusterTasks().get(); + PendingClusterTasksResponse response = getClusterPendingTasks(); assertNotNull(response.pendingTasks()); } finally { setClusterReadOnly(false); @@ -80,7 +80,7 @@ public boolean validateClusterForming() { } }); - assertNotNull(clusterAdmin().preparePendingClusterTasks().get().pendingTasks()); + assertNotNull(getClusterPendingTasks().pendingTasks()); // starting one more node allows the cluster to recover internalCluster().startNode(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java index b1ac5b02f7dd2..c044fafe31efc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; -import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterInfoServiceUtils; import org.elasticsearch.cluster.DiskUsageIntegTestCase; @@ -39,13 +38,16 @@ import org.hamcrest.TypeSafeMatcher; import java.util.Arrays; +import java.util.Comparator; import java.util.HashSet; -import java.util.Locale; +import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; -import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toSet; +import static org.elasticsearch.cluster.routing.RoutingNodesHelper.numberOfShardsWithState; +import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING; import static org.elasticsearch.index.store.Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.empty; @@ -74,26 +76,25 @@ public void testHighWatermarkNotExceeded() throws Exception { final String dataNodeName = internalCluster().startDataOnlyNode(); ensureStableCluster(3); - final InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) internalCluster().getCurrentMasterNodeInstance( - ClusterInfoService.class - ); - internalCluster().getCurrentMasterNodeInstance(ClusterService.class) - .addListener(event -> ClusterInfoServiceUtils.refresh(clusterInfoService)); + final InternalClusterInfoService clusterInfoService = getInternalClusterInfoService(); + internalCluster().getCurrentMasterNodeInstance(ClusterService.class).addListener(event -> { + ClusterInfoServiceUtils.refresh(clusterInfoService); + }); final String dataNode0Id = internalCluster().getInstance(NodeEnvironment.class, dataNodeName).nodeId(); - final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + final String indexName = randomIdentifier(); createIndex(indexName, indexSettings(6, 0).put(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), "0ms").build()); - var smallestShard = createReasonableSizedShards(indexName); + var shardSizes = createReasonableSizedShards(indexName); // reduce disk size of node 0 so that no shards fit below the high watermark, forcing all shards onto the other data node // (subtract the translog size since the disk threshold decider ignores this and may therefore move the shard back again) - getTestFileStore(dataNodeName).setTotalSpace(smallestShard.size + WATERMARK_BYTES - 1L); + getTestFileStore(dataNodeName).setTotalSpace(shardSizes.getSmallestShardSize() + WATERMARK_BYTES - 1L); assertBusyWithDiskUsageRefresh(dataNode0Id, indexName, empty()); // increase disk size of node 0 to allow just enough room for one shard, and check that it's rebalanced back - getTestFileStore(dataNodeName).setTotalSpace(smallestShard.size + WATERMARK_BYTES); - assertBusyWithDiskUsageRefresh(dataNode0Id, indexName, new ContainsExactlyOneOf<>(smallestShard.shardIds)); + getTestFileStore(dataNodeName).setTotalSpace(shardSizes.getSmallestShardSize() + WATERMARK_BYTES); + assertBusyWithDiskUsageRefresh(dataNode0Id, indexName, new ContainsExactlyOneOf<>(shardSizes.getSmallestShardIds())); } public void testRestoreSnapshotAllocationDoesNotExceedWatermark() throws Exception { @@ -108,17 +109,20 @@ public void testRestoreSnapshotAllocationDoesNotExceedWatermark() throws Excepti .setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", randomBoolean())) ); - final InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) internalCluster().getCurrentMasterNodeInstance( - ClusterInfoService.class - ); - internalCluster().getCurrentMasterNodeInstance(ClusterService.class) - .addListener(event -> ClusterInfoServiceUtils.refresh(clusterInfoService)); + final AtomicBoolean allowRelocations = new AtomicBoolean(true); + final InternalClusterInfoService clusterInfoService = getInternalClusterInfoService(); + internalCluster().getCurrentMasterNodeInstance(ClusterService.class).addListener(event -> { + ClusterInfoServiceUtils.refresh(clusterInfoService); + if (allowRelocations.get() == false) { + assertThat(numberOfShardsWithState(event.state().getRoutingNodes(), ShardRoutingState.RELOCATING), equalTo(0)); + } + }); final String dataNode0Id = internalCluster().getInstance(NodeEnvironment.class, dataNodeName).nodeId(); - final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + final String indexName = randomIdentifier(); createIndex(indexName, indexSettings(6, 0).put(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), "0ms").build()); - var smallestShard = createReasonableSizedShards(indexName); + var shardSizes = createReasonableSizedShards(indexName); final CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("repo", "snap") .setWaitForCompletion(true) @@ -128,15 +132,13 @@ public void testRestoreSnapshotAllocationDoesNotExceedWatermark() throws Excepti assertThat(snapshotInfo.state(), is(SnapshotState.SUCCESS)); assertAcked(indicesAdmin().prepareDelete(indexName).get()); + updateClusterSettings(Settings.builder().put(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), Rebalance.NONE.toString())); + allowRelocations.set(false); // reduce disk size of node 0 so that no shards fit below the low watermark, forcing shards to be assigned to the other data node - getTestFileStore(dataNodeName).setTotalSpace(smallestShard.size + WATERMARK_BYTES - 1L); + getTestFileStore(dataNodeName).setTotalSpace(shardSizes.getSmallestShardSize() + WATERMARK_BYTES - 1L); refreshDiskUsage(); - updateClusterSettings( - Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), Rebalance.NONE.toString()) - ); - final RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("repo", "snap") .setWaitForCompletion(true) .get(); @@ -144,13 +146,71 @@ public void testRestoreSnapshotAllocationDoesNotExceedWatermark() throws Excepti assertThat(restoreInfo.successfulShards(), is(snapshotInfo.totalShards())); assertThat(restoreInfo.failedShards(), is(0)); - assertBusy(() -> assertThat(getShardIds(dataNode0Id, indexName), empty())); + assertThat(getShardIds(dataNode0Id, indexName), empty()); - updateClusterSettings(Settings.builder().putNull(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey())); + allowRelocations.set(true); + updateClusterSettings(Settings.builder().putNull(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey())); // increase disk size of node 0 to allow just enough room for one shard, and check that it's rebalanced back - getTestFileStore(dataNodeName).setTotalSpace(smallestShard.size + WATERMARK_BYTES); - assertBusyWithDiskUsageRefresh(dataNode0Id, indexName, new ContainsExactlyOneOf<>(smallestShard.shardIds)); + getTestFileStore(dataNodeName).setTotalSpace(shardSizes.getSmallestShardSize() + WATERMARK_BYTES); + assertBusyWithDiskUsageRefresh(dataNode0Id, indexName, new ContainsExactlyOneOf<>(shardSizes.getSmallestShardIds())); + } + + public void testRestoreSnapshotAllocationDoesNotExceedWatermarkWithMultipleShards() throws Exception { + internalCluster().startMasterOnlyNode(); + internalCluster().startDataOnlyNode(); + final String dataNodeName = internalCluster().startDataOnlyNode(); + ensureStableCluster(3); + + assertAcked( + clusterAdmin().preparePutRepository("repo") + .setType(FsRepository.TYPE) + .setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", randomBoolean())) + ); + + final AtomicBoolean allowRelocations = new AtomicBoolean(true); + final InternalClusterInfoService clusterInfoService = getInternalClusterInfoService(); + internalCluster().getCurrentMasterNodeInstance(ClusterService.class).addListener(event -> { + ClusterInfoServiceUtils.refresh(clusterInfoService); + if (allowRelocations.get() == false) { + assertThat(numberOfShardsWithState(event.state().getRoutingNodes(), ShardRoutingState.RELOCATING), equalTo(0)); + } + }); + + final String dataNode0Id = internalCluster().getInstance(NodeEnvironment.class, dataNodeName).nodeId(); + + final String indexName = randomIdentifier(); + createIndex(indexName, indexSettings(6, 0).put(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), "0ms").build()); + var shardSizes = createReasonableSizedShards(indexName); + + final CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("repo", "snap") + .setWaitForCompletion(true) + .get(); + final SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo(); + assertThat(snapshotInfo.successfulShards(), is(snapshotInfo.totalShards())); + assertThat(snapshotInfo.state(), is(SnapshotState.SUCCESS)); + + assertAcked(indicesAdmin().prepareDelete(indexName).get()); + updateClusterSettings(Settings.builder().put(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), Rebalance.NONE.toString())); + allowRelocations.set(false); + + // reduce disk size of node 0 so that only 1 of 2 smallest shards can be allocated + var usableSpace = shardSizes.sizes().get(1).size(); + getTestFileStore(dataNodeName).setTotalSpace(usableSpace + WATERMARK_BYTES + 1L); + refreshDiskUsage(); + + final RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("repo", "snap") + .setWaitForCompletion(true) + .get(); + final RestoreInfo restoreInfo = restoreSnapshotResponse.getRestoreInfo(); + assertThat(restoreInfo.successfulShards(), is(snapshotInfo.totalShards())); + assertThat(restoreInfo.failedShards(), is(0)); + + assertBusyWithDiskUsageRefresh( + dataNode0Id, + indexName, + new ContainsExactlyOneOf<>(shardSizes.getShardIdsWithSizeSmallerOrEqual(usableSpace)) + ); } private Set getShardIds(final String nodeId, final String indexName) { @@ -178,13 +238,9 @@ private Set getShardIds(final String nodeId, final String indexName) { /** * Index documents until all the shards are at least WATERMARK_BYTES in size, and return the one with the smallest size */ - private SmallestShards createReasonableSizedShards(final String indexName) throws InterruptedException { + private ShardSizes createReasonableSizedShards(final String indexName) throws InterruptedException { while (true) { - final IndexRequestBuilder[] indexRequestBuilders = new IndexRequestBuilder[scaledRandomIntBetween(100, 10000)]; - for (int i = 0; i < indexRequestBuilders.length; i++) { - indexRequestBuilders[i] = prepareIndex(indexName).setSource("field", randomAlphaOfLength(10)); - } - indexRandom(true, indexRequestBuilders); + indexRandom(true, indexName, scaledRandomIntBetween(100, 10000)); forceMerge(); refresh(); @@ -201,23 +257,36 @@ private SmallestShards createReasonableSizedShards(final String indexName) throw .orElseThrow(() -> new AssertionError("no shards")); if (smallestShardSize > WATERMARK_BYTES) { - var smallestShardIds = Arrays.stream(shardStates) - .filter(it -> it.getStats().getStore().sizeInBytes() == smallestShardSize) - .map(it -> removeIndexUUID(it.getShardRouting().shardId())) - .collect(toSet()); - - logger.info( - "Created shards with sizes {}", - Arrays.stream(shardStates) - .collect(toMap(it -> it.getShardRouting().shardId(), it -> it.getStats().getStore().sizeInBytes())) - ); - - return new SmallestShards(smallestShardSize, smallestShardIds); + var shardSizes = Arrays.stream(shardStates) + .map(it -> new ShardSize(removeIndexUUID(it.getShardRouting().shardId()), it.getStats().getStore().sizeInBytes())) + .sorted(Comparator.comparing(ShardSize::size)) + .toList(); + logger.info("Created shards with sizes {}", shardSizes); + return new ShardSizes(shardSizes); } } } - private record SmallestShards(long size, Set shardIds) {} + private record ShardSizes(List sizes) { + + public long getSmallestShardSize() { + return sizes.get(0).size(); + } + + public Set getShardIdsWithSizeSmallerOrEqual(long size) { + return sizes.stream().filter(entry -> entry.size <= size).map(ShardSize::shardId).collect(toSet()); + } + + public Set getSmallestShardIds() { + return getShardIdsWithSizeSmallerOrEqual(getSmallestShardSize()); + } + + public Set getAllShardIds() { + return sizes.stream().map(ShardSize::shardId).collect(toSet()); + } + } + + private record ShardSize(ShardId shardId, long size) {} private static ShardId removeIndexUUID(ShardId shardId) { return ShardId.fromString(shardId.toString()); @@ -246,16 +315,20 @@ private void refreshDiskUsage() { ); } - private void assertBusyWithDiskUsageRefresh(String nodeName, String indexName, Matcher> matcher) throws Exception { + private void assertBusyWithDiskUsageRefresh(String nodeId, String indexName, Matcher> matcher) throws Exception { assertBusy(() -> { // refresh the master's ClusterInfoService before checking the assigned shards because DiskThresholdMonitor might still // be processing a previous ClusterInfo update and will skip the new one (see DiskThresholdMonitor#onNewInfo(ClusterInfo) // and its internal checkInProgress flag) refreshDiskUsage(); - final Set shardRoutings = getShardIds(nodeName, indexName); + final Set shardRoutings = getShardIds(nodeId, indexName); assertThat("Mismatching shard routings: " + shardRoutings, shardRoutings, matcher); - }, 30L, TimeUnit.SECONDS); + }, 5L, TimeUnit.SECONDS); + } + + private InternalClusterInfoService getInternalClusterInfoService() { + return (InternalClusterInfoService) internalCluster().getCurrentMasterNodeInstance(ClusterInfoService.class); } private static final class ContainsExactlyOneOf extends TypeSafeMatcher> { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/service/ClusterServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/service/ClusterServiceIT.java index 873f8083f4e0c..fde465346d4be 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/service/ClusterServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/service/ClusterServiceIT.java @@ -357,7 +357,7 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) assertTrue(controlSources.isEmpty()); controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9", "10")); - PendingClusterTasksResponse response = internalCluster().coordOnlyNodeClient().admin().cluster().preparePendingClusterTasks().get(); + PendingClusterTasksResponse response = getClusterPendingTasks(internalCluster().coordOnlyNodeClient()); assertThat(response.pendingTasks().size(), greaterThanOrEqualTo(10)); assertThat(response.pendingTasks().get(0).getSource().string(), equalTo("1")); assertThat(response.pendingTasks().get(0).isExecuting(), equalTo(true)); @@ -419,7 +419,7 @@ public void onFailure(Exception e) { } assertTrue(controlSources.isEmpty()); - response = internalCluster().coordOnlyNodeClient().admin().cluster().preparePendingClusterTasks().get(); + response = getClusterPendingTasks(internalCluster().coordOnlyNodeClient()); assertThat(response.pendingTasks().size(), greaterThanOrEqualTo(5)); controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5")); for (PendingClusterTask task : response.pendingTasks()) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java index ec79b53ccd174..c1da93140a0b0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java @@ -182,11 +182,7 @@ public void testCorruptFileAndRecover() throws InterruptedException, IOException .waitForNoRelocatingShards(true) ).actionGet(); if (health.isTimedOut()) { - logger.info( - "cluster state:\n{}\n{}", - clusterAdmin().prepareState().get().getState(), - clusterAdmin().preparePendingClusterTasks().get() - ); + logger.info("cluster state:\n{}\n{}", clusterAdmin().prepareState().get().getState(), getClusterPendingTasks()); assertThat("timed out waiting for green state", health.isTimedOut(), equalTo(false)); } assertThat(health.getStatus(), equalTo(ClusterHealthStatus.GREEN)); @@ -295,11 +291,7 @@ public void testCorruptPrimaryNoReplica() throws ExecutionException, Interrupted if (response.getStatus() != ClusterHealthStatus.RED) { logger.info("Cluster turned red in busy loop: {}", didClusterTurnRed); - logger.info( - "cluster state:\n{}\n{}", - clusterAdmin().prepareState().get().getState(), - clusterAdmin().preparePendingClusterTasks().get() - ); + logger.info("cluster state:\n{}\n{}", clusterAdmin().prepareState().get().getState(), getClusterPendingTasks()); } assertThat(response.getStatus(), is(ClusterHealthStatus.RED)); ClusterState state = clusterAdmin().prepareState().get().getState(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java index 0fe5845e9ed32..779072272e59a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java @@ -8,22 +8,23 @@ package org.elasticsearch.indices.recovery; +import org.apache.logging.log4j.Level; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.monitor.jvm.HotThreads; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class IndexPrimaryRelocationIT extends ESIntegTestCase { @@ -71,20 +72,14 @@ public void run() { .setWaitForNoRelocatingShards(true) .get(); if (clusterHealthResponse.isTimedOut()) { - final String hotThreads = clusterAdmin().prepareNodesHotThreads() - .setIgnoreIdleThreads(false) - .get() - .getNodes() - .stream() - .map(NodeHotThreads::getHotThreads) - .collect(Collectors.joining("\n")); - final ClusterState clusterState = clusterAdmin().prepareState().get().getState(); - logger.info( - "timed out for waiting for relocation iteration [{}] \ncluster state {} \nhot threads {}", - i, - clusterState, - hotThreads + HotThreads.logLocalHotThreads( + logger, + Level.INFO, + "timed out waiting for relocation iteration [" + i + "]", + ReferenceDocs.LOGGING ); + final ClusterState clusterState = clusterAdmin().prepareState().get().getState(); + logger.info("timed out for waiting for relocation iteration [{}] \ncluster state {}", i, clusterState); finished.set(true); indexingThread.join(); throw new AssertionError("timed out waiting for relocation iteration [" + i + "] "); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java index 417c9242f690e..8e77628ace397 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.snapshots; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.tests.util.LuceneTestCase; @@ -16,7 +17,6 @@ import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequestBuilder; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequestBuilder; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequestBuilder; @@ -34,6 +34,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; @@ -46,6 +47,7 @@ import org.elasticsearch.core.Releasables; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; +import org.elasticsearch.monitor.jvm.HotThreads; import org.elasticsearch.repositories.RepositoryCleanupResult; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.test.InternalTestCluster; @@ -371,16 +373,11 @@ private void acquirePermitsAtEnd( "--> current cluster state:\n{}", Strings.toString(clusterAdmin().prepareState().get().getState(), true, true) ); - logger.info( - "--> hot threads:\n{}", - clusterAdmin().prepareNodesHotThreads() - .setThreads(99999) - .setIgnoreIdleThreads(false) - .get() - .getNodes() - .stream() - .map(NodeHotThreads::getHotThreads) - .collect(Collectors.joining("\n")) + HotThreads.logLocalHotThreads( + logger, + Level.INFO, + "hot threads while failing to acquire permit [" + label + "]", + ReferenceDocs.LOGGING ); failedPermitAcquisitions.add(label); } diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index ad29384b16f45..625871d25734b 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -192,6 +192,7 @@ static TransportVersion def(int id) { public static final TransportVersion INFERENCE_SERVICE_EMBEDDING_SIZE_ADDED = def(8_559_00_0); public static final TransportVersion ENRICH_ELASTICSEARCH_VERSION_REMOVED = def(8_560_00_0); public static final TransportVersion NODE_STATS_REQUEST_SIMPLIFIED = def(8_561_00_0); + public static final TransportVersion TEXT_EXPANSION_TOKEN_PRUNING_CONFIG_ADDED = def(8_562_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 303f81ab43063..4181b077cb185 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -116,6 +116,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_7_17_14 = new Version(7_17_14_99); public static final Version V_7_17_15 = new Version(7_17_15_99); public static final Version V_7_17_16 = new Version(7_17_16_99); + public static final Version V_7_17_17 = new Version(7_17_17_99); public static final Version V_8_0_0 = new Version(8_00_00_99); public static final Version V_8_0_1 = new Version(8_00_01_99); public static final Version V_8_1_0 = new Version(8_01_00_99); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java deleted file mode 100644 index 6593b90fb7f65..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.action.admin.cluster.node.hotthreads; - -import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder; -import org.elasticsearch.client.internal.ElasticsearchClient; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.monitor.jvm.HotThreads; - -public class NodesHotThreadsRequestBuilder extends NodesOperationRequestBuilder< - NodesHotThreadsRequest, - NodesHotThreadsResponse, - NodesHotThreadsRequestBuilder> { - - public NodesHotThreadsRequestBuilder(ElasticsearchClient client) { - super(client, TransportNodesHotThreadsAction.TYPE, new NodesHotThreadsRequest()); - } - - public NodesHotThreadsRequestBuilder setThreads(int threads) { - request.threads(threads); - return this; - } - - public NodesHotThreadsRequestBuilder setIgnoreIdleThreads(boolean ignoreIdleThreads) { - request.ignoreIdleThreads(ignoreIdleThreads); - return this; - } - - public NodesHotThreadsRequestBuilder setType(HotThreads.ReportType type) { - request.type(type); - return this; - } - - public NodesHotThreadsRequestBuilder setSortOrder(HotThreads.SortOrder sortOrder) { - request.sortOrder(sortOrder); - return this; - } - - public NodesHotThreadsRequestBuilder setInterval(TimeValue interval) { - request.interval(interval); - return this; - } -} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java index 7fa97f1ee14b7..ed63e6d1b4474 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java @@ -122,6 +122,8 @@ protected NodesReloadSecureSettingsResponse.NodeResponse nodeOperation( keystore.decrypt(request.hasPassword() ? request.getSecureSettingsPassword().getChars() : new char[0]); // add the keystore to the original node settings object final Settings settingsWithKeystore = Settings.builder().put(environment.settings(), false).setSecureSettings(keystore).build(); + clusterService.getClusterSettings().validate(settingsWithKeystore, true); + final List exceptions = new ArrayList<>(); // broadcast the new settings object (with the open embedded keystore) to all reloadable plugins pluginsService.filterPlugins(ReloadablePlugin.class).forEach(p -> { diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java index f4e301e0748bb..29bffa3949258 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; import org.elasticsearch.cluster.ClusterState; @@ -112,7 +113,7 @@ public void onNewClusterState(ClusterState newState) { } if (acceptableClusterStatePredicate.test(newState)) { - ActionListener.completeWith(listener, () -> buildResponse(request, newState)); + executor.execute(ActionRunnable.supply(listener, () -> buildResponse(request, newState))); } else { listener.onFailure( new NotMasterException( @@ -150,6 +151,8 @@ private static Map> getClusterFeatures(ClusterState clusterS } private ClusterStateResponse buildResponse(final ClusterStateRequest request, final ClusterState currentState) { + ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT); // too heavy to construct & serialize cluster state without forking + logger.trace("Serving cluster state request using version {}", currentState.version()); ClusterState.Builder builder = ClusterState.builder(currentState.getClusterName()); builder.version(currentState.version()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksRequestBuilder.java deleted file mode 100644 index aa3f226d23c9d..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksRequestBuilder.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.action.admin.cluster.tasks; - -import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder; -import org.elasticsearch.client.internal.ElasticsearchClient; - -public class PendingClusterTasksRequestBuilder extends MasterNodeReadOperationRequestBuilder< - PendingClusterTasksRequest, - PendingClusterTasksResponse, - PendingClusterTasksRequestBuilder> { - - public PendingClusterTasksRequestBuilder(ElasticsearchClient client) { - super(client, TransportPendingClusterTasksAction.TYPE, new PendingClusterTasksRequest()); - } -} diff --git a/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java b/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java index 17d712bdf5af4..9e3bed8cef09a 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java @@ -16,9 +16,6 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequestBuilder; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequestBuilder; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; @@ -88,9 +85,6 @@ import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse; import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequestBuilder; -import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest; -import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequestBuilder; -import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse; import org.elasticsearch.action.admin.indices.dangling.delete.DeleteDanglingIndexRequest; import org.elasticsearch.action.admin.indices.dangling.import_index.ImportDanglingIndexRequest; import org.elasticsearch.action.admin.indices.dangling.list.ListDanglingIndicesRequest; @@ -256,18 +250,6 @@ public interface ClusterAdminClient extends ElasticsearchClient { */ void nodesUsage(NodesUsageRequest request, ActionListener listener); - /** - * Returns top N hot-threads samples per node. The hot-threads are only sampled - * for the node ids specified in the request. - */ - void nodesHotThreads(NodesHotThreadsRequest request, ActionListener listener); - - /** - * Returns a request builder to fetch top N hot-threads samples per node. The hot-threads are only sampled - * for the node ids provided. Note: Use {@code *} to fetch samples for all nodes - */ - NodesHotThreadsRequestBuilder prepareNodesHotThreads(String... nodesIds); - /** * List tasks * @@ -456,18 +438,6 @@ public interface ClusterAdminClient extends ElasticsearchClient { */ RestoreSnapshotRequestBuilder prepareRestoreSnapshot(String repository, String snapshot); - /** - * Returns a list of the pending cluster tasks, that are scheduled to be executed. This includes operations - * that update the cluster state (for example, a create index operation) - */ - void pendingClusterTasks(PendingClusterTasksRequest request, ActionListener listener); - - /** - * Returns a list of the pending cluster tasks, that are scheduled to be executed. This includes operations - * that update the cluster state (for example, a create index operation) - */ - PendingClusterTasksRequestBuilder preparePendingClusterTasks(); - /** * Get snapshot status. */ diff --git a/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java b/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java index 21c01abd52437..075d1a4bb1e66 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java @@ -24,10 +24,6 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequestBuilder; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse; -import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequestBuilder; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; @@ -122,10 +118,6 @@ import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequestBuilder; import org.elasticsearch.action.admin.cluster.storedscripts.TransportDeleteStoredScriptAction; import org.elasticsearch.action.admin.cluster.storedscripts.TransportPutStoredScriptAction; -import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest; -import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequestBuilder; -import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse; -import org.elasticsearch.action.admin.cluster.tasks.TransportPendingClusterTasksAction; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; import org.elasticsearch.action.admin.indices.alias.TransportIndicesAliasesAction; @@ -795,16 +787,6 @@ public ClusterStatsRequestBuilder prepareClusterStats() { return new ClusterStatsRequestBuilder(this); } - @Override - public void nodesHotThreads(NodesHotThreadsRequest request, ActionListener listener) { - execute(TransportNodesHotThreadsAction.TYPE, request, listener); - } - - @Override - public NodesHotThreadsRequestBuilder prepareNodesHotThreads(String... nodesIds) { - return new NodesHotThreadsRequestBuilder(this).setNodesIds(nodesIds); - } - @Override public ActionFuture listTasks(final ListTasksRequest request) { return execute(TransportListTasksAction.TYPE, request); @@ -865,16 +847,6 @@ public ClusterSearchShardsRequestBuilder prepareSearchShards(String... indices) return new ClusterSearchShardsRequestBuilder(this).setIndices(indices); } - @Override - public PendingClusterTasksRequestBuilder preparePendingClusterTasks() { - return new PendingClusterTasksRequestBuilder(this); - } - - @Override - public void pendingClusterTasks(PendingClusterTasksRequest request, ActionListener listener) { - execute(TransportPendingClusterTasksAction.TYPE, request, listener); - } - @Override public void putRepository(PutRepositoryRequest request, ActionListener listener) { execute(TransportPutRepositoryAction.TYPE, request, listener); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimator.java b/server/src/main/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimator.java index 85e201d52f03b..1f364e1ace6e4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimator.java @@ -20,9 +20,13 @@ public class ExpectedShardSizeEstimator { - public static long getExpectedShardSize(ShardRouting shardRouting, long defaultSize, RoutingAllocation allocation) { + public static boolean shouldReserveSpaceForInitializingShard(ShardRouting shard, RoutingAllocation allocation) { + return shouldReserveSpaceForInitializingShard(shard, allocation.metadata()); + } + + public static long getExpectedShardSize(ShardRouting shard, long defaultSize, RoutingAllocation allocation) { return getExpectedShardSize( - shardRouting, + shard, defaultSize, allocation.clusterInfo(), allocation.snapshotShardSizeInfo(), @@ -31,6 +35,27 @@ public static long getExpectedShardSize(ShardRouting shardRouting, long defaultS ); } + public static boolean shouldReserveSpaceForInitializingShard(ShardRouting shard, Metadata metadata) { + assert shard.initializing() : "Expected initializing shard, got: " + shard; + return switch (shard.recoverySource().getType()) { + // No need to reserve disk space when initializing a new empty shard + case EMPTY_STORE -> false; + + // No need to reserve disk space if the shard is already allocated on the disk. Starting it is not going to use more. + case EXISTING_STORE -> false; + + // Peer recovery require downloading all segments locally to start the shard. Reserve disk space for this + case PEER -> true; + + // Snapshot restore (unless it is partial) require downloading all segments locally from the blobstore to start the shard. + case SNAPSHOT -> metadata.getIndexSafe(shard.index()).isPartialSearchableSnapshot() == false; + + // shrink/split/clone operation is going to clone existing locally placed shards using file system hard links + // so no additional space is going to be used until future merges + case LOCAL_SHARDS -> false; + }; + } + /** * Returns the expected shard size for the given shard or the default value provided if not enough information are available * to estimate the shards size. diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java index 5c216b9a5b308..6645fd7d0e895 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.core.Releasable; import org.elasticsearch.gateway.GatewayService; +import org.elasticsearch.snapshots.SnapshotShardSizeInfo; import java.util.ArrayList; import java.util.Collections; @@ -425,6 +426,7 @@ long sizeOfRelocatingShards(RoutingNode routingNode, DiskUsage diskUsage, Cluste true, diskUsage.getPath(), info, + SnapshotShardSizeInfo.EMPTY, reroutedClusterState.metadata(), reroutedClusterState.routingTable(), 0L diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java index e92a6106a6e33..2fa1994f9f74b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java @@ -32,6 +32,7 @@ import java.util.Map; import static org.elasticsearch.cluster.routing.ExpectedShardSizeEstimator.getExpectedShardSize; +import static org.elasticsearch.cluster.routing.ExpectedShardSizeEstimator.shouldReserveSpaceForInitializingShard; /** * The {@link DiskThresholdDecider} checks that the node a shard is potentially @@ -117,6 +118,7 @@ public static long sizeOfUnaccountedShards( boolean subtractShardsMovingAway, String dataPath, ClusterInfo clusterInfo, + SnapshotShardSizeInfo snapshotShardSizeInfo, Metadata metadata, RoutingTable routingTable, long sizeOfUnaccountableSearchableSnapshotShards @@ -129,28 +131,18 @@ public static long sizeOfUnaccountedShards( // Where reserved space is unavailable (e.g. stats are out-of-sync) compute a conservative estimate for initialising shards for (ShardRouting routing : node.initializing()) { - if (routing.relocatingNodeId() == null && metadata.getIndexSafe(routing.index()).isSearchableSnapshot() == false) { - // in practice the only initializing-but-not-relocating non-searchable-snapshot shards with a nonzero expected shard size - // will be ones created - // by a resize (shrink/split/clone) operation which we expect to happen using hard links, so they shouldn't be taking - // any additional space and can be ignored here - continue; - } - if (reservedSpace.containsShardId(routing.shardId())) { - continue; - } - final String actualPath = clusterInfo.getDataPath(routing); - // if we don't yet know the actual path of the incoming shard then conservatively assume it's going to the path with the least - // free space - if (actualPath == null || actualPath.equals(dataPath)) { - totalSize += getExpectedShardSize( - routing, - Math.max(routing.getExpectedShardSize(), 0L), - clusterInfo, - SnapshotShardSizeInfo.EMPTY, - metadata, - routingTable - ); + // Space needs to be reserved only when initializing shards that are going to use additional space + // that is not yet accounted for by `reservedSpace` in case of lengthy recoveries + if (shouldReserveSpaceForInitializingShard(routing, metadata) && reservedSpace.containsShardId(routing.shardId()) == false) { + final String actualPath = clusterInfo.getDataPath(routing); + // if we don't yet know the actual path of the incoming shard then conservatively assume + // it's going to the path with the least free space + if (actualPath == null || actualPath.equals(dataPath)) { + totalSize += Math.max( + routing.getExpectedShardSize(), + getExpectedShardSize(routing, 0L, clusterInfo, snapshotShardSizeInfo, metadata, routingTable) + ); + } } } @@ -159,7 +151,7 @@ public static long sizeOfUnaccountedShards( if (subtractShardsMovingAway) { for (ShardRouting routing : node.relocating()) { if (dataPath.equals(clusterInfo.getDataPath(routing))) { - totalSize -= getExpectedShardSize(routing, 0L, clusterInfo, SnapshotShardSizeInfo.EMPTY, metadata, routingTable); + totalSize -= getExpectedShardSize(routing, 0L, clusterInfo, snapshotShardSizeInfo, metadata, routingTable); } } } @@ -204,6 +196,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing false, usage.getPath(), allocation.clusterInfo(), + allocation.snapshotShardSizeInfo(), allocation.metadata(), allocation.routingTable(), allocation.unaccountedSearchableSnapshotSize(node) @@ -412,6 +405,7 @@ public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting true, usage.getPath(), allocation.clusterInfo(), + allocation.snapshotShardSizeInfo(), allocation.metadata(), allocation.routingTable(), allocation.unaccountedSearchableSnapshotSize(node) @@ -491,6 +485,7 @@ private static DiskUsageWithRelocations getDiskUsage( subtractLeavingShards, usage.getPath(), allocation.clusterInfo(), + allocation.snapshotShardSizeInfo(), allocation.metadata(), allocation.routingTable(), allocation.unaccountedSearchableSnapshotSize(node) diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 125f9529c4165..f4edb4f79d760 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -92,6 +92,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion ES_VERSION_8_12 = def(8_500_004, Version.LUCENE_9_8_0); public static final IndexVersion NORMALIZED_VECTOR_COSINE = def(8_500_005, Version.LUCENE_9_8_0); public static final IndexVersion UPGRADE_LUCENE_9_9 = def(8_500_006, Version.LUCENE_9_9_0); + public static final IndexVersion ES_VERSION_8_13 = def(8_500_007, Version.LUCENE_9_9_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java b/server/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java index ef0901bc17712..1cee2a90ec3f1 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java +++ b/server/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java @@ -64,11 +64,18 @@ public static final class VersionLookup { // Modifies the map of this instance by merging with the given VersionLookup public void merge(VersionLookup versionLookup) { + long existingEntriesSize = 0; + for (var entry : versionLookup.map.entrySet()) { + var existingValue = map.get(entry.getKey()); + existingEntriesSize += existingValue == null ? 0 : mapEntryBytesUsed(entry.getKey(), existingValue); + } map.putAll(versionLookup.map); + adjustRam(versionLookup.ramBytesUsed() - existingEntriesSize); minDeleteTimestamp.accumulateAndGet(versionLookup.minDeleteTimestamp(), Math::min); } - private VersionLookup(Map map) { + // Visible for testing + VersionLookup(Map map) { this.map = map; } @@ -77,7 +84,11 @@ public VersionValue get(BytesRef key) { } VersionValue put(BytesRef key, VersionValue value) { - return map.put(key, value); + long ramAccounting = mapEntryBytesUsed(key, value); + VersionValue previousValue = map.put(key, value); + ramAccounting += previousValue == null ? 0 : -mapEntryBytesUsed(key, previousValue); + adjustRam(ramAccounting); + return previousValue; } public boolean isEmpty() { @@ -96,8 +107,12 @@ void markAsUnsafe() { unsafe = true; } - public VersionValue remove(BytesRef uid) { - return map.remove(uid); + VersionValue remove(BytesRef uid) { + VersionValue previousValue = map.remove(uid); + if (previousValue != null) { + adjustRam(-mapEntryBytesUsed(uid, previousValue)); + } + return previousValue; } public void updateMinDeletedTimestamp(DeleteVersionValue delete) { @@ -107,6 +122,26 @@ public void updateMinDeletedTimestamp(DeleteVersionValue delete) { public long minDeleteTimestamp() { return minDeleteTimestamp.get(); } + + void adjustRam(long value) { + if (value != 0) { + long v = ramBytesUsed.addAndGet(value); + assert v >= 0 : "bytes=" + v; + } + } + + public long ramBytesUsed() { + return ramBytesUsed.get(); + } + + public static long mapEntryBytesUsed(BytesRef key, VersionValue value) { + return (BASE_BYTES_PER_BYTESREF + key.bytes.length) + (BASE_BYTES_PER_CHM_ENTRY + value.ramBytesUsed()); + } + + // Used only for testing + Map getMap() { + return map; + } } private static final class Maps { @@ -170,27 +205,12 @@ Maps invalidateOldMap(LiveVersionMapArchive archive) { } void put(BytesRef uid, VersionValue version) { - long uidRAMBytesUsed = BASE_BYTES_PER_BYTESREF + uid.bytes.length; - long ramAccounting = BASE_BYTES_PER_CHM_ENTRY + version.ramBytesUsed() + uidRAMBytesUsed; - VersionValue previousValue = current.put(uid, version); - ramAccounting += previousValue == null ? 0 : -(BASE_BYTES_PER_CHM_ENTRY + previousValue.ramBytesUsed() + uidRAMBytesUsed); - adjustRam(ramAccounting); - } - - void adjustRam(long value) { - if (value != 0) { - long v = current.ramBytesUsed.addAndGet(value); - assert v >= 0 : "bytes=" + v; - } + current.put(uid, version); } void remove(BytesRef uid, DeleteVersionValue deleted) { - VersionValue previousValue = current.remove(uid); + current.remove(uid); current.updateMinDeletedTimestamp(deleted); - if (previousValue != null) { - long uidRAMBytesUsed = BASE_BYTES_PER_BYTESREF + uid.bytes.length; - adjustRam(-(BASE_BYTES_PER_CHM_ENTRY + previousValue.ramBytesUsed() + uidRAMBytesUsed)); - } if (old != VersionLookup.EMPTY) { // we also need to remove it from the old map here to make sure we don't read this stale value while // we are in the middle of a refresh. Most of the time the old map is an empty map so we can skip it there. @@ -452,7 +472,7 @@ synchronized void clear() { @Override public long ramBytesUsed() { - return maps.ramBytesUsed() + ramBytesUsedTombstones.get(); + return maps.ramBytesUsed() + ramBytesUsedTombstones.get() + ramBytesUsedForArchive(); } /** @@ -463,6 +483,13 @@ long ramBytesUsedForRefresh() { return maps.current.ramBytesUsed.get(); } + /** + * Returns how much RAM would be freed up by cleaning out the LiveVersionMapArchive. + */ + long ramBytesUsedForArchive() { + return archive.getMemoryBytesUsed(); + } + /** * Returns how much RAM is current being freed up by refreshing. This is the RAM usage of the previous version map that needs to stay * around until operations are safely recorded in the Lucene index. diff --git a/server/src/main/java/org/elasticsearch/index/engine/LiveVersionMapArchive.java b/server/src/main/java/org/elasticsearch/index/engine/LiveVersionMapArchive.java index a68a1cea368d4..9ccbf6ac16fed 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/LiveVersionMapArchive.java +++ b/server/src/main/java/org/elasticsearch/index/engine/LiveVersionMapArchive.java @@ -39,6 +39,14 @@ default boolean isUnsafe() { return false; } + /** + * Returns how much memory is currently being used by the archive and would be freed up after + * unpromotables are refreshed. + */ + default long getMemoryBytesUsed() { + return 0L; + } + LiveVersionMapArchive NOOP_ARCHIVE = new LiveVersionMapArchive() { @Override public void afterRefresh(LiveVersionMap.VersionLookup old) {} diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java index de7bfa91dda42..4042b21b865e4 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.cache.Cache; import org.elasticsearch.common.cache.CacheBuilder; import org.elasticsearch.common.cache.CacheLoader; -import org.elasticsearch.common.cache.RemovalListener; import org.elasticsearch.common.cache.RemovalNotification; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Setting; @@ -48,7 +47,7 @@ * There are still several TODOs left in this class, some easily addressable, some more complex, but the support * is functional. */ -public final class IndicesRequestCache implements RemovalListener, Closeable { +public final class IndicesRequestCache implements Closeable { /** * A setting to enable or disable request caching on an index level. Its dynamic by default @@ -73,18 +72,14 @@ public final class IndicesRequestCache implements RemovalListener registeredClosedListeners = ConcurrentCollections.newConcurrentMap(); private final Set keysToClean = ConcurrentCollections.newConcurrentSet(); - private final ByteSizeValue size; - private final TimeValue expire; private final Cache cache; IndicesRequestCache(Settings settings) { - this.size = INDICES_CACHE_QUERY_SIZE.get(settings); - this.expire = INDICES_CACHE_QUERY_EXPIRE.exists(settings) ? INDICES_CACHE_QUERY_EXPIRE.get(settings) : null; - long sizeInBytes = size.getBytes(); + TimeValue expire = INDICES_CACHE_QUERY_EXPIRE.exists(settings) ? INDICES_CACHE_QUERY_EXPIRE.get(settings) : null; CacheBuilder cacheBuilder = CacheBuilder.builder() - .setMaximumWeight(sizeInBytes) + .setMaximumWeight(INDICES_CACHE_QUERY_SIZE.get(settings).getBytes()) .weigher((k, v) -> k.ramBytesUsed() + v.ramBytesUsed()) - .removalListener(this); + .removalListener(notification -> notification.getKey().entity.onRemoval(notification)); if (expire != null) { cacheBuilder.setExpireAfterAccess(expire); } @@ -101,11 +96,6 @@ void clear(CacheEntity entity) { cleanCache(); } - @Override - public void onRemoval(RemovalNotification notification) { - notification.getKey().entity.onRemoval(notification); - } - BytesReference getOrCompute( CacheEntity cacheEntity, CheckedSupplier loader, diff --git a/server/src/main/java/org/elasticsearch/node/NodeService.java b/server/src/main/java/org/elasticsearch/node/NodeService.java index e2283ea9851d7..4b9e5dc83c538 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeService.java +++ b/server/src/main/java/org/elasticsearch/node/NodeService.java @@ -62,6 +62,7 @@ public class NodeService implements Closeable { private final AggregationUsageService aggregationUsageService; private final Coordinator coordinator; private final RepositoriesService repositoriesService; + private final Map componentVersions; NodeService( Settings settings, @@ -100,6 +101,7 @@ public class NodeService implements Closeable { this.indexingPressure = indexingPressure; this.aggregationUsageService = aggregationUsageService; this.repositoriesService = repositoriesService; + this.componentVersions = findComponentVersions(pluginService); clusterService.addStateApplier(ingestService); } @@ -122,7 +124,7 @@ public NodeInfo info( Version.CURRENT.toString(), TransportVersion.current(), IndexVersion.current(), - findComponentVersions(), + componentVersions, Build.current(), transportService.getLocalNode(), settings ? settingsFilter.filter(this.settings) : null, @@ -140,7 +142,7 @@ public NodeInfo info( ); } - private Map findComponentVersions() { + private static Map findComponentVersions(PluginsService pluginService) { var versions = pluginService.loadServiceProviders(ComponentVersionNumber.class) .stream() .collect(Collectors.toUnmodifiableMap(ComponentVersionNumber::componentId, cvn -> cvn.versionNumber().id())); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java index 095abcd14d355..76df8af1889a7 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest; import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse; +import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.core.RestApiVersion; @@ -112,7 +113,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC nodesHotThreadsRequest.interval(TimeValue.parseTimeValue(request.param("interval"), nodesHotThreadsRequest.interval(), "interval")); nodesHotThreadsRequest.snapshots(request.paramAsInt("snapshots", nodesHotThreadsRequest.snapshots())); nodesHotThreadsRequest.timeout(request.param("timeout")); - return channel -> client.admin().cluster().nodesHotThreads(nodesHotThreadsRequest, new RestResponseListener<>(channel) { + return channel -> client.execute(TransportNodesHotThreadsAction.TYPE, nodesHotThreadsRequest, new RestResponseListener<>(channel) { @Override public RestResponse buildResponse(NodesHotThreadsResponse response) { return RestResponse.chunked(RestStatus.OK, fromTextChunks(TEXT_CONTENT_TYPE, response.getTextChunks(), null)); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java index 8442507c36b1c..e9f9b9bf4327d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java @@ -9,6 +9,7 @@ package org.elasticsearch.rest.action.admin.cluster; import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest; +import org.elasticsearch.action.admin.cluster.tasks.TransportPendingClusterTasksAction; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -39,8 +40,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC PendingClusterTasksRequest pendingClusterTasksRequest = new PendingClusterTasksRequest(); pendingClusterTasksRequest.masterNodeTimeout(request.paramAsTime("master_timeout", pendingClusterTasksRequest.masterNodeTimeout())); pendingClusterTasksRequest.local(request.paramAsBoolean("local", pendingClusterTasksRequest.local())); - return channel -> client.admin() - .cluster() - .pendingClusterTasks(pendingClusterTasksRequest, new RestChunkedToXContentListener<>(channel)); + return channel -> client.execute( + TransportPendingClusterTasksAction.TYPE, + pendingClusterTasksRequest, + new RestChunkedToXContentListener<>(channel) + ); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java index 7408bf3ab229e..19ebbd2f19df4 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest; import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse; +import org.elasticsearch.action.admin.cluster.tasks.TransportPendingClusterTasksAction; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.service.PendingClusterTask; import org.elasticsearch.common.Table; @@ -46,15 +47,17 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli PendingClusterTasksRequest pendingClusterTasksRequest = new PendingClusterTasksRequest(); pendingClusterTasksRequest.masterNodeTimeout(request.paramAsTime("master_timeout", pendingClusterTasksRequest.masterNodeTimeout())); pendingClusterTasksRequest.local(request.paramAsBoolean("local", pendingClusterTasksRequest.local())); - return channel -> client.admin() - .cluster() - .pendingClusterTasks(pendingClusterTasksRequest, new RestResponseListener(channel) { + return channel -> client.execute( + TransportPendingClusterTasksAction.TYPE, + pendingClusterTasksRequest, + new RestResponseListener<>(channel) { @Override public RestResponse buildResponse(PendingClusterTasksResponse pendingClusterTasks) throws Exception { Table tab = buildTable(request, pendingClusterTasks); return RestTable.buildResponse(tab, channel); } - }); + } + ); } @Override diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimatorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimatorTests.java index 62fd21defa676..f81d99c55e84e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimatorTests.java @@ -30,33 +30,54 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_RESIZE_SOURCE_NAME_KEY; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_RESIZE_SOURCE_UUID_KEY; import static org.elasticsearch.cluster.routing.ExpectedShardSizeEstimator.getExpectedShardSize; +import static org.elasticsearch.cluster.routing.ExpectedShardSizeEstimator.shouldReserveSpaceForInitializingShard; import static org.elasticsearch.cluster.routing.TestShardRouting.newShardRouting; +import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; +import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE; +import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SNAPSHOT_PARTIAL_SETTING; import static org.hamcrest.Matchers.equalTo; public class ExpectedShardSizeEstimatorTests extends ESAllocationTestCase { private final long defaultValue = randomLongBetween(-1, 0); - public void testShouldFallbackToDefaultValue() { + public void testShouldFallbackToDefaultExpectedShardSize() { var state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata(index("my-index"))).build(); - var shard = newShardRouting("my-index", 0, randomIdentifier(), true, ShardRoutingState.INITIALIZING); + var shard = newShardRouting( + new ShardId("my-index", "_na_", 0), + randomIdentifier(), + true, + ShardRoutingState.INITIALIZING, + randomFrom(RecoverySource.EmptyStoreRecoverySource.INSTANCE, RecoverySource.ExistingStoreRecoverySource.INSTANCE) + ); var allocation = createRoutingAllocation(state, ClusterInfo.EMPTY, SnapshotShardSizeInfo.EMPTY); assertThat(getExpectedShardSize(shard, defaultValue, allocation), equalTo(defaultValue)); + assertFalse( + "Should NOT reserve space for locally initializing primaries", + shouldReserveSpaceForInitializingShard(shard, allocation) + ); } public void testShouldReadExpectedSizeFromClusterInfo() { var shardSize = randomLongBetween(100, 1000); var state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata(index("my-index"))).build(); - var shard = newShardRouting("my-index", 0, randomIdentifier(), true, ShardRoutingState.INITIALIZING); + var shard = newShardRouting( + new ShardId("my-index", "_na_", 0), + randomIdentifier(), + true, + ShardRoutingState.INITIALIZING, + RecoverySource.PeerRecoverySource.INSTANCE + ); var clusterInfo = createClusterInfo(shard, shardSize); var allocation = createRoutingAllocation(state, clusterInfo, SnapshotShardSizeInfo.EMPTY); assertThat(getExpectedShardSize(shard, defaultValue, allocation), equalTo(shardSize)); + assertTrue("Should reserve space for relocating shard", shouldReserveSpaceForInitializingShard(shard, allocation)); } public void testShouldReadExpectedSizeFromPrimaryWhenAddingNewReplica() { @@ -70,21 +91,39 @@ public void testShouldReadExpectedSizeFromPrimaryWhenAddingNewReplica() { var allocation = createRoutingAllocation(state, clusterInfo, SnapshotShardSizeInfo.EMPTY); assertThat(getExpectedShardSize(replica, defaultValue, allocation), equalTo(shardSize)); + assertTrue("Should reserve space for peer recovery", shouldReserveSpaceForInitializingShard(replica, allocation)); } public void testShouldReadExpectedSizeWhenInitializingFromSnapshot() { var snapshotShardSize = randomLongBetween(100, 1000); - var state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata(index("my-index"))).build(); + + var index = switch (randomIntBetween(0, 2)) { + // regular snapshot + case 0 -> index("my-index"); + // searchable snapshot + case 1 -> index("my-index").settings( + indexSettings(IndexVersion.current(), 1, 0) // + .put(INDEX_STORE_TYPE_SETTING.getKey(), SEARCHABLE_SNAPSHOT_STORE_TYPE) // + ); + // partial searchable snapshot + case 2 -> index("my-index").settings( + indexSettings(IndexVersion.current(), 1, 0) // + .put(INDEX_STORE_TYPE_SETTING.getKey(), SEARCHABLE_SNAPSHOT_STORE_TYPE) // + .put(SNAPSHOT_PARTIAL_SETTING.getKey(), true) // + ); + default -> throw new AssertionError("unexpected index type"); + }; + var state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata(index)).build(); var snapshot = new Snapshot("repository", new SnapshotId("snapshot-1", "na")); var indexId = new IndexId("my-index", "_na_"); var shard = newShardRouting( new ShardId("my-index", "_na_", 0), - null, + randomIdentifier(), true, - ShardRoutingState.UNASSIGNED, + ShardRoutingState.INITIALIZING, new RecoverySource.SnapshotRecoverySource(randomUUID(), snapshot, IndexVersion.current(), indexId) ); @@ -94,6 +133,14 @@ public void testShouldReadExpectedSizeWhenInitializingFromSnapshot() { var allocation = createRoutingAllocation(state, ClusterInfo.EMPTY, snapshotShardSizeInfo); assertThat(getExpectedShardSize(shard, defaultValue, allocation), equalTo(snapshotShardSize)); + if (state.metadata().index("my-index").isPartialSearchableSnapshot() == false) { + assertTrue("Should reserve space for snapshot restore", shouldReserveSpaceForInitializingShard(shard, allocation)); + } else { + assertFalse( + "Should NOT reserve space for partial searchable snapshot restore as they do not download all data during initialization", + shouldReserveSpaceForInitializingShard(shard, allocation) + ); + } } public void testShouldReadSizeFromClonedShard() { @@ -127,6 +174,10 @@ public void testShouldReadSizeFromClonedShard() { var allocation = createRoutingAllocation(state, clusterInfo, SnapshotShardSizeInfo.EMPTY); assertThat(getExpectedShardSize(target, defaultValue, allocation), equalTo(sourceShardSize)); + assertFalse( + "Should NOT reserve space when using fs hardlink for clone/shrink/split", + shouldReserveSpaceForInitializingShard(target, state.metadata()) + ); } private static RoutingAllocation createRoutingAllocation( diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java index b4eba769543b8..5e3b6cd02f830 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.DiskUsage; import org.elasticsearch.cluster.ESAllocationTestCase; +import org.elasticsearch.cluster.RestoreInProgress; import org.elasticsearch.cluster.TestShardRoutingRoleStrategies; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; @@ -40,9 +41,14 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.Maps; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.repositories.IndexId; +import org.elasticsearch.snapshots.InternalSnapshotsInfoService.SnapshotShard; +import org.elasticsearch.snapshots.Snapshot; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotShardSizeInfo; import org.elasticsearch.test.MockLogAppender; import org.elasticsearch.threadpool.ThreadPool; @@ -61,6 +67,7 @@ import java.util.function.Function; import static java.util.stream.Collectors.toMap; +import static org.elasticsearch.cluster.ClusterInfo.shardIdentifierFromRouting; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING; import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; @@ -70,6 +77,7 @@ import static org.elasticsearch.test.MockLogAppender.assertThatLogger; import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.everyItem; import static org.hamcrest.Matchers.hasEntry; @@ -623,7 +631,7 @@ public void testDesiredBalanceShouldConvergeInABigCluster() { var thisShardSize = smallShardSizeDeviation(shardSize); var primaryNodeId = pickAndRemoveRandomValueFrom(remainingNodeIds); - shardSizes.put(ClusterInfo.shardIdentifierFromRouting(shardId, true), thisShardSize); + shardSizes.put(shardIdentifierFromRouting(shardId, true), thisShardSize); totalShardsSize += thisShardSize; if (primaryNodeId != null) { dataPath.put(new NodeAndShard(primaryNodeId, shardId), "/data"); @@ -642,7 +650,7 @@ public void testDesiredBalanceShouldConvergeInABigCluster() { ); for (int replica = 0; replica < replicas; replica++) { var replicaNodeId = primaryNodeId == null ? null : pickAndRemoveRandomValueFrom(remainingNodeIds); - shardSizes.put(ClusterInfo.shardIdentifierFromRouting(shardId, false), thisShardSize); + shardSizes.put(shardIdentifierFromRouting(shardId, false), thisShardSize); totalShardsSize += thisShardSize; if (replicaNodeId != null) { dataPath.put(new NodeAndShard(replicaNodeId, shardId), "/data"); @@ -862,6 +870,146 @@ private static ClusterInfo createClusterInfo(List diskUsages, MapnewHashMapWithExpectedSize(5); + var snapshotShardSizes = Maps.newHashMapWithExpectedSize(5); + + var routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY); + // index-1 is allocated according to the desired balance + var indexMetadata1 = IndexMetadata.builder("index-1").settings(indexSettings(IndexVersion.current(), 2, 0)).build(); + routingTableBuilder.add( + IndexRoutingTable.builder(indexMetadata1.getIndex()) + .addShard(newShardRouting(shardIdFrom(indexMetadata1, 0), "node-1", true, STARTED)) + .addShard(newShardRouting(shardIdFrom(indexMetadata1, 1), "node-2", true, STARTED)) + ); + shardSizeInfo.put(shardIdentifierFromRouting(shardIdFrom(indexMetadata1, 0), true), ByteSizeValue.ofGb(8).getBytes()); + shardSizeInfo.put(shardIdentifierFromRouting(shardIdFrom(indexMetadata1, 1), true), ByteSizeValue.ofGb(8).getBytes()); + + // index-2 & index-3 are restored as new from snapshot + var indexMetadata2 = IndexMetadata.builder("index-2") + .settings(indexSettings(IndexVersion.current(), 1, 0).put(IndexMetadata.INDEX_PRIORITY_SETTING.getKey(), 2)) + .build(); + routingTableBuilder.addAsNewRestore( + indexMetadata2, + new RecoverySource.SnapshotRecoverySource("restore", snapshot, IndexVersion.current(), indexIdFrom(indexMetadata2)), + Set.of() + ); + snapshotShardSizes.put( + new SnapshotShard(snapshot, indexIdFrom(indexMetadata2), shardIdFrom(indexMetadata2, 0)), + ByteSizeValue.ofGb(1).getBytes() + ); + + var indexMetadata3 = IndexMetadata.builder("index-3") + .settings(indexSettings(IndexVersion.current(), 2, 0).put(IndexMetadata.INDEX_PRIORITY_SETTING.getKey(), 1)) + .build(); + routingTableBuilder.addAsNewRestore( + indexMetadata3, + new RecoverySource.SnapshotRecoverySource("restore", snapshot, IndexVersion.current(), indexIdFrom(indexMetadata3)), + Set.of() + ); + snapshotShardSizes.put( + new SnapshotShard(snapshot, indexIdFrom(indexMetadata3), shardIdFrom(indexMetadata3, 0)), + ByteSizeValue.ofMb(512).getBytes() + ); + snapshotShardSizes.put( + new SnapshotShard(snapshot, indexIdFrom(indexMetadata3), shardIdFrom(indexMetadata3, 1)), + ByteSizeValue.ofMb(512).getBytes() + ); + + var clusterState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(newNode("node-1")).add(newNode("node-2"))) + .metadata(Metadata.builder().put(indexMetadata1, false).put(indexMetadata2, false).put(indexMetadata3, false).build()) + .routingTable(routingTableBuilder) + .customs( + Map.of( + RestoreInProgress.TYPE, + new RestoreInProgress.Builder().add( + new RestoreInProgress.Entry( + "restore", + snapshot, + RestoreInProgress.State.STARTED, + randomBoolean(), + List.of(indexMetadata2.getIndex().getName(), indexMetadata3.getIndex().getName()), + Map.ofEntries( + Map.entry(shardIdFrom(indexMetadata2, 0), new RestoreInProgress.ShardRestoreStatus(randomUUID())), + Map.entry(shardIdFrom(indexMetadata3, 0), new RestoreInProgress.ShardRestoreStatus(randomUUID())), + Map.entry(shardIdFrom(indexMetadata3, 1), new RestoreInProgress.ShardRestoreStatus(randomUUID())) + ) + ) + ).build() + ) + ) + .build(); + + var clusterInfo = createClusterInfo( + List.of( + // node-1 has enough space to only allocate the only [index-2] shard + new DiskUsage("node-1", "data-1", "/data", ByteSizeValue.ofGb(10).getBytes(), ByteSizeValue.ofGb(2).getBytes()), + // node-2 has enough space to only allocate both shards of [index-3] + new DiskUsage("node-2", "data-2", "/data", ByteSizeValue.ofGb(10).getBytes(), ByteSizeValue.ofGb(2).getBytes()) + ), + shardSizeInfo + ); + var snapshotShardSizeInfo = new SnapshotShardSizeInfo(snapshotShardSizes); + + var settings = Settings.EMPTY; + var allocation = new RoutingAllocation( + randomAllocationDeciders(settings, createBuiltInClusterSettings(settings)), + clusterState, + clusterInfo, + snapshotShardSizeInfo, + 0L + ); + var initialDesiredBalance = new DesiredBalance( + 1, + Map.ofEntries( + Map.entry(shardIdFrom(indexMetadata1, 0), new ShardAssignment(Set.of("node-1"), 1, 0, 0)), + Map.entry(shardIdFrom(indexMetadata1, 1), new ShardAssignment(Set.of("node-2"), 1, 0, 0)) + ) + ); + var nextDesiredBalance = createDesiredBalanceComputer(new BalancedShardsAllocator()).compute( + initialDesiredBalance, + new DesiredBalanceInput(2, allocation, List.of()), + queue(), + input -> true + ); + + assertThat( + nextDesiredBalance.assignments(), + anyOf( + equalTo( + Map.ofEntries( + Map.entry(shardIdFrom(indexMetadata1, 0), new ShardAssignment(Set.of("node-1"), 1, 0, 0)), + Map.entry(shardIdFrom(indexMetadata1, 1), new ShardAssignment(Set.of("node-2"), 1, 0, 0)), + Map.entry(shardIdFrom(indexMetadata2, 0), new ShardAssignment(Set.of("node-1"), 1, 0, 0)), + Map.entry(shardIdFrom(indexMetadata3, 0), new ShardAssignment(Set.of("node-2"), 1, 0, 0)), + Map.entry(shardIdFrom(indexMetadata3, 1), new ShardAssignment(Set.of("node-2"), 1, 0, 0)) + ) + ), + equalTo( + Map.ofEntries( + Map.entry(shardIdFrom(indexMetadata1, 0), new ShardAssignment(Set.of("node-1"), 1, 0, 0)), + Map.entry(shardIdFrom(indexMetadata1, 1), new ShardAssignment(Set.of("node-2"), 1, 0, 0)), + Map.entry(shardIdFrom(indexMetadata2, 0), new ShardAssignment(Set.of("node-2"), 1, 0, 0)), + Map.entry(shardIdFrom(indexMetadata3, 0), new ShardAssignment(Set.of("node-1"), 1, 0, 0)), + Map.entry(shardIdFrom(indexMetadata3, 1), new ShardAssignment(Set.of("node-1"), 1, 0, 0)) + ) + ) + ) + ); + } + + private static IndexId indexIdFrom(IndexMetadata indexMetadata) { + return new IndexId(indexMetadata.getIndex().getName(), indexMetadata.getIndex().getUUID()); + } + + private static ShardId shardIdFrom(IndexMetadata indexMetadata, int shardId) { + return new ShardId(indexMetadata.getIndex(), shardId); + } + public void testShouldLogComputationIteration() { checkIterationLogging( 999, @@ -943,7 +1091,7 @@ public ShardAllocationDecision decideShardAllocation(ShardRouting shard, Routing } private static Map.Entry indexSize(ClusterState clusterState, String name, long size, boolean primary) { - return Map.entry(ClusterInfo.shardIdentifierFromRouting(findShardId(clusterState, name), primary), size); + return Map.entry(shardIdentifierFromRouting(findShardId(clusterState, name), primary), size); } private static ShardId findShardId(ClusterState clusterState, String name) { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java index b54480cdc0856..74d8bc62ff203 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java @@ -43,15 +43,15 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.shard.ShardId; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import static java.util.Collections.emptySet; +import static org.elasticsearch.cluster.ClusterInfo.shardIdentifierFromRouting; import static org.elasticsearch.cluster.routing.ExpectedShardSizeEstimator.getExpectedShardSize; +import static org.elasticsearch.cluster.routing.TestShardRouting.newShardRouting; import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE; import static org.hamcrest.Matchers.containsString; @@ -512,93 +512,64 @@ public void testShardSizeAndRelocatingSize() { } public void testTakesIntoAccountExpectedSizeForInitializingSearchableSnapshots() { - String mainIndexName = "test"; - Index index = new Index(mainIndexName, "1234"); - String anotherIndexName = "another_index"; - Index anotherIndex = new Index(anotherIndexName, "5678"); - Metadata metadata = Metadata.builder() - .put( - IndexMetadata.builder(mainIndexName) - .settings( - settings(IndexVersion.current()).put("index.uuid", "1234") - .put(INDEX_STORE_TYPE_SETTING.getKey(), SEARCHABLE_SNAPSHOT_STORE_TYPE) - ) - .numberOfShards(3) - .numberOfReplicas(1) - ) - .put( - IndexMetadata.builder(anotherIndexName) - .settings(settings(IndexVersion.current()).put("index.uuid", "5678")) - .numberOfShards(1) - .numberOfReplicas(1) - ) + + var searchableSnapshotIndex = IndexMetadata.builder("searchable_snapshot") + .settings(indexSettings(IndexVersion.current(), 3, 0).put(INDEX_STORE_TYPE_SETTING.getKey(), SEARCHABLE_SNAPSHOT_STORE_TYPE)) .build(); + var regularIndex = IndexMetadata.builder("regular_index").settings(indexSettings(IndexVersion.current(), 1, 0)).build(); + String nodeId = "node1"; - String anotherNodeId = "another_node"; - - List shards = new ArrayList<>(); - int anotherNodeShardCounter = 0; - int nodeShardCounter = 0; - Map initializingShardSizes = new HashMap<>(); - for (int i = 1; i <= 3; i++) { - int expectedSize = 10 * i; - shards.add(createShard(index, nodeId, nodeShardCounter++, expectedSize)); - if (randomBoolean()) { - ShardRouting initializingShard = ShardRoutingHelper.initialize( - ShardRouting.newUnassigned( - new ShardId(index, nodeShardCounter++), - true, - EmptyStoreRecoverySource.INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"), - ShardRouting.Role.DEFAULT - ), - nodeId - ); - initializingShardSizes.put(ClusterInfo.shardIdentifierFromRouting(initializingShard), randomLongBetween(10, 50)); - shards.add(initializingShard); - } - // randomly add shards for non-searchable snapshot index - if (randomBoolean()) { - for (int j = 0; j < randomIntBetween(1, 5); j++) { - shards.add(createShard(anotherIndex, anotherNodeId, anotherNodeShardCounter++, expectedSize)); - } - } + var knownShardSizes = new HashMap(); + long unaccountedSearchableSnapshotSizes = 0; + long relocatingShardsSizes = 0; + + var searchableSnapshotIndexRoutingTableBuilder = IndexRoutingTable.builder(searchableSnapshotIndex.getIndex()); + for (int i = 0; i < searchableSnapshotIndex.getNumberOfShards(); i++) { + long expectedSize = randomLongBetween(10, 50); + // a searchable snapshot shard without corresponding entry in cluster info + ShardRouting startedShardWithExpectedSize = newShardRouting( + new ShardId(searchableSnapshotIndex.getIndex(), i), + nodeId, + true, + ShardRoutingState.STARTED, + expectedSize + ); + searchableSnapshotIndexRoutingTableBuilder.addShard(startedShardWithExpectedSize); + unaccountedSearchableSnapshotSizes += expectedSize; + } + var regularIndexRoutingTableBuilder = IndexRoutingTable.builder(regularIndex.getIndex()); + for (int i = 0; i < searchableSnapshotIndex.getNumberOfShards(); i++) { + var shardSize = randomLongBetween(10, 50); + // a shard relocating to this node + ShardRouting initializingShard = newShardRouting( + new ShardId(regularIndex.getIndex(), i), + nodeId, + true, + ShardRoutingState.INITIALIZING, + PeerRecoverySource.INSTANCE + ); + regularIndexRoutingTableBuilder.addShard(initializingShard); + knownShardSizes.put(shardIdentifierFromRouting(initializingShard), shardSize); + relocatingShardsSizes += shardSize; } - DiscoveryNode node = DiscoveryNodeUtils.builder(nodeId).roles(emptySet()).build(); - DiscoveryNode anotherNode = DiscoveryNodeUtils.builder(anotherNodeId).roles(emptySet()).build(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) - .metadata(metadata) - .routingTable( - RoutingTable.builder() - .add( - shards.stream() - .filter(s -> s.getIndexName().equals(mainIndexName)) - .reduce(IndexRoutingTable.builder(index), IndexRoutingTable.Builder::addShard, (a, b) -> a) - ) - .add( - shards.stream() - .filter(s -> s.getIndexName().equals(anotherIndexName)) - .reduce(IndexRoutingTable.builder(anotherIndex), IndexRoutingTable.Builder::addShard, (a, b) -> a) - ) - .build() - ) - .nodes(DiscoveryNodes.builder().add(node).add(anotherNode).build()) + .metadata(Metadata.builder().put(searchableSnapshotIndex, false).put(regularIndex, false)) + .routingTable(RoutingTable.builder().add(searchableSnapshotIndexRoutingTableBuilder).add(regularIndexRoutingTableBuilder)) + .nodes(DiscoveryNodes.builder().add(newNode(nodeId)).build()) .build(); + RoutingAllocation allocation = new RoutingAllocation( null, clusterState, - new DevNullClusterInfo(Map.of(), Map.of(), initializingShardSizes), + new DevNullClusterInfo(Map.of(), Map.of(), knownShardSizes), null, 0 ); - long sizeOfUnaccountedShards = sizeOfUnaccountedShards( - allocation, - RoutingNodesHelper.routingNode(nodeId, node, shards.toArray(ShardRouting[]::new)), - false, - "/dev/null" + assertEquals( + unaccountedSearchableSnapshotSizes + relocatingShardsSizes, + sizeOfUnaccountedShards(allocation, clusterState.getRoutingNodes().node(nodeId), false, "/dev/null") ); - assertEquals(60L + initializingShardSizes.values().stream().mapToLong(Long::longValue).sum(), sizeOfUnaccountedShards); } private ShardRouting createShard(Index index, String nodeId, int i, int expectedSize) { @@ -620,6 +591,7 @@ public long sizeOfUnaccountedShards(RoutingAllocation allocation, RoutingNode no subtractShardsMovingAway, dataPath, allocation.clusterInfo(), + allocation.snapshotShardSizeInfo(), allocation.metadata(), allocation.routingTable(), allocation.unaccountedSearchableSnapshotSize(node) diff --git a/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTestUtils.java b/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTestUtils.java index 2c1daa09340d7..3185769bdab82 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTestUtils.java +++ b/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTestUtils.java @@ -61,11 +61,11 @@ public static void pruneTombstones(LiveVersionMap map, long maxTimestampToPrune, map.pruneTombstones(maxTimestampToPrune, maxSeqNoToPrune); } - static IndexVersionValue randomIndexVersionValue() { + public static IndexVersionValue randomIndexVersionValue() { return new IndexVersionValue(randomTranslogLocation(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()); } - static Translog.Location randomTranslogLocation() { + public static Translog.Location randomTranslogLocation() { if (randomBoolean()) { return null; } else { @@ -93,6 +93,10 @@ public static boolean isSafeAccessRequired(LiveVersionMap map) { return map.isSafeAccessRequired(); } + public static void enforceSafeAccess(LiveVersionMap map) { + map.enforceSafeAccess(); + } + public static LiveVersionMapArchive getArchive(LiveVersionMap map) { return map.getArchive(); } diff --git a/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java b/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java index a9b42ccdef248..5ca7aadc35fa7 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java @@ -13,7 +13,9 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.Constants; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.test.ESTestCase; @@ -23,11 +25,16 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; +import java.util.stream.IntStream; +import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency; +import static org.elasticsearch.core.Tuple.tuple; import static org.elasticsearch.index.engine.LiveVersionMapTestUtils.randomIndexVersionValue; import static org.elasticsearch.index.engine.LiveVersionMapTestUtils.randomTranslogLocation; import static org.hamcrest.Matchers.empty; @@ -36,7 +43,6 @@ import static org.hamcrest.Matchers.nullValue; public class LiveVersionMapTests extends ESTestCase { - public void testRamBytesUsed() throws Exception { LiveVersionMap map = new LiveVersionMap(); for (int i = 0; i < 100000; ++i) { @@ -442,4 +448,51 @@ public void testRandomlyIndexDeleteAndRefresh() throws Exception { } } } + + public void testVersionLookupRamBytesUsed() { + var vl = new LiveVersionMap.VersionLookup(newConcurrentMapWithAggressiveConcurrency()); + assertEquals(0, vl.ramBytesUsed()); + Set existingKeys = new HashSet<>(); + Supplier> randomEntry = () -> { + var key = randomBoolean() || existingKeys.isEmpty() ? uid(randomIdentifier()) : randomFrom(existingKeys); + return tuple(key, randomIndexVersionValue()); + }; + IntStream.range(0, randomIntBetween(10, 100)).forEach(i -> { + switch (randomIntBetween(0, 2)) { + case 0: // put + var entry = randomEntry.get(); + var previousValue = vl.put(entry.v1(), entry.v2()); + if (existingKeys.contains(entry.v1())) { + assertNotNull(previousValue); + } else { + assertNull(previousValue); + existingKeys.add(entry.v1()); + } + break; + case 1: // remove + if (existingKeys.isEmpty() == false) { + var key = randomFrom(existingKeys); + assertNotNull(vl.remove(key)); + existingKeys.remove(key); + } + break; + case 2: // merge + var toMerge = new LiveVersionMap.VersionLookup(ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency()); + IntStream.range(0, randomIntBetween(1, 100)) + .mapToObj(n -> randomEntry.get()) + .forEach(kv -> toMerge.put(kv.v1(), kv.v2())); + vl.merge(toMerge); + existingKeys.addAll(toMerge.getMap().keySet()); + break; + default: + throw new IllegalStateException("branch value unexpected"); + } + }); + long actualRamBytesUsed = vl.getMap() + .entrySet() + .stream() + .mapToLong(entry -> LiveVersionMap.VersionLookup.mapEntryBytesUsed(entry.getKey(), entry.getValue())) + .sum(); + assertEquals(actualRamBytesUsed, vl.ramBytesUsed()); + } } diff --git a/settings.gradle b/settings.gradle index b3a33e11c4ec4..90422913ef441 100644 --- a/settings.gradle +++ b/settings.gradle @@ -97,6 +97,7 @@ List projects = [ 'test:fixtures:minio-fixture', 'test:fixtures:old-elasticsearch', 'test:fixtures:s3-fixture', + 'test:fixtures:testcontainer-utils', 'test:fixtures:geoip-fixture', 'test:fixtures:url-fixture', 'test:fixtures:nginx-fixture', diff --git a/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/MetricsApmIT.java b/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/MetricsApmIT.java index f6c5fdfe4db5c..93d08fbccd376 100644 --- a/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/MetricsApmIT.java +++ b/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/MetricsApmIT.java @@ -56,7 +56,6 @@ protected String getTestRestCluster() { return cluster.getHttpAddresses(); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103286") @SuppressWarnings("unchecked") public void testApmIntegration() throws Exception { Map>> sampleAssertions = new HashMap<>( @@ -108,8 +107,8 @@ public void testApmIntegration() throws Exception { client().performRequest(new Request("GET", "/_use_apm_metrics")); - finished.await(30, TimeUnit.SECONDS); - assertThat(sampleAssertions, Matchers.anEmptyMap()); + assertTrue("Timeout when waiting for assertions to complete.", finished.await(30, TimeUnit.SECONDS)); + assertThat(sampleAssertions, Matchers.equalTo(Collections.emptyMap())); } private Map.Entry>> assertion( diff --git a/test/fixtures/minio-fixture/build.gradle b/test/fixtures/minio-fixture/build.gradle index 66613809068f7..9a71387d7c6b7 100644 --- a/test/fixtures/minio-fixture/build.gradle +++ b/test/fixtures/minio-fixture/build.gradle @@ -5,7 +5,6 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -apply plugin: 'java' apply plugin: 'elasticsearch.java' apply plugin: 'elasticsearch.cache-test-fixtures' @@ -19,6 +18,7 @@ dependencies { testImplementation project(':test:framework') api "junit:junit:${versions.junit}" + api project(':test:fixtures:testcontainer-utils') api "org.testcontainers:testcontainers:${versions.testcontainer}" implementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" implementation "org.slf4j:slf4j-api:${versions.slf4j}" diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java index a7e6ba8d785a1..671632f2e0125 100644 --- a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java +++ b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java @@ -8,12 +8,10 @@ package org.elasticsearch.test.fixtures.minio; -import org.elasticsearch.test.fixtures.CacheableTestFixture; import org.elasticsearch.test.fixtures.testcontainers.DockerEnvironmentAwareTestContainer; -import org.junit.rules.TestRule; import org.testcontainers.images.builder.ImageFromDockerfile; -public final class MinioTestContainer extends DockerEnvironmentAwareTestContainer implements TestRule, CacheableTestFixture { +public final class MinioTestContainer extends DockerEnvironmentAwareTestContainer { private static final int servicePort = 9000; public static final String DOCKER_BASE_IMAGE = "minio/minio:RELEASE.2021-03-01T04-20-55Z"; @@ -25,7 +23,7 @@ public MinioTestContainer() { public MinioTestContainer(boolean enabled) { super( - new ImageFromDockerfile().withDockerfileFromBuilder( + new ImageFromDockerfile("es-minio-testfixture").withDockerfileFromBuilder( builder -> builder.from(DOCKER_BASE_IMAGE) .env("MINIO_ACCESS_KEY", "s3_test_access_key") .env("MINIO_SECRET_KEY", "s3_test_secret_key") @@ -50,13 +48,4 @@ public void start() { public String getAddress() { return "http://127.0.0.1:" + getMappedPort(servicePort); } - - public void cache() { - try { - start(); - stop(); - } catch (RuntimeException e) { - logger().warn("Error while caching container images.", e); - } - } } diff --git a/test/fixtures/testcontainer-utils/build.gradle b/test/fixtures/testcontainer-utils/build.gradle new file mode 100644 index 0000000000000..80886d99087c9 --- /dev/null +++ b/test/fixtures/testcontainer-utils/build.gradle @@ -0,0 +1,15 @@ +apply plugin: 'elasticsearch.java' + + +configurations.all { + transitive = false +} + +dependencies { + testImplementation project(':test:framework') + api "junit:junit:${versions.junit}" + api "org.testcontainers:testcontainers:${versions.testcontainer}" + implementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" + implementation "org.slf4j:slf4j-api:${versions.slf4j}" + implementation "com.github.docker-java:docker-java-api:${versions.dockerJava}" +} diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/CacheableTestFixture.java b/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/CacheableTestFixture.java similarity index 100% rename from test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/CacheableTestFixture.java rename to test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/CacheableTestFixture.java diff --git a/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/ResourceUtils.java b/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/ResourceUtils.java new file mode 100644 index 0000000000000..8fe64ea34d8d4 --- /dev/null +++ b/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/ResourceUtils.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.test.fixtures; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +public final class ResourceUtils { + + public static Path copyResourceToFile(Class clazz, Path targetFolder, String resourcePath) { + try { + ClassLoader classLoader = clazz.getClassLoader(); + URL resourceUrl = classLoader.getResource(resourcePath); + if (resourceUrl == null) { + throw new RuntimeException("Failed to load " + resourcePath + " from classpath"); + } + InputStream inputStream = resourceUrl.openStream(); + File outputFile = new File(targetFolder.toFile(), resourcePath.substring(resourcePath.lastIndexOf("/"))); + Files.copy(inputStream, outputFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + return outputFile.toPath(); + } catch (IOException e) { + throw new RuntimeException("Failed to load ca.jks from classpath", e); + } + } +} diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java b/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java similarity index 88% rename from test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java rename to test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java index c0fb83e5206f4..ce4d6fda861cd 100644 --- a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java +++ b/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java @@ -8,12 +8,14 @@ package org.elasticsearch.test.fixtures.testcontainers; -import org.elasticsearch.test.fixtures.minio.MinioTestContainer; +import org.elasticsearch.test.fixtures.CacheableTestFixture; import org.junit.Assume; +import org.junit.rules.TestRule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.images.builder.ImageFromDockerfile; import java.io.File; @@ -27,7 +29,10 @@ import java.util.Map; import java.util.stream.Collectors; -public class DockerEnvironmentAwareTestContainer extends GenericContainer { +public abstract class DockerEnvironmentAwareTestContainer extends GenericContainer + implements + TestRule, + CacheableTestFixture { protected static final Logger LOGGER = LoggerFactory.getLogger(DockerEnvironmentAwareTestContainer.class); private static final String DOCKER_ON_LINUX_EXCLUSIONS_FILE = ".ci/dockerOnLinuxExclusions"; @@ -59,9 +64,20 @@ public DockerEnvironmentAwareTestContainer(ImageFromDockerfile imageFromDockerfi public void start() { Assume.assumeFalse("Docker support excluded on OS", EXCLUDED_OS); Assume.assumeTrue("Docker probing succesful", DOCKER_PROBING_SUCCESSFUL); + withLogConsumer(new Slf4jLogConsumer(logger())); super.start(); } + @Override + public void cache() { + try { + start(); + stop(); + } catch (RuntimeException e) { + logger().warn("Error while caching container images.", e); + } + } + static String deriveId(Map values) { return values.get("ID") + "-" + values.get("VERSION_ID"); } diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java b/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java similarity index 80% rename from test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java rename to test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java index 1b0dacbacfd1a..f36a3264bffbb 100644 --- a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java +++ b/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java @@ -17,6 +17,8 @@ public class TestContainersThreadFilter implements ThreadFilter { @Override public boolean reject(Thread t) { - return t.getName().startsWith("testcontainers-") || t.getName().startsWith("ducttape"); + return t.getName().startsWith("testcontainers-") + || t.getName().startsWith("ducttape") + || t.getName().startsWith("ForkJoinPool.commonPool-worker-1"); } } diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java index 89c8546d6b7d2..1158e805ba3c1 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java @@ -36,6 +36,16 @@ public static ShardRouting newShardRouting(String index, int shardId, String cur } public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId, boolean primary, ShardRoutingState state) { + return newShardRouting(shardId, currentNodeId, primary, state, -1); + } + + public static ShardRouting newShardRouting( + ShardId shardId, + String currentNodeId, + boolean primary, + ShardRoutingState state, + long expectedShardSize + ) { assertNotEquals(ShardRoutingState.RELOCATING, state); return new ShardRouting( shardId, @@ -47,7 +57,7 @@ public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId buildUnassignedInfo(state), buildRelocationFailureInfo(state), buildAllocationId(state), - -1, + expectedShardSize, ShardRouting.Role.DEFAULT ); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java index 6137b2b0aad18..0a3316b87bd04 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java @@ -281,7 +281,7 @@ static List> alterateQueries(Set queries, Map levels = new LinkedList<>(); @@ -388,7 +388,7 @@ private void queryWrappedInArrayTest(String queryName, String validQuery) { + "[" + validQuery.substring(insertionPosition, endArrayPosition) + "]" - + validQuery.substring(endArrayPosition, validQuery.length()); + + validQuery.substring(endArrayPosition); ParsingException e = expectThrows(ParsingException.class, () -> parseQuery(testQuery)); assertEquals("[" + queryName + "] query malformed, no start_object after query name", e.getMessage()); @@ -464,13 +464,15 @@ public void testToQuery() throws IOException { assertNotNull("toQuery should not return null", firstLuceneQuery); assertLuceneQuery(firstQuery, firstLuceneQuery, context); // remove after assertLuceneQuery since the assertLuceneQuery impl might access the context as well - assertTrue( + assertEquals( "query is not equal to its copy after calling toQuery, firstQuery: " + firstQuery + ", secondQuery: " + controlQuery, - firstQuery.equals(controlQuery) + firstQuery, + controlQuery ); - assertTrue( + assertEquals( "equals is not symmetric after calling toQuery, firstQuery: " + firstQuery + ", secondQuery: " + controlQuery, - controlQuery.equals(firstQuery) + controlQuery, + firstQuery ); assertThat( "query copy's hashcode is different from original hashcode after calling toQuery, firstQuery: " @@ -552,7 +554,7 @@ protected boolean supportsQueryName() { * and {@link SearchExecutionContext}. Verifies that named queries and boost are properly handled and delegates to * {@link #doAssertLuceneQuery(AbstractQueryBuilder, Query, SearchExecutionContext)} for query specific checks. */ - private void assertLuceneQuery(QB queryBuilder, Query query, SearchExecutionContext context) throws IOException { + protected void assertLuceneQuery(QB queryBuilder, Query query, SearchExecutionContext context) throws IOException { if (queryBuilder.queryName() != null && query instanceof MatchNoDocsQuery == false) { Query namedQuery = context.copyNamedQueries().get(queryBuilder.queryName()); assertThat(namedQuery, equalTo(query)); @@ -671,7 +673,7 @@ protected QB changeNameOrBoost(QB original) throws IOException { // we use the streaming infra to create a copy of the query provided as argument @SuppressWarnings("unchecked") - private QB copyQuery(QB query) throws IOException { + protected QB copyQuery(QB query) throws IOException { Reader reader = (Reader) namedWriteableRegistry().getReader(QueryBuilder.class, query.getWriteableName()); return copyWriteable(query, namedWriteableRegistry(), reader); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index fee38d09336f1..96502c18b7e20 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -26,12 +26,12 @@ import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest; import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse; +import org.elasticsearch.action.admin.cluster.tasks.TransportPendingClusterTasksAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.flush.FlushResponse; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; @@ -119,6 +119,7 @@ import org.elasticsearch.indices.IndicesQueryCache; import org.elasticsearch.indices.IndicesRequestCache; import org.elasticsearch.indices.store.IndicesStore; +import org.elasticsearch.monitor.jvm.HotThreads; import org.elasticsearch.node.NodeMocksPlugin; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.plugins.NetworkPlugin; @@ -153,6 +154,7 @@ import org.junit.BeforeClass; import java.io.IOException; +import java.io.StringWriter; import java.lang.annotation.Annotation; import java.lang.annotation.ElementType; import java.lang.annotation.Inherited; @@ -860,7 +862,10 @@ public void waitNoPendingTasksOnAll() throws Exception { for (Client client : clients()) { ClusterHealthResponse clusterHealth = client.admin().cluster().prepareHealth().setLocal(true).get(); assertThat("client " + client + " still has in flight fetch", clusterHealth.getNumberOfInFlightFetch(), equalTo(0)); - PendingClusterTasksResponse pendingTasks = client.admin().cluster().preparePendingClusterTasks().setLocal(true).get(); + PendingClusterTasksResponse pendingTasks = client.execute( + TransportPendingClusterTasksAction.TYPE, + new PendingClusterTasksRequest().local(true) + ).get(); assertThat( "client " + client + " still has pending tasks " + pendingTasks, pendingTasks.pendingTasks(), @@ -971,17 +976,24 @@ private ClusterHealthStatus ensureColor( final var allocationExplainRef = new AtomicReference(); final var clusterStateRef = new AtomicReference(); final var pendingTasksRef = new AtomicReference(); - final var hotThreadsRef = new AtomicReference(); + final var hotThreadsRef = new AtomicReference(); final var detailsFuture = new PlainActionFuture(); try (var listeners = new RefCountingListener(detailsFuture)) { clusterAdmin().prepareAllocationExplain().execute(listeners.acquire(allocationExplainRef::set)); clusterAdmin().prepareState().execute(listeners.acquire(clusterStateRef::set)); - clusterAdmin().preparePendingClusterTasks().execute(listeners.acquire(pendingTasksRef::set)); - clusterAdmin().prepareNodesHotThreads() - .setThreads(9999) - .setIgnoreIdleThreads(false) - .execute(listeners.acquire(hotThreadsRef::set)); + client().execute( + TransportPendingClusterTasksAction.TYPE, + new PendingClusterTasksRequest(), + listeners.acquire(pendingTasksRef::set) + ); + try (var writer = new StringWriter()) { + new HotThreads().busiestThreads(9999).ignoreIdleThreads(false).detect(writer); + hotThreadsRef.set(writer.toString()); + } catch (Exception e) { + logger.error("exception capturing hot threads", e); + hotThreadsRef.set("exception capturing hot threads: " + e); + } } try { @@ -996,10 +1008,7 @@ private ClusterHealthStatus ensureColor( safeFormat(allocationExplainRef.get(), r -> Strings.toString(r.getExplanation(), true, true)), safeFormat(clusterStateRef.get(), r -> r.getState().toString()), safeFormat(pendingTasksRef.get(), r -> Strings.toString(r, true, true)), - safeFormat( - hotThreadsRef.get(), - r -> r.getNodes().stream().map(NodeHotThreads::getHotThreads).collect(Collectors.joining("\n")) - ) + hotThreadsRef.get() ); fail("timed out waiting for " + color + " state"); } @@ -1039,7 +1048,7 @@ public ClusterHealthStatus waitForRelocation(ClusterHealthStatus status) { "waitForRelocation timed out (status={}), cluster state:\n{}\n{}", status, clusterAdmin().prepareState().get().getState(), - clusterAdmin().preparePendingClusterTasks().get() + getClusterPendingTasks() ); assertThat("timed out waiting for relocation", actionGet.isTimedOut(), equalTo(false)); } @@ -1049,6 +1058,18 @@ public ClusterHealthStatus waitForRelocation(ClusterHealthStatus status) { return actionGet.getStatus(); } + public static PendingClusterTasksResponse getClusterPendingTasks() { + return getClusterPendingTasks(client()); + } + + public static PendingClusterTasksResponse getClusterPendingTasks(Client client) { + try { + return client.execute(TransportPendingClusterTasksAction.TYPE, new PendingClusterTasksRequest()).get(10, TimeUnit.SECONDS); + } catch (Exception e) { + return fail(e); + } + } + /** * Waits until at least a give number of document is visible for searchers * @@ -1145,11 +1166,7 @@ public static DiscoveryNode waitAndGetHealthNode(InternalTestCluster internalClu * Prints the current cluster state as debug logging. */ public void logClusterState() { - logger.debug( - "cluster state:\n{}\n{}", - clusterAdmin().prepareState().get().getState(), - clusterAdmin().preparePendingClusterTasks().get() - ); + logger.debug("cluster state:\n{}\n{}", clusterAdmin().prepareState().get().getState(), getClusterPendingTasks()); } protected void ensureClusterSizeConsistency() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 1517571878fa2..b201d58ac0e23 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -421,7 +421,7 @@ public ClusterHealthStatus ensureGreen(TimeValue timeout, String... indices) { logger.info( "ensureGreen timed out, cluster state:\n{}\n{}", clusterAdmin().prepareState().get().getState(), - clusterAdmin().preparePendingClusterTasks().get() + ESIntegTestCase.getClusterPendingTasks(client()) ); assertThat("timed out waiting for green state", actionGet.isTimedOut(), equalTo(false)); } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java index 2c313da69b42e..49fb38b518dce 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java @@ -16,7 +16,6 @@ */ public enum FeatureFlag { TIME_SERIES_MODE("es.index_mode_feature_flag_registered=true", Version.fromString("8.0.0"), null), - LEARNING_TO_RANK("es.learning_to_rank_feature_flag_enabled=true", Version.fromString("8.12.0"), null), FAILURE_STORE_ENABLED("es.failure_store_feature_flag_enabled=true", Version.fromString("8.12.0"), null); public final String systemProperty; diff --git a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java index eb9d440106dea..9189cdff74547 100644 --- a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java +++ b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java @@ -137,6 +137,7 @@ public void testThatIndependentTemplatesAreAddedImmediatelyIfMissing() throws Ex assertThat(actualInstalledIndexTemplates.get(), equalTo(0)); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102797") public void testIngestPipelines() { DiscoveryNode node = DiscoveryNodeUtils.create("node"); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java index a15ff51fc24ee..12291799b430a 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java @@ -55,12 +55,12 @@ import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider; import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -520,25 +520,26 @@ public void testUnmovableSize() { stateBuilder.metadata(metaBuilder); ClusterState clusterState = stateBuilder.build(); - Set shards = IntStream.range(0, between(1, 10)) + var shards = IntStream.range(0, between(1, 10)) .mapToObj(i -> Tuple.tuple(new ShardId(indexMetadata.getIndex(), randomInt(10)), randomBoolean())) .distinct() .map(t -> TestShardRouting.newShardRouting(t.v1(), nodeId, t.v2(), ShardRoutingState.STARTED)) - .collect(Collectors.toSet()); + .toList(); long minShardSize = randomLongBetween(1, 10); - Map diskUsages = new HashMap<>(); - diskUsages.put(nodeId, new DiskUsage(nodeId, null, null, ByteSizeUnit.KB.toBytes(100), ByteSizeUnit.KB.toBytes(5))); - Map shardSize = new HashMap<>(); ShardRouting missingShard = randomBoolean() ? randomFrom(shards) : null; - Collection shardsWithSizes = shards.stream().filter(s -> s != missingShard).collect(Collectors.toSet()); - for (ShardRouting shard : shardsWithSizes) { - shardSize.put(shardIdentifier(shard), ByteSizeUnit.KB.toBytes(randomLongBetween(minShardSize, 100))); + Map shardSize = new HashMap<>(); + for (ShardRouting shard : shards) { + if (shard != missingShard) { + shardSize.put(shardIdentifier(shard), ByteSizeUnit.KB.toBytes(randomLongBetween(minShardSize, 100))); + } } - if (shardsWithSizes.isEmpty() == false) { - shardSize.put(shardIdentifier(randomFrom(shardsWithSizes)), ByteSizeUnit.KB.toBytes(minShardSize)); + if (shardSize.isEmpty() == false) { + shardSize.put(randomFrom(shardSize.keySet()), ByteSizeUnit.KB.toBytes(minShardSize)); } + + var diskUsages = Map.of(nodeId, new DiskUsage(nodeId, null, null, ByteSizeUnit.KB.toBytes(100), ByteSizeUnit.KB.toBytes(5))); ClusterInfo info = new ClusterInfo(diskUsages, diskUsages, shardSize, Map.of(), Map.of(), Map.of()); ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState( @@ -553,11 +554,12 @@ public void testUnmovableSize() { ); long result = allocationState.unmovableSize(nodeId, shards); - if (missingShard != null - && (missingShard.primary() - || clusterState.getRoutingNodes().activePrimary(missingShard.shardId()) == null - || info.getShardSize(clusterState.getRoutingNodes().activePrimary(missingShard.shardId())) == null) - || minShardSize < 5) { + + Predicate shardSizeKnown = shard -> shard.primary() + ? info.getShardSize(shard.shardId(), true) != null + : info.getShardSize(shard.shardId(), true) != null || info.getShardSize(shard.shardId(), false) != null; + + if ((missingShard != null && shardSizeKnown.test(missingShard) == false) || minShardSize < 5) { // the diff between used and high watermark is 5 KB. assertThat(result, equalTo(ByteSizeUnit.KB.toBytes(5))); } else { diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java index 047a2d6225035..ea4bc8c92047a 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java @@ -7,10 +7,10 @@ package org.elasticsearch.xpack; +import org.apache.logging.log4j.Level; import org.apache.lucene.store.AlreadyClosedException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; @@ -38,6 +38,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.network.NetworkModule; @@ -59,6 +60,7 @@ import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.license.LicenseSettings; import org.elasticsearch.license.LicensesMetadata; +import org.elasticsearch.monitor.jvm.HotThreads; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchResponseUtils; @@ -421,24 +423,19 @@ private ClusterHealthStatus ensureColor( {} timed out: leader cluster state: {} - leader cluster hot threads: - {} leader cluster tasks: {} follower cluster state: {} - follower cluster hot threads: - {} follower cluster tasks: {}""", method, leaderClient().admin().cluster().prepareState().get().getState(), - getHotThreads(leaderClient()), - leaderClient().admin().cluster().preparePendingClusterTasks().get(), + ESIntegTestCase.getClusterPendingTasks(leaderClient()), followerClient().admin().cluster().prepareState().get().getState(), - getHotThreads(followerClient()), - followerClient().admin().cluster().preparePendingClusterTasks().get() + ESIntegTestCase.getClusterPendingTasks(followerClient()) ); + HotThreads.logLocalHotThreads(logger, Level.INFO, "hot threads at timeout", ReferenceDocs.LOGGING); fail("timed out waiting for " + color + " state"); } assertThat( @@ -450,19 +447,6 @@ private ClusterHealthStatus ensureColor( return actionGet.getStatus(); } - static String getHotThreads(Client client) { - return client.admin() - .cluster() - .prepareNodesHotThreads() - .setThreads(99999) - .setIgnoreIdleThreads(false) - .get() - .getNodes() - .stream() - .map(NodeHotThreads::getHotThreads) - .collect(Collectors.joining("\n")); - } - protected final Index resolveLeaderIndex(String index) { GetIndexResponse getIndexResponse = leaderClient().admin().indices().prepareGetIndex().setIndices(index).get(); assertTrue("index " + index + " not found", getIndexResponse.getSettings().containsKey(index)); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java index ad9ab7088fef5..c05c73bc31ddf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java @@ -374,8 +374,10 @@ public static class TaskParams implements MlTaskParams, Writeable, ToXContentObj // TODO add support for other roles? If so, it may have to be an instance method... // NOTE, whatever determines assignment should not be dynamically set on the node // Otherwise assignment logic might fail - public static boolean mayAssignToNode(DiscoveryNode node) { - return node.getRoles().contains(DiscoveryNodeRole.ML_ROLE) && MlConfigVersion.fromNode(node).onOrAfter(VERSION_INTRODUCED); + public static boolean mayAssignToNode(@Nullable DiscoveryNode node) { + return node != null + && node.getRoles().contains(DiscoveryNodeRole.ML_ROLE) + && MlConfigVersion.fromNode(node).onOrAfter(VERSION_INTRODUCED); } public static final MlConfigVersion VERSION_INTRODUCED = MlConfigVersion.V_8_0_0; diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java index 826c958de4c18..f248da8a7842a 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java @@ -126,11 +126,7 @@ public boolean validateClusterForming() { } })).start(); - waitUntil( - () -> cluster.client().admin().cluster().preparePendingClusterTasks().get().pendingTasks().isEmpty(), - 60, - TimeUnit.SECONDS - ); + waitUntil(() -> getClusterPendingTasks(cluster.client()).pendingTasks().isEmpty(), 60, TimeUnit.SECONDS); ensureStableCluster(cluster.numDataAndMasterNodes()); final String targetIndex = "downsample-5m-" + sourceIndex; diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java index fccd5e6fd0b39..84ffefb067eed 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java @@ -203,7 +203,7 @@ public boolean validateClusterForming() { } })).start(); startDownsampleTaskDuringDisruption(sourceIndex, targetIndex, config, disruptionStart, disruptionEnd); - waitUntil(() -> cluster.client().admin().cluster().preparePendingClusterTasks().get().pendingTasks().isEmpty()); + waitUntil(() -> getClusterPendingTasks(cluster.client()).pendingTasks().isEmpty()); ensureStableCluster(cluster.numDataAndMasterNodes()); assertTargetIndex(cluster, sourceIndex, targetIndex, indexedDocs); } @@ -265,7 +265,7 @@ public boolean validateClusterForming() { })).start(); startDownsampleTaskDuringDisruption(sourceIndex, targetIndex, config, disruptionStart, disruptionEnd); - waitUntil(() -> cluster.client().admin().cluster().preparePendingClusterTasks().get().pendingTasks().isEmpty()); + waitUntil(() -> getClusterPendingTasks(cluster.client()).pendingTasks().isEmpty()); ensureStableCluster(cluster.numDataAndMasterNodes()); assertTargetIndex(cluster, sourceIndex, targetIndex, indexedDocs); } @@ -354,7 +354,7 @@ public boolean validateClusterForming() { })).start(); startDownsampleTaskDuringDisruption(sourceIndex, downsampleIndex, config, disruptionStart, disruptionEnd); - waitUntil(() -> cluster.client().admin().cluster().preparePendingClusterTasks().get().pendingTasks().isEmpty()); + waitUntil(() -> getClusterPendingTasks(cluster.client()).pendingTasks().isEmpty()); ensureStableCluster(cluster.numDataAndMasterNodes()); assertTargetIndex(cluster, sourceIndex, downsampleIndex, indexedDocs); } diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java index 6d3c348c10627..2b2824e3041a8 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java @@ -194,7 +194,7 @@ public boolean validateClusterForming() { final String targetIndex = "downsample-1h-" + sourceIndex; startDownsampleTaskViaIlm(sourceIndex, targetIndex, disruptionStart, disruptionEnd); - waitUntil(() -> cluster.client().admin().cluster().preparePendingClusterTasks().get().pendingTasks().isEmpty()); + waitUntil(() -> getClusterPendingTasks(cluster.client()).pendingTasks().isEmpty(), 60, TimeUnit.SECONDS); ensureStableCluster(cluster.numDataAndMasterNodes()); assertTargetIndex(cluster, targetIndex, indexedDocs); } diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/335_connector_update_configuration.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/335_connector_update_configuration.yml index 260e1784d29e2..5a012853b4bf9 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/335_connector_update_configuration.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/335_connector_update_configuration.yml @@ -85,6 +85,41 @@ setup: - match: { configuration.some_field.value: 456 } - match: { status: configured } +--- +"Update Connector Configuration with null tooltip": + - do: + connector.update_configuration: + connector_id: test-connector + body: + configuration: + some_field: + default_value: null + depends_on: + - field: some_field + value: 31 + display: numeric + label: Very important field + options: [ ] + order: 4 + required: true + sensitive: false + tooltip: null + type: str + ui_restrictions: [ ] + validations: + - constraint: 0 + type: greater_than + value: 123 + + + - match: { result: updated } + + - do: + connector.get: + connector_id: test-connector + + - match: { configuration.some_field.tooltip: null } + --- "Update Connector Configuration - Connector doesn't exist": - do: diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/420_connector_sync_job_check_in.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/420_connector_sync_job_check_in.yml index d08f7f6a51c91..caa6543b6985a 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/420_connector_sync_job_check_in.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/420_connector_sync_job_check_in.yml @@ -34,7 +34,7 @@ setup: connector_sync_job.check_in: connector_sync_job_id: $sync-job-id-to-check-in - - match: { acknowledged: true } + - match: { result: updated } - do: connector_sync_job.get: diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/430_connector_sync_job_cancel.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/430_connector_sync_job_cancel.yml index d934b7c674f25..633c1a8cecb7b 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/430_connector_sync_job_cancel.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/430_connector_sync_job_cancel.yml @@ -27,7 +27,7 @@ setup: connector_sync_job.cancel: connector_sync_job_id: $sync-job-id-to-cancel - - match: { acknowledged: true } + - match: { result: updated } - do: connector_sync_job.get: diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/450_connector_sync_job_error.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/450_connector_sync_job_error.yml index 6f525a2ac2883..a565d28c3e788 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/450_connector_sync_job_error.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/450_connector_sync_job_error.yml @@ -29,7 +29,7 @@ setup: body: error: error - - match: { acknowledged: true } + - match: { result: updated } - do: connector_sync_job.get: diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/460_connector_sync_job_update_stats.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/460_connector_sync_job_update_stats.yml index 0c7300bd2b436..94572870f9164 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/460_connector_sync_job_update_stats.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/460_connector_sync_job_update_stats.yml @@ -32,7 +32,7 @@ setup: indexed_document_count: 20 indexed_document_volume: 1000 - - match: { acknowledged: true } + - match: { result: updated } - do: connector_sync_job.get: @@ -143,7 +143,7 @@ setup: indexed_document_volume: 1000 total_document_count: 20 - - match: { acknowledged: true } + - match: { result: updated } - do: connector_sync_job.get: @@ -173,7 +173,7 @@ setup: indexed_document_volume: 1000 last_seen: 2023-12-04T08:45:50.567149Z - - match: { acknowledged: true } + - match: { result: updated } - do: connector_sync_job.get: diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorConfiguration.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorConfiguration.java index 103c647f180b4..8ed7c417a1af1 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorConfiguration.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorConfiguration.java @@ -54,6 +54,7 @@ public class ConnectorConfiguration implements Writeable, ToXContentObject { private final String placeholder; private final boolean required; private final boolean sensitive; + @Nullable private final String tooltip; private final ConfigurationFieldType type; private final List uiRestrictions; @@ -199,7 +200,7 @@ public ConnectorConfiguration(StreamInput in) throws IOException { PARSER.declareString(optionalConstructorArg(), PLACEHOLDER_FIELD); PARSER.declareBoolean(constructorArg(), REQUIRED_FIELD); PARSER.declareBoolean(constructorArg(), SENSITIVE_FIELD); - PARSER.declareStringOrNull(constructorArg(), TOOLTIP_FIELD); + PARSER.declareStringOrNull(optionalConstructorArg(), TOOLTIP_FIELD); PARSER.declareField( constructorArg(), (p, c) -> ConfigurationFieldType.fieldType(p.text()), diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorUpdateActionResponse.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorUpdateActionResponse.java new file mode 100644 index 0000000000000..b77f04e7d9289 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorUpdateActionResponse.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.application.connector.Connector; +import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob; + +import java.io.IOException; +import java.util.Objects; + +/** + * Represents a response for update actions related to {@link Connector} and {@link ConnectorSyncJob}. + * The response encapsulates the result of the update action, represented by a {@link DocWriteResponse.Result}. + */ +public class ConnectorUpdateActionResponse extends ActionResponse implements ToXContentObject { + final DocWriteResponse.Result result; + + public ConnectorUpdateActionResponse(StreamInput in) throws IOException { + super(in); + result = DocWriteResponse.Result.readFrom(in); + } + + public ConnectorUpdateActionResponse(DocWriteResponse.Result result) { + this.result = result; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + this.result.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("result", this.result.getLowercase()); + builder.endObject(); + return builder; + } + + public RestStatus status() { + return switch (result) { + case NOT_FOUND -> RestStatus.NOT_FOUND; + default -> RestStatus.OK; + }; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConnectorUpdateActionResponse that = (ConnectorUpdateActionResponse) o; + return Objects.equals(result, that.result); + } + + @Override + public int hashCode() { + return Objects.hash(result); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorConfigurationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorConfigurationAction.java index aa46353d47999..12c96d212f77a 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorConfigurationAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorConfigurationAction.java @@ -39,7 +39,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> client.execute( UpdateConnectorConfigurationAction.INSTANCE, request, - new RestToXContentListener<>(channel, UpdateConnectorConfigurationAction.Response::status, r -> null) + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorErrorAction.java index ea8bd1b4ee50f..8b4b70b994ec1 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorErrorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorErrorAction.java @@ -39,7 +39,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> client.execute( UpdateConnectorErrorAction.INSTANCE, request, - new RestToXContentListener<>(channel, UpdateConnectorErrorAction.Response::status, r -> null) + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorFilteringAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorFilteringAction.java index 63ae3e81fe563..4908e9e09d73f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorFilteringAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorFilteringAction.java @@ -39,7 +39,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> client.execute( UpdateConnectorFilteringAction.INSTANCE, request, - new RestToXContentListener<>(channel, UpdateConnectorFilteringAction.Response::status, r -> null) + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorLastSeenAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorLastSeenAction.java index b2ebaa74984b1..c2c6ee12a7767 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorLastSeenAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorLastSeenAction.java @@ -35,7 +35,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> client.execute( UpdateConnectorLastSeenAction.INSTANCE, request, - new RestToXContentListener<>(channel, UpdateConnectorLastSeenAction.Response::status, r -> null) + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorLastSyncStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorLastSyncStatsAction.java index 8e373ce48caf3..ff3ba53e34a9d 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorLastSyncStatsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorLastSyncStatsAction.java @@ -39,7 +39,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> client.execute( UpdateConnectorLastSyncStatsAction.INSTANCE, request, - new RestToXContentListener<>(channel, UpdateConnectorLastSyncStatsAction.Response::status, r -> null) + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorNameAction.java index 54ce2c9af79e8..c51744e57b1df 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorNameAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorNameAction.java @@ -39,7 +39,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> client.execute( UpdateConnectorNameAction.INSTANCE, request, - new RestToXContentListener<>(channel, UpdateConnectorNameAction.Response::status, r -> null) + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorPipelineAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorPipelineAction.java index ba83bd42dac11..8192099b832dd 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorPipelineAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorPipelineAction.java @@ -39,7 +39,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> client.execute( UpdateConnectorPipelineAction.INSTANCE, request, - new RestToXContentListener<>(channel, UpdateConnectorPipelineAction.Response::status, r -> null) + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorSchedulingAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorSchedulingAction.java index 06a6cb527544e..fda9fa03af913 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorSchedulingAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorSchedulingAction.java @@ -39,7 +39,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> client.execute( UpdateConnectorSchedulingAction.INSTANCE, request, - new RestToXContentListener<>(channel, UpdateConnectorSchedulingAction.Response::status, r -> null) + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorConfigurationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorConfigurationAction.java index 211c3b5a3a670..d4a7e0cf58df2 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorConfigurationAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorConfigurationAction.java @@ -20,7 +20,7 @@ public class TransportUpdateConnectorConfigurationAction extends HandledTransportAction< UpdateConnectorConfigurationAction.Request, - UpdateConnectorConfigurationAction.Response> { + ConnectorUpdateActionResponse> { protected final ConnectorIndexService connectorIndexService; @@ -45,11 +45,8 @@ public TransportUpdateConnectorConfigurationAction( protected void doExecute( Task task, UpdateConnectorConfigurationAction.Request request, - ActionListener listener + ActionListener listener ) { - connectorIndexService.updateConnectorConfiguration( - request, - listener.map(r -> new UpdateConnectorConfigurationAction.Response(r.getResult())) - ); + connectorIndexService.updateConnectorConfiguration(request, listener.map(r -> new ConnectorUpdateActionResponse(r.getResult()))); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorErrorAction.java index 629fd14861cf6..5d9be2ec93f45 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorErrorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorErrorAction.java @@ -20,7 +20,7 @@ public class TransportUpdateConnectorErrorAction extends HandledTransportAction< UpdateConnectorErrorAction.Request, - UpdateConnectorErrorAction.Response> { + ConnectorUpdateActionResponse> { protected final ConnectorIndexService connectorIndexService; @@ -45,8 +45,8 @@ public TransportUpdateConnectorErrorAction( protected void doExecute( Task task, UpdateConnectorErrorAction.Request request, - ActionListener listener + ActionListener listener ) { - connectorIndexService.updateConnectorError(request, listener.map(r -> new UpdateConnectorErrorAction.Response(r.getResult()))); + connectorIndexService.updateConnectorError(request, listener.map(r -> new ConnectorUpdateActionResponse(r.getResult()))); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFilteringAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFilteringAction.java index e871eb4bb79e5..658a8075121af 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFilteringAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFilteringAction.java @@ -20,7 +20,7 @@ public class TransportUpdateConnectorFilteringAction extends HandledTransportAction< UpdateConnectorFilteringAction.Request, - UpdateConnectorFilteringAction.Response> { + ConnectorUpdateActionResponse> { protected final ConnectorIndexService connectorIndexService; @@ -45,11 +45,8 @@ public TransportUpdateConnectorFilteringAction( protected void doExecute( Task task, UpdateConnectorFilteringAction.Request request, - ActionListener listener + ActionListener listener ) { - connectorIndexService.updateConnectorFiltering( - request, - listener.map(r -> new UpdateConnectorFilteringAction.Response(r.getResult())) - ); + connectorIndexService.updateConnectorFiltering(request, listener.map(r -> new ConnectorUpdateActionResponse(r.getResult()))); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSeenAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSeenAction.java index 3d3d2c9ee04b7..60c75bce8314a 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSeenAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSeenAction.java @@ -20,7 +20,7 @@ public class TransportUpdateConnectorLastSeenAction extends HandledTransportAction< UpdateConnectorLastSeenAction.Request, - UpdateConnectorLastSeenAction.Response> { + ConnectorUpdateActionResponse> { protected final ConnectorIndexService connectorIndexService; @@ -45,11 +45,8 @@ public TransportUpdateConnectorLastSeenAction( protected void doExecute( Task task, UpdateConnectorLastSeenAction.Request request, - ActionListener listener + ActionListener listener ) { - connectorIndexService.updateConnectorLastSeen( - request, - listener.map(r -> new UpdateConnectorLastSeenAction.Response(r.getResult())) - ); + connectorIndexService.updateConnectorLastSeen(request, listener.map(r -> new ConnectorUpdateActionResponse(r.getResult()))); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSyncStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSyncStatsAction.java index 9ec0105668fbc..ad934b04c772e 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSyncStatsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSyncStatsAction.java @@ -20,7 +20,7 @@ public class TransportUpdateConnectorLastSyncStatsAction extends HandledTransportAction< UpdateConnectorLastSyncStatsAction.Request, - UpdateConnectorLastSyncStatsAction.Response> { + ConnectorUpdateActionResponse> { protected final ConnectorIndexService connectorIndexService; @@ -45,11 +45,8 @@ public TransportUpdateConnectorLastSyncStatsAction( protected void doExecute( Task task, UpdateConnectorLastSyncStatsAction.Request request, - ActionListener listener + ActionListener listener ) { - connectorIndexService.updateConnectorLastSyncStats( - request, - listener.map(r -> new UpdateConnectorLastSyncStatsAction.Response(r.getResult())) - ); + connectorIndexService.updateConnectorLastSyncStats(request, listener.map(r -> new ConnectorUpdateActionResponse(r.getResult()))); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorNameAction.java index 252734aab1c51..db79ed7be2836 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorNameAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorNameAction.java @@ -20,7 +20,7 @@ public class TransportUpdateConnectorNameAction extends HandledTransportAction< UpdateConnectorNameAction.Request, - UpdateConnectorNameAction.Response> { + ConnectorUpdateActionResponse> { protected final ConnectorIndexService connectorIndexService; @@ -42,14 +42,10 @@ public TransportUpdateConnectorNameAction( } @Override - protected void doExecute( - Task task, - UpdateConnectorNameAction.Request request, - ActionListener listener - ) { + protected void doExecute(Task task, UpdateConnectorNameAction.Request request, ActionListener listener) { connectorIndexService.updateConnectorNameOrDescription( request, - listener.map(r -> new UpdateConnectorNameAction.Response(r.getResult())) + listener.map(r -> new ConnectorUpdateActionResponse(r.getResult())) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorPipelineAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorPipelineAction.java index c54d3db1215bc..11862d568c4ff 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorPipelineAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorPipelineAction.java @@ -20,7 +20,7 @@ public class TransportUpdateConnectorPipelineAction extends HandledTransportAction< UpdateConnectorPipelineAction.Request, - UpdateConnectorPipelineAction.Response> { + ConnectorUpdateActionResponse> { protected final ConnectorIndexService connectorIndexService; @@ -45,11 +45,8 @@ public TransportUpdateConnectorPipelineAction( protected void doExecute( Task task, UpdateConnectorPipelineAction.Request request, - ActionListener listener + ActionListener listener ) { - connectorIndexService.updateConnectorPipeline( - request, - listener.map(r -> new UpdateConnectorPipelineAction.Response(r.getResult())) - ); + connectorIndexService.updateConnectorPipeline(request, listener.map(r -> new ConnectorUpdateActionResponse(r.getResult()))); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorSchedulingAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorSchedulingAction.java index 186edb2328f38..bb5e1f858bd94 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorSchedulingAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorSchedulingAction.java @@ -20,7 +20,7 @@ public class TransportUpdateConnectorSchedulingAction extends HandledTransportAction< UpdateConnectorSchedulingAction.Request, - UpdateConnectorSchedulingAction.Response> { + ConnectorUpdateActionResponse> { protected final ConnectorIndexService connectorIndexService; @@ -45,11 +45,8 @@ public TransportUpdateConnectorSchedulingAction( protected void doExecute( Task task, UpdateConnectorSchedulingAction.Request request, - ActionListener listener + ActionListener listener ) { - connectorIndexService.updateConnectorScheduling( - request, - listener.map(r -> new UpdateConnectorSchedulingAction.Response(r.getResult())) - ); + connectorIndexService.updateConnectorScheduling(request, listener.map(r -> new ConnectorUpdateActionResponse(r.getResult()))); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java index 6b5f52f3afda7..19e7628746485 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java @@ -10,15 +10,12 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ToXContentObject; @@ -37,13 +34,13 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; -public class UpdateConnectorConfigurationAction extends ActionType { +public class UpdateConnectorConfigurationAction extends ActionType { public static final UpdateConnectorConfigurationAction INSTANCE = new UpdateConnectorConfigurationAction(); public static final String NAME = "cluster:admin/xpack/connector/update_configuration"; public UpdateConnectorConfigurationAction() { - super(NAME, UpdateConnectorConfigurationAction.Response::new); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { @@ -152,51 +149,4 @@ public int hashCode() { return Objects.hash(connectorId, configuration); } } - - public static class Response extends ActionResponse implements ToXContentObject { - - final DocWriteResponse.Result result; - - public Response(StreamInput in) throws IOException { - super(in); - result = DocWriteResponse.Result.readFrom(in); - } - - public Response(DocWriteResponse.Result result) { - this.result = result; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - this.result.writeTo(out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("result", this.result.getLowercase()); - builder.endObject(); - return builder; - } - - public RestStatus status() { - return switch (result) { - case NOT_FOUND -> RestStatus.NOT_FOUND; - default -> RestStatus.OK; - }; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Response that = (Response) o; - return Objects.equals(result, that.result); - } - - @Override - public int hashCode() { - return Objects.hash(result); - } - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java index c9e48dac08cd5..ad2036ecbaf81 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java @@ -10,16 +10,13 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -34,13 +31,13 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; -public class UpdateConnectorErrorAction extends ActionType { +public class UpdateConnectorErrorAction extends ActionType { public static final UpdateConnectorErrorAction INSTANCE = new UpdateConnectorErrorAction(); public static final String NAME = "cluster:admin/xpack/connector/update_error"; public UpdateConnectorErrorAction() { - super(NAME, UpdateConnectorErrorAction.Response::new); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { @@ -136,51 +133,4 @@ public int hashCode() { return Objects.hash(connectorId, error); } } - - public static class Response extends ActionResponse implements ToXContentObject { - - final DocWriteResponse.Result result; - - public Response(StreamInput in) throws IOException { - super(in); - result = DocWriteResponse.Result.readFrom(in); - } - - public Response(DocWriteResponse.Result result) { - this.result = result; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - this.result.writeTo(out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("result", this.result.getLowercase()); - builder.endObject(); - return builder; - } - - public RestStatus status() { - return switch (result) { - case NOT_FOUND -> RestStatus.NOT_FOUND; - default -> RestStatus.OK; - }; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Response that = (Response) o; - return Objects.equals(result, that.result); - } - - @Override - public int hashCode() { - return Objects.hash(result); - } - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java index 68c644cb9d9db..dabb87f2afc22 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java @@ -10,15 +10,12 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -35,13 +32,13 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; -public class UpdateConnectorFilteringAction extends ActionType { +public class UpdateConnectorFilteringAction extends ActionType { public static final UpdateConnectorFilteringAction INSTANCE = new UpdateConnectorFilteringAction(); public static final String NAME = "cluster:admin/xpack/connector/update_filtering"; public UpdateConnectorFilteringAction() { - super(NAME, UpdateConnectorFilteringAction.Response::new); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { @@ -141,51 +138,4 @@ public int hashCode() { return Objects.hash(connectorId, filtering); } } - - public static class Response extends ActionResponse implements ToXContentObject { - - final DocWriteResponse.Result result; - - public Response(StreamInput in) throws IOException { - super(in); - result = DocWriteResponse.Result.readFrom(in); - } - - public Response(DocWriteResponse.Result result) { - this.result = result; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - this.result.writeTo(out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("result", this.result.getLowercase()); - builder.endObject(); - return builder; - } - - public RestStatus status() { - return switch (result) { - case NOT_FOUND -> RestStatus.NOT_FOUND; - default -> RestStatus.OK; - }; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Response that = (Response) o; - return Objects.equals(result, that.result); - } - - @Override - public int hashCode() { - return Objects.hash(result); - } - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java index 976be76ba84af..bd20513e47033 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java @@ -9,13 +9,10 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.application.connector.Connector; @@ -26,13 +23,13 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; -public class UpdateConnectorLastSeenAction extends ActionType { +public class UpdateConnectorLastSeenAction extends ActionType { public static final UpdateConnectorLastSeenAction INSTANCE = new UpdateConnectorLastSeenAction(); public static final String NAME = "cluster:admin/xpack/connector/update_last_seen"; public UpdateConnectorLastSeenAction() { - super(NAME, UpdateConnectorLastSeenAction.Response::new); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { @@ -97,51 +94,4 @@ public int hashCode() { return Objects.hash(connectorId, lastSeen); } } - - public static class Response extends ActionResponse implements ToXContentObject { - - final DocWriteResponse.Result result; - - public Response(StreamInput in) throws IOException { - super(in); - result = DocWriteResponse.Result.readFrom(in); - } - - public Response(DocWriteResponse.Result result) { - this.result = result; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - this.result.writeTo(out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("result", this.result.getLowercase()); - builder.endObject(); - return builder; - } - - public RestStatus status() { - return switch (result) { - case NOT_FOUND -> RestStatus.NOT_FOUND; - default -> RestStatus.OK; - }; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Response that = (Response) o; - return Objects.equals(result, that.result); - } - - @Override - public int hashCode() { - return Objects.hash(result); - } - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java index 328831cf0b840..7d82c28ca4af1 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java @@ -10,15 +10,12 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ToXContentObject; @@ -36,13 +33,13 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; -public class UpdateConnectorLastSyncStatsAction extends ActionType { +public class UpdateConnectorLastSyncStatsAction extends ActionType { public static final UpdateConnectorLastSyncStatsAction INSTANCE = new UpdateConnectorLastSyncStatsAction(); public static final String NAME = "cluster:admin/xpack/connector/update_last_sync_stats"; public UpdateConnectorLastSyncStatsAction() { - super(NAME, UpdateConnectorLastSyncStatsAction.Response::new); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { @@ -190,51 +187,4 @@ public int hashCode() { return Objects.hash(connectorId, syncInfo); } } - - public static class Response extends ActionResponse implements ToXContentObject { - - final DocWriteResponse.Result result; - - public Response(StreamInput in) throws IOException { - super(in); - result = DocWriteResponse.Result.readFrom(in); - } - - public Response(DocWriteResponse.Result result) { - this.result = result; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - this.result.writeTo(out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("result", this.result.getLowercase()); - builder.endObject(); - return builder; - } - - public RestStatus status() { - return switch (result) { - case NOT_FOUND -> RestStatus.NOT_FOUND; - default -> RestStatus.OK; - }; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Response that = (Response) o; - return Objects.equals(result, that.result); - } - - @Override - public int hashCode() { - return Objects.hash(result); - } - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java index 1db9bbe3aad9d..6b5c580e396ad 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java @@ -10,16 +10,13 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -35,13 +32,13 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; -public class UpdateConnectorNameAction extends ActionType { +public class UpdateConnectorNameAction extends ActionType { public static final UpdateConnectorNameAction INSTANCE = new UpdateConnectorNameAction(); public static final String NAME = "cluster:admin/xpack/connector/update_name"; public UpdateConnectorNameAction() { - super(NAME, UpdateConnectorNameAction.Response::new); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { @@ -159,51 +156,4 @@ public int hashCode() { return Objects.hash(connectorId, name, description); } } - - public static class Response extends ActionResponse implements ToXContentObject { - - final DocWriteResponse.Result result; - - public Response(StreamInput in) throws IOException { - super(in); - result = DocWriteResponse.Result.readFrom(in); - } - - public Response(DocWriteResponse.Result result) { - this.result = result; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - this.result.writeTo(out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("result", this.result.getLowercase()); - builder.endObject(); - return builder; - } - - public RestStatus status() { - return switch (result) { - case NOT_FOUND -> RestStatus.NOT_FOUND; - default -> RestStatus.OK; - }; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Response that = (Response) o; - return Objects.equals(result, that.result); - } - - @Override - public int hashCode() { - return Objects.hash(result); - } - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java index 68babb2d4b517..ba5b0e702bf0e 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java @@ -10,15 +10,12 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -34,13 +31,13 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; -public class UpdateConnectorPipelineAction extends ActionType { +public class UpdateConnectorPipelineAction extends ActionType { public static final UpdateConnectorPipelineAction INSTANCE = new UpdateConnectorPipelineAction(); public static final String NAME = "cluster:admin/xpack/connector/update_pipeline"; public UpdateConnectorPipelineAction() { - super(NAME, UpdateConnectorPipelineAction.Response::new); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { @@ -139,52 +136,4 @@ public int hashCode() { return Objects.hash(connectorId, pipeline); } } - - public static class Response extends ActionResponse implements ToXContentObject { - - final DocWriteResponse.Result result; - - public Response(StreamInput in) throws IOException { - super(in); - result = DocWriteResponse.Result.readFrom(in); - } - - public Response(DocWriteResponse.Result result) { - this.result = result; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - this.result.writeTo(out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("result", this.result.getLowercase()); - builder.endObject(); - return builder; - } - - public RestStatus status() { - return switch (result) { - case NOT_FOUND -> RestStatus.NOT_FOUND; - default -> RestStatus.OK; - }; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Response that = (Response) o; - return Objects.equals(result, that.result); - } - - @Override - public int hashCode() { - return Objects.hash(result); - } - - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java index 9867830c5d211..df76e9a09547a 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java @@ -10,15 +10,12 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -34,13 +31,13 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; -public class UpdateConnectorSchedulingAction extends ActionType { +public class UpdateConnectorSchedulingAction extends ActionType { public static final UpdateConnectorSchedulingAction INSTANCE = new UpdateConnectorSchedulingAction(); public static final String NAME = "cluster:admin/xpack/connector/update_scheduling"; public UpdateConnectorSchedulingAction() { - super(NAME, UpdateConnectorSchedulingAction.Response::new); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { @@ -139,51 +136,4 @@ public int hashCode() { return Objects.hash(connectorId, scheduling); } } - - public static class Response extends ActionResponse implements ToXContentObject { - - final DocWriteResponse.Result result; - - public Response(StreamInput in) throws IOException { - super(in); - result = DocWriteResponse.Result.readFrom(in); - } - - public Response(DocWriteResponse.Result result) { - this.result = result; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - this.result.writeTo(out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("result", this.result.getLowercase()); - builder.endObject(); - return builder; - } - - public RestStatus status() { - return switch (result) { - case NOT_FOUND -> RestStatus.NOT_FOUND; - default -> RestStatus.OK; - }; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Response that = (Response) o; - return Objects.equals(result, that.result); - } - - @Override - public int hashCode() { - return Objects.hash(result); - } - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/configuration/ConfigurationDisplayType.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/configuration/ConfigurationDisplayType.java index d6b3d83d705b9..df8dee04d61b9 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/configuration/ConfigurationDisplayType.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/configuration/ConfigurationDisplayType.java @@ -10,6 +10,7 @@ import java.util.Locale; public enum ConfigurationDisplayType { + TEXT, TEXTBOX, TEXTAREA, NUMERIC, diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java index 7179bbb3a62f2..111828680455c 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -19,6 +18,7 @@ import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import java.io.IOException; import java.util.Objects; @@ -27,13 +27,13 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobConstants.EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE; -public class CancelConnectorSyncJobAction extends ActionType { +public class CancelConnectorSyncJobAction extends ActionType { public static final CancelConnectorSyncJobAction INSTANCE = new CancelConnectorSyncJobAction(); public static final String NAME = "cluster:admin/xpack/connector/sync_job/cancel"; private CancelConnectorSyncJobAction() { - super(NAME, AcknowledgedResponse::readFrom); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java index 3e5e1578cd54d..54ba26ec1533a 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -19,6 +18,7 @@ import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobConstants; import java.io.IOException; @@ -27,13 +27,13 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; -public class CheckInConnectorSyncJobAction extends ActionType { +public class CheckInConnectorSyncJobAction extends ActionType { public static final CheckInConnectorSyncJobAction INSTANCE = new CheckInConnectorSyncJobAction(); public static final String NAME = "cluster:admin/xpack/connector/sync_job/check_in"; private CheckInConnectorSyncJobAction() { - super(NAME, AcknowledgedResponse::readFrom); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestCancelConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestCancelConnectorSyncJobAction.java index 82d679c6f0ad0..7cfce07aca48d 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestCancelConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestCancelConnectorSyncJobAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.application.EnterpriseSearch; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import java.io.IOException; import java.util.List; @@ -42,6 +43,10 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient CancelConnectorSyncJobAction.Request request = new CancelConnectorSyncJobAction.Request( restRequest.param(CONNECTOR_SYNC_JOB_ID_PARAM) ); - return restChannel -> client.execute(CancelConnectorSyncJobAction.INSTANCE, request, new RestToXContentListener<>(restChannel)); + return restChannel -> client.execute( + CancelConnectorSyncJobAction.INSTANCE, + request, + new RestToXContentListener<>(restChannel, ConnectorUpdateActionResponse::status) + ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestCheckInConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestCheckInConnectorSyncJobAction.java index 86f97f4c5fdb4..882227e45169a 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestCheckInConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestCheckInConnectorSyncJobAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.application.EnterpriseSearch; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import java.io.IOException; import java.util.List; @@ -41,6 +42,10 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient restRequest.param(CONNECTOR_SYNC_JOB_ID_PARAM) ); - return restChannel -> client.execute(CheckInConnectorSyncJobAction.INSTANCE, request, new RestToXContentListener<>(restChannel)); + return restChannel -> client.execute( + CheckInConnectorSyncJobAction.INSTANCE, + request, + new RestToXContentListener<>(restChannel, ConnectorUpdateActionResponse::status) + ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestUpdateConnectorSyncJobErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestUpdateConnectorSyncJobErrorAction.java index e19a9675beebb..a05be4a92e6e3 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestUpdateConnectorSyncJobErrorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestUpdateConnectorSyncJobErrorAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.application.EnterpriseSearch; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import java.io.IOException; import java.util.List; @@ -46,7 +47,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return restChannel -> client.execute( UpdateConnectorSyncJobErrorAction.INSTANCE, request, - new RestToXContentListener<>(restChannel) + new RestToXContentListener<>(restChannel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestUpdateConnectorSyncJobIngestionStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestUpdateConnectorSyncJobIngestionStatsAction.java index aedd1605b8bfb..57a362b55ee9b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestUpdateConnectorSyncJobIngestionStatsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestUpdateConnectorSyncJobIngestionStatsAction.java @@ -10,9 +10,9 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.application.EnterpriseSearch; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import java.io.IOException; import java.util.List; @@ -46,7 +46,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> client.execute( UpdateConnectorSyncJobIngestionStatsAction.INSTANCE, request, - new RestToXContentListener<>(channel, r -> RestStatus.OK) + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCancelConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCancelConnectorSyncJobAction.java index ac61dcdf08a61..b0d4628e06202 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCancelConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCancelConnectorSyncJobAction.java @@ -10,18 +10,18 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService; public class TransportCancelConnectorSyncJobAction extends HandledTransportAction< CancelConnectorSyncJobAction.Request, - AcknowledgedResponse> { + ConnectorUpdateActionResponse> { protected ConnectorSyncJobIndexService connectorSyncJobIndexService; @@ -43,7 +43,14 @@ public TransportCancelConnectorSyncJobAction( } @Override - protected void doExecute(Task task, CancelConnectorSyncJobAction.Request request, ActionListener listener) { - connectorSyncJobIndexService.cancelConnectorSyncJob(request.getConnectorSyncJobId(), listener.map(r -> AcknowledgedResponse.TRUE)); + protected void doExecute( + Task task, + CancelConnectorSyncJobAction.Request request, + ActionListener listener + ) { + connectorSyncJobIndexService.cancelConnectorSyncJob( + request.getConnectorSyncJobId(), + listener.map(r -> new ConnectorUpdateActionResponse(r.getResult())) + ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCheckInConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCheckInConnectorSyncJobAction.java index ebaadc80f4c27..9b57b090c3bd7 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCheckInConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCheckInConnectorSyncJobAction.java @@ -10,18 +10,18 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService; public class TransportCheckInConnectorSyncJobAction extends HandledTransportAction< CheckInConnectorSyncJobAction.Request, - AcknowledgedResponse> { + ConnectorUpdateActionResponse> { protected final ConnectorSyncJobIndexService connectorSyncJobIndexService; @@ -43,7 +43,14 @@ public TransportCheckInConnectorSyncJobAction( } @Override - protected void doExecute(Task task, CheckInConnectorSyncJobAction.Request request, ActionListener listener) { - connectorSyncJobIndexService.checkInConnectorSyncJob(request.getConnectorSyncJobId(), listener.map(r -> AcknowledgedResponse.TRUE)); + protected void doExecute( + Task task, + CheckInConnectorSyncJobAction.Request request, + ActionListener listener + ) { + connectorSyncJobIndexService.checkInConnectorSyncJob( + request.getConnectorSyncJobId(), + listener.map(r -> new ConnectorUpdateActionResponse(r.getResult())) + ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobErrorAction.java index c814092f2e7a2..d70d5cdd956fa 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobErrorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobErrorAction.java @@ -10,18 +10,18 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService; public class TransportUpdateConnectorSyncJobErrorAction extends HandledTransportAction< UpdateConnectorSyncJobErrorAction.Request, - AcknowledgedResponse> { + ConnectorUpdateActionResponse> { protected final ConnectorSyncJobIndexService connectorSyncJobIndexService; @@ -43,11 +43,15 @@ public TransportUpdateConnectorSyncJobErrorAction( } @Override - protected void doExecute(Task task, UpdateConnectorSyncJobErrorAction.Request request, ActionListener listener) { + protected void doExecute( + Task task, + UpdateConnectorSyncJobErrorAction.Request request, + ActionListener listener + ) { connectorSyncJobIndexService.updateConnectorSyncJobError( request.getConnectorSyncJobId(), request.getError(), - listener.map(r -> AcknowledgedResponse.TRUE) + listener.map(r -> new ConnectorUpdateActionResponse(r.getResult())) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobIngestionStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobIngestionStatsAction.java index 864da6ca3095b..49c722b84e63b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobIngestionStatsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobIngestionStatsAction.java @@ -10,18 +10,18 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService; public class TransportUpdateConnectorSyncJobIngestionStatsAction extends HandledTransportAction< UpdateConnectorSyncJobIngestionStatsAction.Request, - AcknowledgedResponse> { + ConnectorUpdateActionResponse> { protected final ConnectorSyncJobIndexService connectorSyncJobIndexService; @@ -46,8 +46,11 @@ public TransportUpdateConnectorSyncJobIngestionStatsAction( protected void doExecute( Task task, UpdateConnectorSyncJobIngestionStatsAction.Request request, - ActionListener listener + ActionListener listener ) { - connectorSyncJobIndexService.updateConnectorSyncJobIngestionStats(request, listener.map(r -> AcknowledgedResponse.TRUE)); + connectorSyncJobIndexService.updateConnectorSyncJobIngestionStats( + request, + listener.map(r -> new ConnectorUpdateActionResponse(r.getResult())) + ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java index 820630bccee03..fe0893c82e27d 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; @@ -23,6 +22,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobConstants; @@ -32,14 +32,14 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; -public class UpdateConnectorSyncJobErrorAction extends ActionType { +public class UpdateConnectorSyncJobErrorAction extends ActionType { public static final UpdateConnectorSyncJobErrorAction INSTANCE = new UpdateConnectorSyncJobErrorAction(); public static final String NAME = "cluster:admin/xpack/connector/sync_job/update_error"; public static final String ERROR_EMPTY_MESSAGE = "[error] of the connector sync job cannot be null or empty"; private UpdateConnectorSyncJobErrorAction() { - super(NAME, AcknowledgedResponse::readFrom); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java index 34d8be2af4881..b9c57cb6a0c61 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; @@ -25,6 +24,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob; import java.io.IOException; @@ -36,13 +36,13 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; import static org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobConstants.EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE; -public class UpdateConnectorSyncJobIngestionStatsAction extends ActionType { +public class UpdateConnectorSyncJobIngestionStatsAction extends ActionType { public static final UpdateConnectorSyncJobIngestionStatsAction INSTANCE = new UpdateConnectorSyncJobIngestionStatsAction(); public static final String NAME = "cluster:admin/xpack/connector/sync_job/update_stats"; public UpdateConnectorSyncJobIngestionStatsAction() { - super(NAME, AcknowledgedResponse::readFrom); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java index 9401a2a58403e..cdfa3dea8a6fa 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java @@ -98,6 +98,31 @@ public void testToXContent() throws IOException { } ], "value":"" + }, + "field_with_null_tooltip":{ + "default_value":null, + "depends_on":[ + { + "field":"some_field", + "value":true + } + ], + "display":"textbox", + "label":"Very important field", + "options":[], + "order":4, + "required":true, + "sensitive":false, + "tooltip":null, + "type":"str", + "ui_restrictions":[], + "validations":[ + { + "constraint":0, + "type":"greater_than" + } + ], + "value":"" } }, "description":"test-connector", diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ConnectorUpdateActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ConnectorUpdateActionResponseBWCSerializingTests.java new file mode 100644 index 0000000000000..d70aa892788ce --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ConnectorUpdateActionResponseBWCSerializingTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; + +import java.io.IOException; + +public class ConnectorUpdateActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return ConnectorUpdateActionResponse::new; + } + + @Override + protected ConnectorUpdateActionResponse createTestInstance() { + return new ConnectorUpdateActionResponse(randomFrom(DocWriteResponse.Result.values())); + } + + @Override + protected ConnectorUpdateActionResponse mutateInstance(ConnectorUpdateActionResponse instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected ConnectorUpdateActionResponse mutateInstanceForVersion(ConnectorUpdateActionResponse instance, TransportVersion version) { + return instance; + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationActionResponseBWCSerializingTests.java deleted file mode 100644 index d4aa4f12b36d3..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationActionResponseBWCSerializingTests.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; - -import java.io.IOException; - -public class UpdateConnectorConfigurationActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< - UpdateConnectorConfigurationAction.Response> { - - @Override - protected Writeable.Reader instanceReader() { - return UpdateConnectorConfigurationAction.Response::new; - } - - @Override - protected UpdateConnectorConfigurationAction.Response createTestInstance() { - return new UpdateConnectorConfigurationAction.Response(randomFrom(DocWriteResponse.Result.values())); - } - - @Override - protected UpdateConnectorConfigurationAction.Response mutateInstance(UpdateConnectorConfigurationAction.Response instance) - throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected UpdateConnectorConfigurationAction.Response mutateInstanceForVersion( - UpdateConnectorConfigurationAction.Response instance, - TransportVersion version - ) { - return instance; - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorActionResponseBWCSerializingTests.java deleted file mode 100644 index a39fcac3d2f04..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorActionResponseBWCSerializingTests.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; - -import java.io.IOException; - -public class UpdateConnectorErrorActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< - UpdateConnectorErrorAction.Response> { - - @Override - protected Writeable.Reader instanceReader() { - return UpdateConnectorErrorAction.Response::new; - } - - @Override - protected UpdateConnectorErrorAction.Response createTestInstance() { - return new UpdateConnectorErrorAction.Response(randomFrom(DocWriteResponse.Result.values())); - } - - @Override - protected UpdateConnectorErrorAction.Response mutateInstance(UpdateConnectorErrorAction.Response instance) throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected UpdateConnectorErrorAction.Response mutateInstanceForVersion( - UpdateConnectorErrorAction.Response instance, - TransportVersion version - ) { - return instance; - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringActionResponseBWCSerializingTests.java deleted file mode 100644 index 0f33eeac8dfb5..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringActionResponseBWCSerializingTests.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; - -import java.io.IOException; - -public class UpdateConnectorFilteringActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< - UpdateConnectorFilteringAction.Response> { - - @Override - protected Writeable.Reader instanceReader() { - return UpdateConnectorFilteringAction.Response::new; - } - - @Override - protected UpdateConnectorFilteringAction.Response createTestInstance() { - return new UpdateConnectorFilteringAction.Response(randomFrom(DocWriteResponse.Result.values())); - } - - @Override - protected UpdateConnectorFilteringAction.Response mutateInstance(UpdateConnectorFilteringAction.Response instance) throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected UpdateConnectorFilteringAction.Response mutateInstanceForVersion( - UpdateConnectorFilteringAction.Response instance, - TransportVersion version - ) { - return instance; - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenActionResponseBWCSerializingTests.java deleted file mode 100644 index d992f1b5f188e..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenActionResponseBWCSerializingTests.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; - -import java.io.IOException; - -public class UpdateConnectorLastSeenActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< - UpdateConnectorLastSeenAction.Response> { - - @Override - protected Writeable.Reader instanceReader() { - return UpdateConnectorLastSeenAction.Response::new; - } - - @Override - protected UpdateConnectorLastSeenAction.Response createTestInstance() { - return new UpdateConnectorLastSeenAction.Response(randomFrom(DocWriteResponse.Result.values())); - } - - @Override - protected UpdateConnectorLastSeenAction.Response mutateInstance(UpdateConnectorLastSeenAction.Response instance) throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected UpdateConnectorLastSeenAction.Response mutateInstanceForVersion( - UpdateConnectorLastSeenAction.Response instance, - TransportVersion version - ) { - return instance; - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsActionResponseBWCSerializingTests.java deleted file mode 100644 index dd214e10699ef..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsActionResponseBWCSerializingTests.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; - -import java.io.IOException; - -public class UpdateConnectorLastSyncStatsActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< - UpdateConnectorLastSyncStatsAction.Response> { - - @Override - protected Writeable.Reader instanceReader() { - return UpdateConnectorLastSyncStatsAction.Response::new; - } - - @Override - protected UpdateConnectorLastSyncStatsAction.Response createTestInstance() { - return new UpdateConnectorLastSyncStatsAction.Response(randomFrom(DocWriteResponse.Result.values())); - } - - @Override - protected UpdateConnectorLastSyncStatsAction.Response mutateInstance(UpdateConnectorLastSyncStatsAction.Response instance) - throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected UpdateConnectorLastSyncStatsAction.Response mutateInstanceForVersion( - UpdateConnectorLastSyncStatsAction.Response instance, - TransportVersion version - ) { - return instance; - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameActionResponseBWCSerializingTests.java deleted file mode 100644 index 2297ccb565b5e..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameActionResponseBWCSerializingTests.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; - -import java.io.IOException; - -public class UpdateConnectorNameActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< - UpdateConnectorNameAction.Response> { - - @Override - protected Writeable.Reader instanceReader() { - return UpdateConnectorNameAction.Response::new; - } - - @Override - protected UpdateConnectorNameAction.Response createTestInstance() { - return new UpdateConnectorNameAction.Response(randomFrom(DocWriteResponse.Result.values())); - } - - @Override - protected UpdateConnectorNameAction.Response mutateInstance(UpdateConnectorNameAction.Response instance) throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected UpdateConnectorNameAction.Response mutateInstanceForVersion( - UpdateConnectorNameAction.Response instance, - TransportVersion version - ) { - return instance; - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineActionResponseBWCSerializingTests.java deleted file mode 100644 index 065dafcaf00a4..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineActionResponseBWCSerializingTests.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; - -import java.io.IOException; - -public class UpdateConnectorPipelineActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< - UpdateConnectorPipelineAction.Response> { - @Override - protected Writeable.Reader instanceReader() { - return UpdateConnectorPipelineAction.Response::new; - } - - @Override - protected UpdateConnectorPipelineAction.Response createTestInstance() { - return new UpdateConnectorPipelineAction.Response(randomFrom(DocWriteResponse.Result.values())); - } - - @Override - protected UpdateConnectorPipelineAction.Response mutateInstance(UpdateConnectorPipelineAction.Response instance) throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected UpdateConnectorPipelineAction.Response mutateInstanceForVersion( - UpdateConnectorPipelineAction.Response instance, - TransportVersion version - ) { - return instance; - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingActionResponseBWCSerializingTests.java deleted file mode 100644 index a03713fa61a36..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingActionResponseBWCSerializingTests.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; - -import java.io.IOException; - -public class UpdateConnectorSchedulingActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< - UpdateConnectorSchedulingAction.Response> { - - @Override - protected Writeable.Reader instanceReader() { - return UpdateConnectorSchedulingAction.Response::new; - } - - @Override - protected UpdateConnectorSchedulingAction.Response createTestInstance() { - return new UpdateConnectorSchedulingAction.Response(randomFrom(DocWriteResponse.Result.values())); - } - - @Override - protected UpdateConnectorSchedulingAction.Response mutateInstance(UpdateConnectorSchedulingAction.Response instance) - throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected UpdateConnectorSchedulingAction.Response mutateInstanceForVersion( - UpdateConnectorSchedulingAction.Response instance, - TransportVersion version - ) { - return instance; - } -} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClient.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClient.java index 707964a93ab9e..4e4817d4c041d 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClient.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClient.java @@ -46,10 +46,12 @@ public class PITAwareQueryClient extends BasicQueryClient { private String pitId; private final TimeValue keepAlive; + private final QueryBuilder filter; public PITAwareQueryClient(EqlSession eqlSession) { super(eqlSession); this.keepAlive = eqlSession.configuration().requestTimeout(); + this.filter = eqlSession.configuration().filter(); } @Override @@ -131,6 +133,7 @@ private ActionListener pitListener(Function void openPIT(ActionListener listener, Runnable runnable) { OpenPointInTimeRequest request = new OpenPointInTimeRequest(indices).indicesOptions(IndexResolver.FIELD_CAPS_INDICES_OPTIONS) .keepAlive(keepAlive); + request.indexFilter(filter); client.execute(TransportOpenPointInTimeAction.TYPE, request, listener.delegateFailureAndWrap((l, r) -> { pitId = r.getPointInTimeId(); runnable.run(); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java new file mode 100644 index 0000000000000..e12eec4833199 --- /dev/null +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java @@ -0,0 +1,285 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.eql.execution.search; + +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.search.ClosePointInTimeRequest; +import org.elasticsearch.action.search.ClosePointInTimeResponse; +import org.elasticsearch.action.search.OpenPointInTimeRequest; +import org.elasticsearch.action.search.OpenPointInTimeResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchResponseSections; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.breaker.NoopCircuitBreaker; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchSortValues; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.async.AsyncExecutionId; +import org.elasticsearch.xpack.eql.action.EqlSearchAction; +import org.elasticsearch.xpack.eql.action.EqlSearchTask; +import org.elasticsearch.xpack.eql.analysis.PostAnalyzer; +import org.elasticsearch.xpack.eql.analysis.PreAnalyzer; +import org.elasticsearch.xpack.eql.analysis.Verifier; +import org.elasticsearch.xpack.eql.execution.assembler.BoxedQueryRequest; +import org.elasticsearch.xpack.eql.execution.assembler.SequenceCriterion; +import org.elasticsearch.xpack.eql.execution.search.extractor.ImplicitTiebreakerHitExtractor; +import org.elasticsearch.xpack.eql.execution.sequence.SequenceMatcher; +import org.elasticsearch.xpack.eql.execution.sequence.TumblingWindow; +import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry; +import org.elasticsearch.xpack.eql.optimizer.Optimizer; +import org.elasticsearch.xpack.eql.planner.Planner; +import org.elasticsearch.xpack.eql.session.EqlConfiguration; +import org.elasticsearch.xpack.eql.session.EqlSession; +import org.elasticsearch.xpack.eql.stats.Metrics; +import org.elasticsearch.xpack.ql.execution.search.extractor.HitExtractor; +import org.elasticsearch.xpack.ql.index.IndexResolver; +import org.elasticsearch.xpack.ql.type.DefaultDataTypeRegistry; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static org.elasticsearch.action.ActionListener.wrap; +import static org.elasticsearch.index.query.QueryBuilders.idsQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.index.query.QueryBuilders.termsQuery; +import static org.elasticsearch.xpack.eql.EqlTestUtils.booleanArrayOf; + +public class PITAwareQueryClientTests extends ESTestCase { + + private final List keyExtractors = emptyList(); + private static final QueryBuilder[] FILTERS = new QueryBuilder[] { + rangeQuery("some_timestamp_field").gte("2023-12-07"), + termQuery("tier", "hot"), + idsQuery().addIds("1", "2", "3") }; + private static final String[] INDICES = new String[] { "test1", "test2", "test3" }; + + public void testQueryFilterUsedInPitAndSearches() { + try (var threadPool = createThreadPool()) { + final var filter = frequently() ? randomFrom(FILTERS) : null; + int stages = randomIntBetween(2, 5); + final var esClient = new ESMockClient(threadPool, filter, stages); + + EqlConfiguration eqlConfiguration = new EqlConfiguration( + INDICES, + org.elasticsearch.xpack.ql.util.DateUtils.UTC, + "nobody", + "cluster", + filter, + emptyMap(), + null, + TimeValue.timeValueSeconds(30), + null, + 123, + 1, + "", + new TaskId("test", 123), + new EqlSearchTask( + randomLong(), + "transport", + EqlSearchAction.NAME, + "", + null, + emptyMap(), + emptyMap(), + new AsyncExecutionId("", new TaskId(randomAlphaOfLength(10), 1)), + TimeValue.timeValueDays(5) + ) + ); + IndexResolver indexResolver = new IndexResolver(esClient, "cluster", DefaultDataTypeRegistry.INSTANCE, () -> emptySet()); + CircuitBreaker cb = new NoopCircuitBreaker("testcb"); + EqlSession eqlSession = new EqlSession( + esClient, + eqlConfiguration, + indexResolver, + new PreAnalyzer(), + new PostAnalyzer(), + new EqlFunctionRegistry(), + new Verifier(new Metrics()), + new Optimizer(), + new Planner(), + cb + ); + QueryClient eqlClient = new PITAwareQueryClient(eqlSession) { + @Override + public void fetchHits(Iterable> refs, ActionListener>> listener) { + List> searchHits = new ArrayList<>(); + for (List ref : refs) { + List hits = new ArrayList<>(ref.size()); + for (HitReference hitRef : ref) { + hits.add(new SearchHit(-1, hitRef.id())); + } + searchHits.add(hits); + } + listener.onResponse(searchHits); + } + }; + + List criteria = new ArrayList<>(stages); + for (int i = 0; i < stages; i++) { + final int j = i; + criteria.add( + new SequenceCriterion( + i, + new BoxedQueryRequest( + () -> SearchSourceBuilder.searchSource().size(10).query(matchAllQuery()).terminateAfter(j), + "@timestamp", + emptyList(), + emptySet() + ), + keyExtractors, + TimestampExtractor.INSTANCE, + null, + ImplicitTiebreakerHitExtractor.INSTANCE, + false, + false + ) + ); + } + + SequenceMatcher matcher = new SequenceMatcher(stages, false, TimeValue.MINUS_ONE, null, booleanArrayOf(stages, false), cb); + TumblingWindow window = new TumblingWindow(eqlClient, criteria, null, matcher, Collections.emptyList()); + window.execute(wrap(response -> { + // do nothing, we don't care about the query results + }, ex -> { fail("Shouldn't have failed"); })); + } + } + + /** + * This class is used by {@code PITFailureTests.testPitCloseOnFailure} method + * to test that PIT close is never (wrongly) invoked if PIT open failed. + */ + private class ESMockClient extends NoOpClient { + private final QueryBuilder filter; + private final String pitId = "test_pit_id"; + private boolean openedPIT = false; + private int searchRequestsRemainingCount; + + ESMockClient(ThreadPool threadPool, QueryBuilder filter, int stages) { + super(threadPool); + this.filter = filter; + this.searchRequestsRemainingCount = stages; + } + + @SuppressWarnings("unchecked") + @Override + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + if (request instanceof OpenPointInTimeRequest openPIT) { + assertFalse(openedPIT); + assertEquals(filter, openPIT.indexFilter()); // check that the filter passed on to the eql query is used in opening the pit + assertArrayEquals(INDICES, openPIT.indices()); // indices for opening pit should be the same as for the eql query itself + + openedPIT = true; + OpenPointInTimeResponse response = new OpenPointInTimeResponse(pitId); + listener.onResponse((Response) response); + } else if (request instanceof ClosePointInTimeRequest closePIT) { + assertTrue(openedPIT); + assertEquals(pitId, closePIT.getId()); + + openedPIT = false; + ClosePointInTimeResponse response = new ClosePointInTimeResponse(true, 1); + listener.onResponse((Response) response); + } else if (request instanceof SearchRequest searchRequest) { + assertTrue(openedPIT); + searchRequestsRemainingCount--; + assertTrue(searchRequestsRemainingCount >= 0); + + assertEquals(pitId, searchRequest.source().pointInTimeBuilder().getEncodedId()); + assertEquals(0, searchRequest.indices().length); // no indices set in the search request + assertEquals(1, searchRequest.source().subSearches().size()); + + BoolQueryBuilder actualQuery = (BoolQueryBuilder) searchRequest.source().subSearches().get(0).getQueryBuilder(); + assertEquals(3, actualQuery.filter().size()); + assertTrue(actualQuery.filter().get(0) instanceof MatchAllQueryBuilder); // the match_all we used when building the criteria + assertTrue(actualQuery.filter().get(1) instanceof RangeQueryBuilder); + QueryBuilder expectedQuery = termsQuery("_index", INDICES); // indices should be used as a filter further on + assertEquals(expectedQuery, actualQuery.filter().get(2)); + + handleSearchRequest(listener, searchRequest); + } else { + super.doExecute(action, request, listener); + } + } + + @SuppressWarnings("unchecked") + void handleSearchRequest(ActionListener listener, SearchRequest searchRequest) { + int ordinal = searchRequest.source().terminateAfter(); + SearchHit searchHit = new SearchHit(ordinal, String.valueOf(ordinal)); + searchHit.sortValues( + new SearchSortValues(new Long[] { (long) ordinal, 1L }, new DocValueFormat[] { DocValueFormat.RAW, DocValueFormat.RAW }) + ); + + SearchHits searchHits = new SearchHits(new SearchHit[] { searchHit }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 0.0f); + SearchResponseSections internal = new SearchResponseSections(searchHits, null, null, false, false, null, 0); + SearchResponse response = new SearchResponse( + internal, + null, + 2, + 0, + 0, + 0, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY, + searchRequest.pointInTimeBuilder().getEncodedId() + ); + + ActionListener.respondAndRelease(listener, (Response) response); + } + } + + private static class TimestampExtractor implements HitExtractor { + + static final TimestampExtractor INSTANCE = new TimestampExtractor(); + + @Override + public String getWriteableName() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException {} + + @Override + public String hitName() { + return null; + } + + @Override + public Timestamp extract(SearchHit hit) { + return Timestamp.of(String.valueOf(hit.docId())); + } + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index f6c0666c54ed8..591d395661afa 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -139,9 +139,9 @@ emp_no:integer | birth_date:date | x:date ; evalDateTruncGrouping -from employees | eval y = date_trunc(1 year, hire_date) | stats count(emp_no) by y | sort y | keep y, count(emp_no) | limit 5; +from employees | eval y = date_trunc(1 year, hire_date) | stats c = count(emp_no) by y | sort y | keep y, c | limit 5; -y:date | count(emp_no):long +y:date | c:long 1985-01-01T00:00:00.000Z | 11 1986-01-01T00:00:00.000Z | 11 1987-01-01T00:00:00.000Z | 15 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec index a3bc9c6c6dcf6..3a1ae3985e129 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec @@ -316,3 +316,11 @@ a:keyword | e:keyword a | a ; + +//see https://github.com/elastic/elasticsearch/issues/103331 +keepStarMvExpand#[skip:-8.12.99] +from employees | where emp_no == 10001 | keep * | mv_expand first_name; + +avg_worked_seconds:long | birth_date:date | emp_no:integer | first_name:keyword | gender:keyword | height:double | height.float:double | height.half_float:double | height.scaled_float:double | hire_date:date | is_rehired:boolean | job_positions:keyword | languages:integer | languages.byte:integer | languages.long:long | languages.short:integer | last_name:keyword | salary:integer | salary_change:double | salary_change.int:integer | salary_change.keyword:keyword | salary_change.long:long | still_hired:boolean +268728049 | 1953-09-02T00:00:00.000Z | 10001 | Georgi | M | 2.03 | 2.0299999713897705 | 2.029296875 | 2.0300000000000002 | 1986-06-26T00:00:00.000Z | [false, true] | [Accountant, Senior Python Developer] | 2 | 2 | 2 | 2 | Facello | 57305 | 1.19 | 1 | 1.19 | 1 | true +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index dc96d1736858c..0ad759feeeea0 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -782,3 +782,12 @@ FROM sample_data median_duration:double | client_ip:ip ; + +fieldEscaping#[skip:-8.12.99, reason:Fixed bug in 8.13 of removing the leading/trailing backquotes of an identifier] +FROM sample_data +| stats count(`event_duration`) | keep `count(``event_duration``)` +; + +count(`event_duration`):l +7 +; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeBasedIndicesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeBasedIndicesIT.java new file mode 100644 index 0000000000000..a1fbee17ef8ec --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeBasedIndicesIT.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.action; + +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.RangeQueryBuilder; + +import java.util.List; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList; +import static org.hamcrest.Matchers.hasSize; + +public class TimeBasedIndicesIT extends AbstractEsqlIntegTestCase { + + public void testFilter() { + long epoch = System.currentTimeMillis(); + assertAcked(client().admin().indices().prepareCreate("test").setMapping("@timestamp", "type=date", "value", "type=long")); + BulkRequestBuilder bulk = client().prepareBulk("test").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + int oldDocs = between(10, 100); + for (int i = 0; i < oldDocs; i++) { + long timestamp = epoch - TimeValue.timeValueHours(between(1, 2)).millis(); + bulk.add(new IndexRequest().source("@timestamp", timestamp, "value", -i)); + } + int newDocs = between(10, 100); + for (int i = 0; i < newDocs; i++) { + long timestamp = epoch + TimeValue.timeValueHours(between(1, 2)).millis(); + bulk.add(new IndexRequest().source("@timestamp", timestamp, "value", i)); + } + bulk.get(); + { + EsqlQueryRequest request = new EsqlQueryRequest(); + request.query("FROM test | limit 1000"); + request.filter(new RangeQueryBuilder("@timestamp").from(epoch - TimeValue.timeValueHours(3).millis()).to("now")); + try (var resp = run(request)) { + List> values = getValuesList(resp); + assertThat(values, hasSize(oldDocs)); + } + } + { + EsqlQueryRequest request = new EsqlQueryRequest(); + request.query("FROM test | limit 1000"); + request.filter(new RangeQueryBuilder("@timestamp").from("now").to(epoch + TimeValue.timeValueHours(3).millis())); + try (var resp = run(request)) { + List> values = getValuesList(resp); + assertThat(values, hasSize(newDocs)); + } + } + } +} diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 index 747c1fdcd1921..dbaefa2e5aebf 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 @@ -1,24 +1,24 @@ lexer grammar EsqlBaseLexer; -DISSECT : 'dissect' -> pushMode(EXPRESSION); -DROP : 'drop' -> pushMode(SOURCE_IDENTIFIERS); -ENRICH : 'enrich' -> pushMode(SOURCE_IDENTIFIERS); -EVAL : 'eval' -> pushMode(EXPRESSION); -EXPLAIN : 'explain' -> pushMode(EXPLAIN_MODE); -FROM : 'from' -> pushMode(SOURCE_IDENTIFIERS); -GROK : 'grok' -> pushMode(EXPRESSION); -INLINESTATS : 'inlinestats' -> pushMode(EXPRESSION); -KEEP : 'keep' -> pushMode(SOURCE_IDENTIFIERS); -LIMIT : 'limit' -> pushMode(EXPRESSION); -MV_EXPAND : 'mv_expand' -> pushMode(SOURCE_IDENTIFIERS); -PROJECT : 'project' -> pushMode(SOURCE_IDENTIFIERS); -RENAME : 'rename' -> pushMode(SOURCE_IDENTIFIERS); -ROW : 'row' -> pushMode(EXPRESSION); -SHOW : 'show' -> pushMode(EXPRESSION); -SORT : 'sort' -> pushMode(EXPRESSION); -STATS : 'stats' -> pushMode(EXPRESSION); -WHERE : 'where' -> pushMode(EXPRESSION); -UNKNOWN_CMD : ~[ \r\n\t[\]/]+ -> pushMode(EXPRESSION); +DISSECT : 'dissect' -> pushMode(EXPRESSION_MODE); +DROP : 'drop' -> pushMode(PROJECT_MODE); +ENRICH : 'enrich' -> pushMode(ENRICH_MODE); +EVAL : 'eval' -> pushMode(EXPRESSION_MODE); +EXPLAIN : 'explain' -> pushMode(EXPLAIN_MODE); +FROM : 'from' -> pushMode(FROM_MODE); +GROK : 'grok' -> pushMode(EXPRESSION_MODE); +INLINESTATS : 'inlinestats' -> pushMode(EXPRESSION_MODE); +KEEP : 'keep' -> pushMode(PROJECT_MODE); +LIMIT : 'limit' -> pushMode(EXPRESSION_MODE); +MV_EXPAND : 'mv_expand' -> pushMode(MVEXPAND_MODE); +PROJECT : 'project' -> pushMode(PROJECT_MODE); +RENAME : 'rename' -> pushMode(RENAME_MODE); +ROW : 'row' -> pushMode(EXPRESSION_MODE); +SHOW : 'show' -> pushMode(SHOW_MODE); +SORT : 'sort' -> pushMode(EXPRESSION_MODE); +STATS : 'stats' -> pushMode(EXPRESSION_MODE); +WHERE : 'where' -> pushMode(EXPRESSION_MODE); +UNKNOWN_CMD : ~[ \r\n\t[\]/]+ -> pushMode(EXPRESSION_MODE); LINE_COMMENT : '//' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN) @@ -31,16 +31,20 @@ MULTILINE_COMMENT WS : [ \r\n\t]+ -> channel(HIDDEN) ; - - +// +// Explain +// mode EXPLAIN_MODE; -EXPLAIN_OPENING_BRACKET : '[' -> type(OPENING_BRACKET), pushMode(DEFAULT_MODE); -EXPLAIN_PIPE : '|' -> type(PIPE), popMode; +EXPLAIN_OPENING_BRACKET : OPENING_BRACKET -> type(OPENING_BRACKET), pushMode(DEFAULT_MODE); +EXPLAIN_PIPE : PIPE -> type(PIPE), popMode; EXPLAIN_WS : WS -> channel(HIDDEN); EXPLAIN_LINE_COMMENT : LINE_COMMENT -> channel(HIDDEN); EXPLAIN_MULTILINE_COMMENT : MULTILINE_COMMENT -> channel(HIDDEN); -mode EXPRESSION; +// +// Expression - used by most command +// +mode EXPRESSION_MODE; PIPE : '|' -> popMode; @@ -64,6 +68,27 @@ fragment EXPONENT : [Ee] [+-]? DIGIT+ ; +fragment ASPERAND + : '@' + ; + +fragment BACKQUOTE + : '`' + ; + +fragment BACKQUOTE_BLOCK + : ~'`' + | '``' + ; + +fragment UNDERSCORE + : '_' + ; + +fragment UNQUOTED_ID_BODY + : (LETTER | DIGIT | UNDERSCORE) + ; + STRING : '"' (ESCAPE_SEQUENCE | UNESCAPED_CHARS)* '"' | '"""' (~[\r\n])*? '"""' '"'? '"'? @@ -103,8 +128,6 @@ PARAM: '?'; RLIKE: 'rlike'; RP : ')'; TRUE : 'true'; -INFO : 'info'; -FUNCTIONS : 'functions'; EQ : '=='; NEQ : '!='; @@ -124,19 +147,18 @@ PERCENT : '%'; // mode. Thus, the two popModes on CLOSING_BRACKET. The other way could as // the start of a multivalued field constant. To line up with the double pop // the explain mode needs, we double push when we see that. -OPENING_BRACKET : '[' -> pushMode(EXPRESSION), pushMode(EXPRESSION); +OPENING_BRACKET : '[' -> pushMode(EXPRESSION_MODE), pushMode(EXPRESSION_MODE); CLOSING_BRACKET : ']' -> popMode, popMode; - UNQUOTED_IDENTIFIER - : LETTER (LETTER | DIGIT | '_')* + : LETTER UNQUOTED_ID_BODY* // only allow @ at beginning of identifier to keep the option to allow @ as infix operator in the future // also, single `_` and `@` characters are not valid identifiers - | ('_' | '@') (LETTER | DIGIT | '_')+ + | (UNDERSCORE | ASPERAND) UNQUOTED_ID_BODY+ ; QUOTED_IDENTIFIER - : '`' ( ~'`' | '``' )* '`' + : BACKQUOTE BACKQUOTE_BLOCK+ BACKQUOTE ; EXPR_LINE_COMMENT @@ -150,42 +172,204 @@ EXPR_MULTILINE_COMMENT EXPR_WS : WS -> channel(HIDDEN) ; +// +// FROM command +// +mode FROM_MODE; +FROM_PIPE : PIPE -> type(PIPE), popMode; +FROM_OPENING_BRACKET : OPENING_BRACKET -> type(OPENING_BRACKET), pushMode(FROM_MODE), pushMode(FROM_MODE); +FROM_CLOSING_BRACKET : CLOSING_BRACKET -> type(CLOSING_BRACKET), popMode, popMode; +FROM_COMMA : COMMA -> type(COMMA); +FROM_ASSIGN : ASSIGN -> type(ASSIGN); +METADATA: 'metadata'; +fragment FROM_UNQUOTED_IDENTIFIER_PART + : ~[=`|,[\]/ \t\r\n] + | '/' ~[*/] // allow single / but not followed by another / or * which would start a comment + ; -mode SOURCE_IDENTIFIERS; +FROM_UNQUOTED_IDENTIFIER + : FROM_UNQUOTED_IDENTIFIER_PART+ + ; + +FROM_QUOTED_IDENTIFIER + : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER) + ; + +FROM_LINE_COMMENT + : LINE_COMMENT -> channel(HIDDEN) + ; + +FROM_MULTILINE_COMMENT + : MULTILINE_COMMENT -> channel(HIDDEN) + ; + +FROM_WS + : WS -> channel(HIDDEN) + ; +// +// DROP, KEEP, PROJECT +// +mode PROJECT_MODE; +PROJECT_PIPE : PIPE -> type(PIPE), popMode; +PROJECT_DOT: DOT -> type(DOT); +PROJECT_COMMA : COMMA -> type(COMMA); + +fragment UNQUOTED_ID_BODY_WITH_PATTERN + : (LETTER | DIGIT | UNDERSCORE | ASTERISK) + ; + +PROJECT_UNQUOTED_IDENTIFIER + : (LETTER | ASTERISK) UNQUOTED_ID_BODY_WITH_PATTERN* + | (UNDERSCORE | ASPERAND) UNQUOTED_ID_BODY_WITH_PATTERN+ + ; + +PROJECT_QUOTED_IDENTIFIER + : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER) + ; + +PROJECT_LINE_COMMENT + : LINE_COMMENT -> channel(HIDDEN) + ; + +PROJECT_MULTILINE_COMMENT + : MULTILINE_COMMENT -> channel(HIDDEN) + ; + +PROJECT_WS + : WS -> channel(HIDDEN) + ; +// +// | RENAME a.b AS x, c AS y +// +mode RENAME_MODE; +RENAME_PIPE : PIPE -> type(PIPE), popMode; +RENAME_ASSIGN : ASSIGN -> type(ASSIGN); +RENAME_COMMA : COMMA -> type(COMMA); +RENAME_DOT: DOT -> type(DOT); -SRC_PIPE : '|' -> type(PIPE), popMode; -SRC_OPENING_BRACKET : '[' -> type(OPENING_BRACKET), pushMode(SOURCE_IDENTIFIERS), pushMode(SOURCE_IDENTIFIERS); -SRC_CLOSING_BRACKET : ']' -> popMode, popMode, type(CLOSING_BRACKET); -SRC_COMMA : ',' -> type(COMMA); -SRC_ASSIGN : '=' -> type(ASSIGN); AS : 'as'; -METADATA: 'metadata'; -ON : 'on'; -WITH : 'with'; -SRC_UNQUOTED_IDENTIFIER - : SRC_UNQUOTED_IDENTIFIER_PART+ +RENAME_QUOTED_IDENTIFIER + : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER) ; -fragment SRC_UNQUOTED_IDENTIFIER_PART - : ~[=`|,[\]/ \t\r\n]+ - | '/' ~[*/] // allow single / but not followed by another / or * which would start a comment +// use the unquoted pattern to let the parser invalidate fields with * +RENAME_UNQUOTED_IDENTIFIER + : PROJECT_UNQUOTED_IDENTIFIER -> type(PROJECT_UNQUOTED_IDENTIFIER) + ; + +RENAME_LINE_COMMENT + : LINE_COMMENT -> channel(HIDDEN) ; -SRC_QUOTED_IDENTIFIER - : QUOTED_IDENTIFIER +RENAME_MULTILINE_COMMENT + : MULTILINE_COMMENT -> channel(HIDDEN) ; -SRC_LINE_COMMENT +RENAME_WS + : WS -> channel(HIDDEN) + ; + +// | ENRICH ON key WITH fields +mode ENRICH_MODE; +ENRICH_PIPE : PIPE -> type(PIPE), popMode; + +ON : 'on' -> pushMode(ENRICH_FIELD_MODE); +WITH : 'with' -> pushMode(ENRICH_FIELD_MODE); + +// use the unquoted pattern to let the parser invalidate fields with * +ENRICH_POLICY_UNQUOTED_IDENTIFIER + : FROM_UNQUOTED_IDENTIFIER -> type(FROM_UNQUOTED_IDENTIFIER) + ; + +ENRICH_QUOTED_IDENTIFIER + : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER) + ; + +ENRICH_LINE_COMMENT + : LINE_COMMENT -> channel(HIDDEN) + ; + +ENRICH_MULTILINE_COMMENT + : MULTILINE_COMMENT -> channel(HIDDEN) + ; + +ENRICH_WS + : WS -> channel(HIDDEN) + ; + +// submode for Enrich to allow different lexing between policy identifier (loose) and field identifiers +mode ENRICH_FIELD_MODE; +ENRICH_FIELD_PIPE : PIPE -> type(PIPE), popMode, popMode; +ENRICH_FIELD_ASSIGN : ASSIGN -> type(ASSIGN); +ENRICH_FIELD_COMMA : COMMA -> type(COMMA); +ENRICH_FIELD_DOT: DOT -> type(DOT); + +ENRICH_FIELD_WITH : WITH -> type(WITH) ; + +ENRICH_FIELD_UNQUOTED_IDENTIFIER + : PROJECT_UNQUOTED_IDENTIFIER -> type(PROJECT_UNQUOTED_IDENTIFIER) + ; + +ENRICH_FIELD_QUOTED_IDENTIFIER + : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER) + ; + +ENRICH_FIELD_LINE_COMMENT + : LINE_COMMENT -> channel(HIDDEN) + ; + +ENRICH_FIELD_MULTILINE_COMMENT + : MULTILINE_COMMENT -> channel(HIDDEN) + ; + +ENRICH_FIELD_WS + : WS -> channel(HIDDEN) + ; + +mode MVEXPAND_MODE; +MVEXPAND_PIPE : PIPE -> type(PIPE), popMode; +MVEXPAND_DOT: DOT -> type(DOT); + +MVEXPAND_QUOTED_IDENTIFIER + : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER) + ; + +MVEXPAND_UNQUOTED_IDENTIFIER + : UNQUOTED_IDENTIFIER -> type(UNQUOTED_IDENTIFIER) + ; + +MVEXPAND_LINE_COMMENT + : LINE_COMMENT -> channel(HIDDEN) + ; + +MVEXPAND_MULTILINE_COMMENT + : MULTILINE_COMMENT -> channel(HIDDEN) + ; + +MVEXPAND_WS + : WS -> channel(HIDDEN) + ; + +// +// SHOW INFO +// +mode SHOW_MODE; +SHOW_PIPE : PIPE -> type(PIPE), popMode; + +INFO : 'info'; +FUNCTIONS : 'functions'; + +SHOW_LINE_COMMENT : LINE_COMMENT -> channel(HIDDEN) ; -SRC_MULTILINE_COMMENT +SHOW_MULTILINE_COMMENT : MULTILINE_COMMENT -> channel(HIDDEN) ; -SRC_WS +SHOW_WS : WS -> channel(HIDDEN) ; diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens deleted file mode 100644 index d8761f5eb0d73..0000000000000 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens +++ /dev/null @@ -1,137 +0,0 @@ -DISSECT=1 -DROP=2 -ENRICH=3 -EVAL=4 -EXPLAIN=5 -FROM=6 -GROK=7 -INLINESTATS=8 -KEEP=9 -LIMIT=10 -MV_EXPAND=11 -PROJECT=12 -RENAME=13 -ROW=14 -SHOW=15 -SORT=16 -STATS=17 -WHERE=18 -UNKNOWN_CMD=19 -LINE_COMMENT=20 -MULTILINE_COMMENT=21 -WS=22 -EXPLAIN_WS=23 -EXPLAIN_LINE_COMMENT=24 -EXPLAIN_MULTILINE_COMMENT=25 -PIPE=26 -STRING=27 -INTEGER_LITERAL=28 -DECIMAL_LITERAL=29 -BY=30 -AND=31 -ASC=32 -ASSIGN=33 -COMMA=34 -DESC=35 -DOT=36 -FALSE=37 -FIRST=38 -LAST=39 -LP=40 -IN=41 -IS=42 -LIKE=43 -NOT=44 -NULL=45 -NULLS=46 -OR=47 -PARAM=48 -RLIKE=49 -RP=50 -TRUE=51 -INFO=52 -FUNCTIONS=53 -EQ=54 -NEQ=55 -LT=56 -LTE=57 -GT=58 -GTE=59 -PLUS=60 -MINUS=61 -ASTERISK=62 -SLASH=63 -PERCENT=64 -OPENING_BRACKET=65 -CLOSING_BRACKET=66 -UNQUOTED_IDENTIFIER=67 -QUOTED_IDENTIFIER=68 -EXPR_LINE_COMMENT=69 -EXPR_MULTILINE_COMMENT=70 -EXPR_WS=71 -AS=72 -METADATA=73 -ON=74 -WITH=75 -SRC_UNQUOTED_IDENTIFIER=76 -SRC_QUOTED_IDENTIFIER=77 -SRC_LINE_COMMENT=78 -SRC_MULTILINE_COMMENT=79 -SRC_WS=80 -EXPLAIN_PIPE=81 -'dissect'=1 -'drop'=2 -'enrich'=3 -'eval'=4 -'explain'=5 -'from'=6 -'grok'=7 -'inlinestats'=8 -'keep'=9 -'limit'=10 -'mv_expand'=11 -'project'=12 -'rename'=13 -'row'=14 -'show'=15 -'sort'=16 -'stats'=17 -'where'=18 -'by'=30 -'and'=31 -'asc'=32 -'desc'=35 -'.'=36 -'false'=37 -'first'=38 -'last'=39 -'('=40 -'in'=41 -'is'=42 -'like'=43 -'not'=44 -'null'=45 -'nulls'=46 -'or'=47 -'?'=48 -'rlike'=49 -')'=50 -'true'=51 -'info'=52 -'functions'=53 -'=='=54 -'!='=55 -'<'=56 -'<='=57 -'>'=58 -'>='=59 -'+'=60 -'-'=61 -'*'=62 -'/'=63 -'%'=64 -']'=66 -'as'=72 -'metadata'=73 -'on'=74 -'with'=75 diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 index 044e920744375..cdf0cea58b230 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 @@ -98,11 +98,11 @@ field ; fromCommand - : FROM sourceIdentifier (COMMA sourceIdentifier)* metadata? + : FROM fromIdentifier (COMMA fromIdentifier)* metadata? ; metadata - : OPENING_BRACKET METADATA sourceIdentifier (COMMA sourceIdentifier)* CLOSING_BRACKET + : OPENING_BRACKET METADATA fromIdentifier (COMMA fromIdentifier)* CLOSING_BRACKET ; @@ -122,21 +122,29 @@ grouping : qualifiedName (COMMA qualifiedName)* ; -sourceIdentifier - : SRC_UNQUOTED_IDENTIFIER - | SRC_QUOTED_IDENTIFIER +fromIdentifier + : FROM_UNQUOTED_IDENTIFIER + | QUOTED_IDENTIFIER ; qualifiedName : identifier (DOT identifier)* ; +qualifiedNamePattern + : identifierPattern (DOT identifierPattern)* + ; identifier : UNQUOTED_IDENTIFIER | QUOTED_IDENTIFIER ; +identifierPattern + : PROJECT_UNQUOTED_IDENTIFIER + | QUOTED_IDENTIFIER + ; + constant : NULL #nullLiteral | integerValue UNQUOTED_IDENTIFIER #qualifiedIntegerLiteral @@ -163,12 +171,12 @@ orderExpression ; keepCommand - : KEEP sourceIdentifier (COMMA sourceIdentifier)* - | PROJECT sourceIdentifier (COMMA sourceIdentifier)* + : KEEP qualifiedNamePattern (COMMA qualifiedNamePattern)* + | PROJECT qualifiedNamePattern (COMMA qualifiedNamePattern)* ; dropCommand - : DROP sourceIdentifier (COMMA sourceIdentifier)* + : DROP qualifiedNamePattern (COMMA qualifiedNamePattern)* ; renameCommand @@ -176,7 +184,7 @@ renameCommand ; renameClause: - oldName=sourceIdentifier AS newName=sourceIdentifier + oldName=qualifiedNamePattern AS newName=qualifiedNamePattern ; dissectCommand @@ -188,7 +196,7 @@ grokCommand ; mvExpandCommand - : MV_EXPAND sourceIdentifier + : MV_EXPAND qualifiedName ; commandOptions @@ -238,9 +246,9 @@ showCommand ; enrichCommand - : ENRICH policyName=sourceIdentifier (ON matchField=sourceIdentifier)? (WITH enrichWithClause (COMMA enrichWithClause)*)? + : ENRICH policyName=fromIdentifier (ON matchField=qualifiedNamePattern)? (WITH enrichWithClause (COMMA enrichWithClause)*)? ; enrichWithClause - : (newName=sourceIdentifier ASSIGN)? enrichField=sourceIdentifier + : (newName=qualifiedNamePattern ASSIGN)? enrichField=qualifiedNamePattern ; diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens deleted file mode 100644 index d8761f5eb0d73..0000000000000 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens +++ /dev/null @@ -1,137 +0,0 @@ -DISSECT=1 -DROP=2 -ENRICH=3 -EVAL=4 -EXPLAIN=5 -FROM=6 -GROK=7 -INLINESTATS=8 -KEEP=9 -LIMIT=10 -MV_EXPAND=11 -PROJECT=12 -RENAME=13 -ROW=14 -SHOW=15 -SORT=16 -STATS=17 -WHERE=18 -UNKNOWN_CMD=19 -LINE_COMMENT=20 -MULTILINE_COMMENT=21 -WS=22 -EXPLAIN_WS=23 -EXPLAIN_LINE_COMMENT=24 -EXPLAIN_MULTILINE_COMMENT=25 -PIPE=26 -STRING=27 -INTEGER_LITERAL=28 -DECIMAL_LITERAL=29 -BY=30 -AND=31 -ASC=32 -ASSIGN=33 -COMMA=34 -DESC=35 -DOT=36 -FALSE=37 -FIRST=38 -LAST=39 -LP=40 -IN=41 -IS=42 -LIKE=43 -NOT=44 -NULL=45 -NULLS=46 -OR=47 -PARAM=48 -RLIKE=49 -RP=50 -TRUE=51 -INFO=52 -FUNCTIONS=53 -EQ=54 -NEQ=55 -LT=56 -LTE=57 -GT=58 -GTE=59 -PLUS=60 -MINUS=61 -ASTERISK=62 -SLASH=63 -PERCENT=64 -OPENING_BRACKET=65 -CLOSING_BRACKET=66 -UNQUOTED_IDENTIFIER=67 -QUOTED_IDENTIFIER=68 -EXPR_LINE_COMMENT=69 -EXPR_MULTILINE_COMMENT=70 -EXPR_WS=71 -AS=72 -METADATA=73 -ON=74 -WITH=75 -SRC_UNQUOTED_IDENTIFIER=76 -SRC_QUOTED_IDENTIFIER=77 -SRC_LINE_COMMENT=78 -SRC_MULTILINE_COMMENT=79 -SRC_WS=80 -EXPLAIN_PIPE=81 -'dissect'=1 -'drop'=2 -'enrich'=3 -'eval'=4 -'explain'=5 -'from'=6 -'grok'=7 -'inlinestats'=8 -'keep'=9 -'limit'=10 -'mv_expand'=11 -'project'=12 -'rename'=13 -'row'=14 -'show'=15 -'sort'=16 -'stats'=17 -'where'=18 -'by'=30 -'and'=31 -'asc'=32 -'desc'=35 -'.'=36 -'false'=37 -'first'=38 -'last'=39 -'('=40 -'in'=41 -'is'=42 -'like'=43 -'not'=44 -'null'=45 -'nulls'=46 -'or'=47 -'?'=48 -'rlike'=49 -')'=50 -'true'=51 -'info'=52 -'functions'=53 -'=='=54 -'!='=55 -'<'=56 -'<='=57 -'>'=58 -'>='=59 -'+'=60 -'-'=61 -'*'=62 -'/'=63 -'%'=64 -']'=66 -'as'=72 -'metadata'=73 -'on'=74 -'with'=75 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 6959c04345d31..674a32db1f0fb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -68,6 +68,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -416,6 +417,40 @@ private LogicalPlan resolveEval(Eval eval, List childOutput) { return changed ? new Eval(eval.source(), eval.child(), newFields) : eval; } + /** + * resolve each item manually. + * + * Fields are added in the order they appear. + * + * If one field matches multiple expressions, the following precedence rules apply (higher to lower): + * 1. complete field name (ie. no wildcards) + * 2. partial wildcard expressions (eg. fieldNam*) + * 3. wildcard only (ie. *) + * + * If a field name matches multiple expressions with the same precedence, last one is used. + * + * A few examples below: + * + * // full name + * row foo = 1, bar = 2 | keep foo, bar, foo -> bar, foo + * + * // the full name has precedence on wildcard expression + * row foo = 1, bar = 2 | keep foo, bar, foo* -> foo, bar + * + * // the two wildcard expressions have the same priority, even though the first one is more specific + * // so last one wins + * row foo = 1, bar = 2 | keep foo*, bar, fo* -> bar, foo + * + * // * has the lowest priority + * row foo = 1, bar = 2 | keep *, foo -> bar, foo + * row foo = 1, bar = 2 | keep foo, * -> foo, bar + * row foo = 1, bar = 2 | keep bar*, foo, * -> bar, foo + * + * + * @param p + * @param childOutput + * @return + */ private LogicalPlan resolveKeep(Project p, List childOutput) { List resolvedProjections = new ArrayList<>(); var projections = p.projections(); @@ -427,26 +462,31 @@ private LogicalPlan resolveKeep(Project p, List childOutput) { } // otherwise resolve them else { - var starPosition = -1; // no star - // resolve each item manually while paying attention to: - // 1. name patterns a*, *b, a*b - // 2. star * - which can only appear once and signifies "everything else" - this will be added at the end - for (var ne : projections) { - if (ne instanceof UnresolvedStar) { - starPosition = resolvedProjections.size(); - } else if (ne instanceof UnresolvedAttribute ua) { - resolvedProjections.addAll(resolveAgainstList(ua, childOutput)); - } else { - // if this gets here it means it was already resolved - resolvedProjections.add(ne); + Map priorities = new LinkedHashMap<>(); + for (Attribute attribute : childOutput) { + for (var proj : projections) { + List resolved; + int priority; + if (proj instanceof UnresolvedStar) { + resolved = childOutput; + priority = 2; + } else if (proj instanceof UnresolvedAttribute ua) { + resolved = resolveAgainstList(ua, childOutput); + priority = Regex.isSimpleMatchPattern(ua.name()) ? 1 : 0; + } else { + resolved = List.of(attribute); + priority = 0; + } + for (Attribute attr : resolved) { + Integer previousPrio = priorities.get(attr); + if (previousPrio == null || previousPrio >= priority) { + priorities.remove(attr); + priorities.put(attr, priority); + } + } } } - // compute star if specified and add it to the list - if (starPosition >= 0) { - var remainingProjections = new ArrayList<>(childOutput); - remainingProjections.removeAll(resolvedProjections); - resolvedProjections.addAll(starPosition, remainingProjections); - } + resolvedProjections = new ArrayList<>(priorities.keySet()); } return new EsqlProject(p.source(), p.child(), resolvedProjections); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java index 3451a3981d3e3..e05dd9a00c567 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java @@ -7,10 +7,12 @@ package org.elasticsearch.xpack.esql.optimizer; +import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.TopN; import org.elasticsearch.xpack.esql.stats.SearchStats; import org.elasticsearch.xpack.ql.expression.Alias; +import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.expression.NamedExpression; @@ -37,7 +39,13 @@ public LocalLogicalPlanOptimizer(LocalLogicalOptimizerContext localLogicalOptimi @Override protected List> batches() { - var local = new Batch<>("Local rewrite", new ReplaceTopNWithLimitAndSort(), new ReplaceMissingFieldWithNull()); + var local = new Batch<>( + "Local rewrite", + Limiter.ONCE, + new ReplaceTopNWithLimitAndSort(), + new ReplaceMissingFieldWithNull(), + new InferIsNotNull() + ); var rules = new ArrayList>(); rules.add(local); @@ -116,6 +124,14 @@ else if (plan instanceof Project project) { } } + static class InferIsNotNull extends OptimizerRules.InferIsNotNull { + + @Override + protected boolean skipExpression(Expression e) { + return e instanceof Coalesce; + } + } + abstract static class ParameterizedOptimizerRule extends ParameterizedRule { public final LogicalPlan apply(LogicalPlan plan, P context) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp index 12542878c3ed3..585f722065e6f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp @@ -25,15 +25,15 @@ null null null null -null +'|' null null null 'by' 'and' 'asc' -null -null +'=' +',' 'desc' '.' 'false' @@ -51,8 +51,6 @@ null 'rlike' ')' 'true' -'info' -'functions' '==' '!=' '<' @@ -71,8 +69,19 @@ null null null null -'as' 'metadata' +null +null +null +null +null +null +null +null +'as' +null +null +null 'on' 'with' null @@ -81,6 +90,14 @@ null null null null +null +null +null +'info' +'functions' +null +null +null token symbolic names: null @@ -135,8 +152,6 @@ PARAM RLIKE RP TRUE -INFO -FUNCTIONS EQ NEQ LT @@ -155,16 +170,35 @@ QUOTED_IDENTIFIER EXPR_LINE_COMMENT EXPR_MULTILINE_COMMENT EXPR_WS -AS METADATA +FROM_UNQUOTED_IDENTIFIER +FROM_LINE_COMMENT +FROM_MULTILINE_COMMENT +FROM_WS +PROJECT_UNQUOTED_IDENTIFIER +PROJECT_LINE_COMMENT +PROJECT_MULTILINE_COMMENT +PROJECT_WS +AS +RENAME_LINE_COMMENT +RENAME_MULTILINE_COMMENT +RENAME_WS ON WITH -SRC_UNQUOTED_IDENTIFIER -SRC_QUOTED_IDENTIFIER -SRC_LINE_COMMENT -SRC_MULTILINE_COMMENT -SRC_WS -EXPLAIN_PIPE +ENRICH_LINE_COMMENT +ENRICH_MULTILINE_COMMENT +ENRICH_WS +ENRICH_FIELD_LINE_COMMENT +ENRICH_FIELD_MULTILINE_COMMENT +ENRICH_FIELD_WS +MVEXPAND_LINE_COMMENT +MVEXPAND_MULTILINE_COMMENT +MVEXPAND_WS +INFO +FUNCTIONS +SHOW_LINE_COMMENT +SHOW_MULTILINE_COMMENT +SHOW_WS rule names: DISSECT @@ -200,6 +234,11 @@ LETTER ESCAPE_SEQUENCE UNESCAPED_CHARS EXPONENT +ASPERAND +BACKQUOTE +BACKQUOTE_BLOCK +UNDERSCORE +UNQUOTED_ID_BODY STRING INTEGER_LITERAL DECIMAL_LITERAL @@ -225,8 +264,6 @@ PARAM RLIKE RP TRUE -INFO -FUNCTIONS EQ NEQ LT @@ -245,21 +282,68 @@ QUOTED_IDENTIFIER EXPR_LINE_COMMENT EXPR_MULTILINE_COMMENT EXPR_WS -SRC_PIPE -SRC_OPENING_BRACKET -SRC_CLOSING_BRACKET -SRC_COMMA -SRC_ASSIGN -AS +FROM_PIPE +FROM_OPENING_BRACKET +FROM_CLOSING_BRACKET +FROM_COMMA +FROM_ASSIGN METADATA +FROM_UNQUOTED_IDENTIFIER_PART +FROM_UNQUOTED_IDENTIFIER +FROM_QUOTED_IDENTIFIER +FROM_LINE_COMMENT +FROM_MULTILINE_COMMENT +FROM_WS +PROJECT_PIPE +PROJECT_DOT +PROJECT_COMMA +UNQUOTED_ID_BODY_WITH_PATTERN +PROJECT_UNQUOTED_IDENTIFIER +PROJECT_QUOTED_IDENTIFIER +PROJECT_LINE_COMMENT +PROJECT_MULTILINE_COMMENT +PROJECT_WS +RENAME_PIPE +RENAME_ASSIGN +RENAME_COMMA +RENAME_DOT +AS +RENAME_QUOTED_IDENTIFIER +RENAME_UNQUOTED_IDENTIFIER +RENAME_LINE_COMMENT +RENAME_MULTILINE_COMMENT +RENAME_WS +ENRICH_PIPE ON WITH -SRC_UNQUOTED_IDENTIFIER -SRC_UNQUOTED_IDENTIFIER_PART -SRC_QUOTED_IDENTIFIER -SRC_LINE_COMMENT -SRC_MULTILINE_COMMENT -SRC_WS +ENRICH_POLICY_UNQUOTED_IDENTIFIER +ENRICH_QUOTED_IDENTIFIER +ENRICH_LINE_COMMENT +ENRICH_MULTILINE_COMMENT +ENRICH_WS +ENRICH_FIELD_PIPE +ENRICH_FIELD_ASSIGN +ENRICH_FIELD_COMMA +ENRICH_FIELD_DOT +ENRICH_FIELD_WITH +ENRICH_FIELD_UNQUOTED_IDENTIFIER +ENRICH_FIELD_QUOTED_IDENTIFIER +ENRICH_FIELD_LINE_COMMENT +ENRICH_FIELD_MULTILINE_COMMENT +ENRICH_FIELD_WS +MVEXPAND_PIPE +MVEXPAND_DOT +MVEXPAND_QUOTED_IDENTIFIER +MVEXPAND_UNQUOTED_IDENTIFIER +MVEXPAND_LINE_COMMENT +MVEXPAND_MULTILINE_COMMENT +MVEXPAND_WS +SHOW_PIPE +INFO +FUNCTIONS +SHOW_LINE_COMMENT +SHOW_MULTILINE_COMMENT +SHOW_WS channel names: DEFAULT_TOKEN_CHANNEL @@ -268,8 +352,14 @@ HIDDEN mode names: DEFAULT_MODE EXPLAIN_MODE -EXPRESSION -SOURCE_IDENTIFIERS +EXPRESSION_MODE +FROM_MODE +PROJECT_MODE +RENAME_MODE +ENRICH_MODE +ENRICH_FIELD_MODE +MVEXPAND_MODE +SHOW_MODE atn: -[4, 0, 81, 764, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 4, 18, 345, 8, 18, 11, 18, 12, 18, 346, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 5, 19, 355, 8, 19, 10, 19, 12, 19, 358, 9, 19, 1, 19, 3, 19, 361, 8, 19, 1, 19, 3, 19, 364, 8, 19, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 5, 20, 373, 8, 20, 10, 20, 12, 20, 376, 9, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 21, 4, 21, 384, 8, 21, 11, 21, 12, 21, 385, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 27, 1, 28, 1, 28, 1, 29, 1, 29, 1, 30, 1, 30, 1, 30, 1, 31, 1, 31, 1, 32, 1, 32, 3, 32, 427, 8, 32, 1, 32, 4, 32, 430, 8, 32, 11, 32, 12, 32, 431, 1, 33, 1, 33, 1, 33, 5, 33, 437, 8, 33, 10, 33, 12, 33, 440, 9, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 5, 33, 448, 8, 33, 10, 33, 12, 33, 451, 9, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 3, 33, 458, 8, 33, 1, 33, 3, 33, 461, 8, 33, 3, 33, 463, 8, 33, 1, 34, 4, 34, 466, 8, 34, 11, 34, 12, 34, 467, 1, 35, 4, 35, 471, 8, 35, 11, 35, 12, 35, 472, 1, 35, 1, 35, 5, 35, 477, 8, 35, 10, 35, 12, 35, 480, 9, 35, 1, 35, 1, 35, 4, 35, 484, 8, 35, 11, 35, 12, 35, 485, 1, 35, 4, 35, 489, 8, 35, 11, 35, 12, 35, 490, 1, 35, 1, 35, 5, 35, 495, 8, 35, 10, 35, 12, 35, 498, 9, 35, 3, 35, 500, 8, 35, 1, 35, 1, 35, 1, 35, 1, 35, 4, 35, 506, 8, 35, 11, 35, 12, 35, 507, 1, 35, 1, 35, 3, 35, 512, 8, 35, 1, 36, 1, 36, 1, 36, 1, 37, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 64, 1, 64, 1, 65, 1, 65, 1, 65, 1, 66, 1, 66, 1, 67, 1, 67, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 71, 1, 71, 1, 71, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 73, 1, 73, 1, 73, 1, 73, 5, 73, 654, 8, 73, 10, 73, 12, 73, 657, 9, 73, 1, 73, 1, 73, 1, 73, 1, 73, 4, 73, 663, 8, 73, 11, 73, 12, 73, 664, 3, 73, 667, 8, 73, 1, 74, 1, 74, 1, 74, 1, 74, 5, 74, 673, 8, 74, 10, 74, 12, 74, 676, 9, 74, 1, 74, 1, 74, 1, 75, 1, 75, 1, 75, 1, 75, 1, 76, 1, 76, 1, 76, 1, 76, 1, 77, 1, 77, 1, 77, 1, 77, 1, 78, 1, 78, 1, 78, 1, 78, 1, 78, 1, 79, 1, 79, 1, 79, 1, 79, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 4, 87, 738, 8, 87, 11, 87, 12, 87, 739, 1, 88, 4, 88, 743, 8, 88, 11, 88, 12, 88, 744, 1, 88, 1, 88, 3, 88, 749, 8, 88, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 2, 374, 449, 0, 93, 4, 1, 6, 2, 8, 3, 10, 4, 12, 5, 14, 6, 16, 7, 18, 8, 20, 9, 22, 10, 24, 11, 26, 12, 28, 13, 30, 14, 32, 15, 34, 16, 36, 17, 38, 18, 40, 19, 42, 20, 44, 21, 46, 22, 48, 0, 50, 81, 52, 23, 54, 24, 56, 25, 58, 26, 60, 0, 62, 0, 64, 0, 66, 0, 68, 0, 70, 27, 72, 28, 74, 29, 76, 30, 78, 31, 80, 32, 82, 33, 84, 34, 86, 35, 88, 36, 90, 37, 92, 38, 94, 39, 96, 40, 98, 41, 100, 42, 102, 43, 104, 44, 106, 45, 108, 46, 110, 47, 112, 48, 114, 49, 116, 50, 118, 51, 120, 52, 122, 53, 124, 54, 126, 55, 128, 56, 130, 57, 132, 58, 134, 59, 136, 60, 138, 61, 140, 62, 142, 63, 144, 64, 146, 65, 148, 66, 150, 67, 152, 68, 154, 69, 156, 70, 158, 71, 160, 0, 162, 0, 164, 0, 166, 0, 168, 0, 170, 72, 172, 73, 174, 74, 176, 75, 178, 76, 180, 0, 182, 77, 184, 78, 186, 79, 188, 80, 4, 0, 1, 2, 3, 13, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 5, 0, 34, 34, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 69, 69, 101, 101, 2, 0, 43, 43, 45, 45, 2, 0, 64, 64, 95, 95, 1, 0, 96, 96, 10, 0, 9, 10, 13, 13, 32, 32, 44, 44, 47, 47, 61, 61, 91, 91, 93, 93, 96, 96, 124, 124, 2, 0, 42, 42, 47, 47, 792, 0, 4, 1, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 8, 1, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 12, 1, 0, 0, 0, 0, 14, 1, 0, 0, 0, 0, 16, 1, 0, 0, 0, 0, 18, 1, 0, 0, 0, 0, 20, 1, 0, 0, 0, 0, 22, 1, 0, 0, 0, 0, 24, 1, 0, 0, 0, 0, 26, 1, 0, 0, 0, 0, 28, 1, 0, 0, 0, 0, 30, 1, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 34, 1, 0, 0, 0, 0, 36, 1, 0, 0, 0, 0, 38, 1, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 42, 1, 0, 0, 0, 0, 44, 1, 0, 0, 0, 0, 46, 1, 0, 0, 0, 1, 48, 1, 0, 0, 0, 1, 50, 1, 0, 0, 0, 1, 52, 1, 0, 0, 0, 1, 54, 1, 0, 0, 0, 1, 56, 1, 0, 0, 0, 2, 58, 1, 0, 0, 0, 2, 70, 1, 0, 0, 0, 2, 72, 1, 0, 0, 0, 2, 74, 1, 0, 0, 0, 2, 76, 1, 0, 0, 0, 2, 78, 1, 0, 0, 0, 2, 80, 1, 0, 0, 0, 2, 82, 1, 0, 0, 0, 2, 84, 1, 0, 0, 0, 2, 86, 1, 0, 0, 0, 2, 88, 1, 0, 0, 0, 2, 90, 1, 0, 0, 0, 2, 92, 1, 0, 0, 0, 2, 94, 1, 0, 0, 0, 2, 96, 1, 0, 0, 0, 2, 98, 1, 0, 0, 0, 2, 100, 1, 0, 0, 0, 2, 102, 1, 0, 0, 0, 2, 104, 1, 0, 0, 0, 2, 106, 1, 0, 0, 0, 2, 108, 1, 0, 0, 0, 2, 110, 1, 0, 0, 0, 2, 112, 1, 0, 0, 0, 2, 114, 1, 0, 0, 0, 2, 116, 1, 0, 0, 0, 2, 118, 1, 0, 0, 0, 2, 120, 1, 0, 0, 0, 2, 122, 1, 0, 0, 0, 2, 124, 1, 0, 0, 0, 2, 126, 1, 0, 0, 0, 2, 128, 1, 0, 0, 0, 2, 130, 1, 0, 0, 0, 2, 132, 1, 0, 0, 0, 2, 134, 1, 0, 0, 0, 2, 136, 1, 0, 0, 0, 2, 138, 1, 0, 0, 0, 2, 140, 1, 0, 0, 0, 2, 142, 1, 0, 0, 0, 2, 144, 1, 0, 0, 0, 2, 146, 1, 0, 0, 0, 2, 148, 1, 0, 0, 0, 2, 150, 1, 0, 0, 0, 2, 152, 1, 0, 0, 0, 2, 154, 1, 0, 0, 0, 2, 156, 1, 0, 0, 0, 2, 158, 1, 0, 0, 0, 3, 160, 1, 0, 0, 0, 3, 162, 1, 0, 0, 0, 3, 164, 1, 0, 0, 0, 3, 166, 1, 0, 0, 0, 3, 168, 1, 0, 0, 0, 3, 170, 1, 0, 0, 0, 3, 172, 1, 0, 0, 0, 3, 174, 1, 0, 0, 0, 3, 176, 1, 0, 0, 0, 3, 178, 1, 0, 0, 0, 3, 182, 1, 0, 0, 0, 3, 184, 1, 0, 0, 0, 3, 186, 1, 0, 0, 0, 3, 188, 1, 0, 0, 0, 4, 190, 1, 0, 0, 0, 6, 200, 1, 0, 0, 0, 8, 207, 1, 0, 0, 0, 10, 216, 1, 0, 0, 0, 12, 223, 1, 0, 0, 0, 14, 233, 1, 0, 0, 0, 16, 240, 1, 0, 0, 0, 18, 247, 1, 0, 0, 0, 20, 261, 1, 0, 0, 0, 22, 268, 1, 0, 0, 0, 24, 276, 1, 0, 0, 0, 26, 288, 1, 0, 0, 0, 28, 298, 1, 0, 0, 0, 30, 307, 1, 0, 0, 0, 32, 313, 1, 0, 0, 0, 34, 320, 1, 0, 0, 0, 36, 327, 1, 0, 0, 0, 38, 335, 1, 0, 0, 0, 40, 344, 1, 0, 0, 0, 42, 350, 1, 0, 0, 0, 44, 367, 1, 0, 0, 0, 46, 383, 1, 0, 0, 0, 48, 389, 1, 0, 0, 0, 50, 394, 1, 0, 0, 0, 52, 399, 1, 0, 0, 0, 54, 403, 1, 0, 0, 0, 56, 407, 1, 0, 0, 0, 58, 411, 1, 0, 0, 0, 60, 415, 1, 0, 0, 0, 62, 417, 1, 0, 0, 0, 64, 419, 1, 0, 0, 0, 66, 422, 1, 0, 0, 0, 68, 424, 1, 0, 0, 0, 70, 462, 1, 0, 0, 0, 72, 465, 1, 0, 0, 0, 74, 511, 1, 0, 0, 0, 76, 513, 1, 0, 0, 0, 78, 516, 1, 0, 0, 0, 80, 520, 1, 0, 0, 0, 82, 524, 1, 0, 0, 0, 84, 526, 1, 0, 0, 0, 86, 528, 1, 0, 0, 0, 88, 533, 1, 0, 0, 0, 90, 535, 1, 0, 0, 0, 92, 541, 1, 0, 0, 0, 94, 547, 1, 0, 0, 0, 96, 552, 1, 0, 0, 0, 98, 554, 1, 0, 0, 0, 100, 557, 1, 0, 0, 0, 102, 560, 1, 0, 0, 0, 104, 565, 1, 0, 0, 0, 106, 569, 1, 0, 0, 0, 108, 574, 1, 0, 0, 0, 110, 580, 1, 0, 0, 0, 112, 583, 1, 0, 0, 0, 114, 585, 1, 0, 0, 0, 116, 591, 1, 0, 0, 0, 118, 593, 1, 0, 0, 0, 120, 598, 1, 0, 0, 0, 122, 603, 1, 0, 0, 0, 124, 613, 1, 0, 0, 0, 126, 616, 1, 0, 0, 0, 128, 619, 1, 0, 0, 0, 130, 621, 1, 0, 0, 0, 132, 624, 1, 0, 0, 0, 134, 626, 1, 0, 0, 0, 136, 629, 1, 0, 0, 0, 138, 631, 1, 0, 0, 0, 140, 633, 1, 0, 0, 0, 142, 635, 1, 0, 0, 0, 144, 637, 1, 0, 0, 0, 146, 639, 1, 0, 0, 0, 148, 644, 1, 0, 0, 0, 150, 666, 1, 0, 0, 0, 152, 668, 1, 0, 0, 0, 154, 679, 1, 0, 0, 0, 156, 683, 1, 0, 0, 0, 158, 687, 1, 0, 0, 0, 160, 691, 1, 0, 0, 0, 162, 696, 1, 0, 0, 0, 164, 702, 1, 0, 0, 0, 166, 708, 1, 0, 0, 0, 168, 712, 1, 0, 0, 0, 170, 716, 1, 0, 0, 0, 172, 719, 1, 0, 0, 0, 174, 728, 1, 0, 0, 0, 176, 731, 1, 0, 0, 0, 178, 737, 1, 0, 0, 0, 180, 748, 1, 0, 0, 0, 182, 750, 1, 0, 0, 0, 184, 752, 1, 0, 0, 0, 186, 756, 1, 0, 0, 0, 188, 760, 1, 0, 0, 0, 190, 191, 5, 100, 0, 0, 191, 192, 5, 105, 0, 0, 192, 193, 5, 115, 0, 0, 193, 194, 5, 115, 0, 0, 194, 195, 5, 101, 0, 0, 195, 196, 5, 99, 0, 0, 196, 197, 5, 116, 0, 0, 197, 198, 1, 0, 0, 0, 198, 199, 6, 0, 0, 0, 199, 5, 1, 0, 0, 0, 200, 201, 5, 100, 0, 0, 201, 202, 5, 114, 0, 0, 202, 203, 5, 111, 0, 0, 203, 204, 5, 112, 0, 0, 204, 205, 1, 0, 0, 0, 205, 206, 6, 1, 1, 0, 206, 7, 1, 0, 0, 0, 207, 208, 5, 101, 0, 0, 208, 209, 5, 110, 0, 0, 209, 210, 5, 114, 0, 0, 210, 211, 5, 105, 0, 0, 211, 212, 5, 99, 0, 0, 212, 213, 5, 104, 0, 0, 213, 214, 1, 0, 0, 0, 214, 215, 6, 2, 1, 0, 215, 9, 1, 0, 0, 0, 216, 217, 5, 101, 0, 0, 217, 218, 5, 118, 0, 0, 218, 219, 5, 97, 0, 0, 219, 220, 5, 108, 0, 0, 220, 221, 1, 0, 0, 0, 221, 222, 6, 3, 0, 0, 222, 11, 1, 0, 0, 0, 223, 224, 5, 101, 0, 0, 224, 225, 5, 120, 0, 0, 225, 226, 5, 112, 0, 0, 226, 227, 5, 108, 0, 0, 227, 228, 5, 97, 0, 0, 228, 229, 5, 105, 0, 0, 229, 230, 5, 110, 0, 0, 230, 231, 1, 0, 0, 0, 231, 232, 6, 4, 2, 0, 232, 13, 1, 0, 0, 0, 233, 234, 5, 102, 0, 0, 234, 235, 5, 114, 0, 0, 235, 236, 5, 111, 0, 0, 236, 237, 5, 109, 0, 0, 237, 238, 1, 0, 0, 0, 238, 239, 6, 5, 1, 0, 239, 15, 1, 0, 0, 0, 240, 241, 5, 103, 0, 0, 241, 242, 5, 114, 0, 0, 242, 243, 5, 111, 0, 0, 243, 244, 5, 107, 0, 0, 244, 245, 1, 0, 0, 0, 245, 246, 6, 6, 0, 0, 246, 17, 1, 0, 0, 0, 247, 248, 5, 105, 0, 0, 248, 249, 5, 110, 0, 0, 249, 250, 5, 108, 0, 0, 250, 251, 5, 105, 0, 0, 251, 252, 5, 110, 0, 0, 252, 253, 5, 101, 0, 0, 253, 254, 5, 115, 0, 0, 254, 255, 5, 116, 0, 0, 255, 256, 5, 97, 0, 0, 256, 257, 5, 116, 0, 0, 257, 258, 5, 115, 0, 0, 258, 259, 1, 0, 0, 0, 259, 260, 6, 7, 0, 0, 260, 19, 1, 0, 0, 0, 261, 262, 5, 107, 0, 0, 262, 263, 5, 101, 0, 0, 263, 264, 5, 101, 0, 0, 264, 265, 5, 112, 0, 0, 265, 266, 1, 0, 0, 0, 266, 267, 6, 8, 1, 0, 267, 21, 1, 0, 0, 0, 268, 269, 5, 108, 0, 0, 269, 270, 5, 105, 0, 0, 270, 271, 5, 109, 0, 0, 271, 272, 5, 105, 0, 0, 272, 273, 5, 116, 0, 0, 273, 274, 1, 0, 0, 0, 274, 275, 6, 9, 0, 0, 275, 23, 1, 0, 0, 0, 276, 277, 5, 109, 0, 0, 277, 278, 5, 118, 0, 0, 278, 279, 5, 95, 0, 0, 279, 280, 5, 101, 0, 0, 280, 281, 5, 120, 0, 0, 281, 282, 5, 112, 0, 0, 282, 283, 5, 97, 0, 0, 283, 284, 5, 110, 0, 0, 284, 285, 5, 100, 0, 0, 285, 286, 1, 0, 0, 0, 286, 287, 6, 10, 1, 0, 287, 25, 1, 0, 0, 0, 288, 289, 5, 112, 0, 0, 289, 290, 5, 114, 0, 0, 290, 291, 5, 111, 0, 0, 291, 292, 5, 106, 0, 0, 292, 293, 5, 101, 0, 0, 293, 294, 5, 99, 0, 0, 294, 295, 5, 116, 0, 0, 295, 296, 1, 0, 0, 0, 296, 297, 6, 11, 1, 0, 297, 27, 1, 0, 0, 0, 298, 299, 5, 114, 0, 0, 299, 300, 5, 101, 0, 0, 300, 301, 5, 110, 0, 0, 301, 302, 5, 97, 0, 0, 302, 303, 5, 109, 0, 0, 303, 304, 5, 101, 0, 0, 304, 305, 1, 0, 0, 0, 305, 306, 6, 12, 1, 0, 306, 29, 1, 0, 0, 0, 307, 308, 5, 114, 0, 0, 308, 309, 5, 111, 0, 0, 309, 310, 5, 119, 0, 0, 310, 311, 1, 0, 0, 0, 311, 312, 6, 13, 0, 0, 312, 31, 1, 0, 0, 0, 313, 314, 5, 115, 0, 0, 314, 315, 5, 104, 0, 0, 315, 316, 5, 111, 0, 0, 316, 317, 5, 119, 0, 0, 317, 318, 1, 0, 0, 0, 318, 319, 6, 14, 0, 0, 319, 33, 1, 0, 0, 0, 320, 321, 5, 115, 0, 0, 321, 322, 5, 111, 0, 0, 322, 323, 5, 114, 0, 0, 323, 324, 5, 116, 0, 0, 324, 325, 1, 0, 0, 0, 325, 326, 6, 15, 0, 0, 326, 35, 1, 0, 0, 0, 327, 328, 5, 115, 0, 0, 328, 329, 5, 116, 0, 0, 329, 330, 5, 97, 0, 0, 330, 331, 5, 116, 0, 0, 331, 332, 5, 115, 0, 0, 332, 333, 1, 0, 0, 0, 333, 334, 6, 16, 0, 0, 334, 37, 1, 0, 0, 0, 335, 336, 5, 119, 0, 0, 336, 337, 5, 104, 0, 0, 337, 338, 5, 101, 0, 0, 338, 339, 5, 114, 0, 0, 339, 340, 5, 101, 0, 0, 340, 341, 1, 0, 0, 0, 341, 342, 6, 17, 0, 0, 342, 39, 1, 0, 0, 0, 343, 345, 8, 0, 0, 0, 344, 343, 1, 0, 0, 0, 345, 346, 1, 0, 0, 0, 346, 344, 1, 0, 0, 0, 346, 347, 1, 0, 0, 0, 347, 348, 1, 0, 0, 0, 348, 349, 6, 18, 0, 0, 349, 41, 1, 0, 0, 0, 350, 351, 5, 47, 0, 0, 351, 352, 5, 47, 0, 0, 352, 356, 1, 0, 0, 0, 353, 355, 8, 1, 0, 0, 354, 353, 1, 0, 0, 0, 355, 358, 1, 0, 0, 0, 356, 354, 1, 0, 0, 0, 356, 357, 1, 0, 0, 0, 357, 360, 1, 0, 0, 0, 358, 356, 1, 0, 0, 0, 359, 361, 5, 13, 0, 0, 360, 359, 1, 0, 0, 0, 360, 361, 1, 0, 0, 0, 361, 363, 1, 0, 0, 0, 362, 364, 5, 10, 0, 0, 363, 362, 1, 0, 0, 0, 363, 364, 1, 0, 0, 0, 364, 365, 1, 0, 0, 0, 365, 366, 6, 19, 3, 0, 366, 43, 1, 0, 0, 0, 367, 368, 5, 47, 0, 0, 368, 369, 5, 42, 0, 0, 369, 374, 1, 0, 0, 0, 370, 373, 3, 44, 20, 0, 371, 373, 9, 0, 0, 0, 372, 370, 1, 0, 0, 0, 372, 371, 1, 0, 0, 0, 373, 376, 1, 0, 0, 0, 374, 375, 1, 0, 0, 0, 374, 372, 1, 0, 0, 0, 375, 377, 1, 0, 0, 0, 376, 374, 1, 0, 0, 0, 377, 378, 5, 42, 0, 0, 378, 379, 5, 47, 0, 0, 379, 380, 1, 0, 0, 0, 380, 381, 6, 20, 3, 0, 381, 45, 1, 0, 0, 0, 382, 384, 7, 2, 0, 0, 383, 382, 1, 0, 0, 0, 384, 385, 1, 0, 0, 0, 385, 383, 1, 0, 0, 0, 385, 386, 1, 0, 0, 0, 386, 387, 1, 0, 0, 0, 387, 388, 6, 21, 3, 0, 388, 47, 1, 0, 0, 0, 389, 390, 5, 91, 0, 0, 390, 391, 1, 0, 0, 0, 391, 392, 6, 22, 4, 0, 392, 393, 6, 22, 5, 0, 393, 49, 1, 0, 0, 0, 394, 395, 5, 124, 0, 0, 395, 396, 1, 0, 0, 0, 396, 397, 6, 23, 6, 0, 397, 398, 6, 23, 7, 0, 398, 51, 1, 0, 0, 0, 399, 400, 3, 46, 21, 0, 400, 401, 1, 0, 0, 0, 401, 402, 6, 24, 3, 0, 402, 53, 1, 0, 0, 0, 403, 404, 3, 42, 19, 0, 404, 405, 1, 0, 0, 0, 405, 406, 6, 25, 3, 0, 406, 55, 1, 0, 0, 0, 407, 408, 3, 44, 20, 0, 408, 409, 1, 0, 0, 0, 409, 410, 6, 26, 3, 0, 410, 57, 1, 0, 0, 0, 411, 412, 5, 124, 0, 0, 412, 413, 1, 0, 0, 0, 413, 414, 6, 27, 7, 0, 414, 59, 1, 0, 0, 0, 415, 416, 7, 3, 0, 0, 416, 61, 1, 0, 0, 0, 417, 418, 7, 4, 0, 0, 418, 63, 1, 0, 0, 0, 419, 420, 5, 92, 0, 0, 420, 421, 7, 5, 0, 0, 421, 65, 1, 0, 0, 0, 422, 423, 8, 6, 0, 0, 423, 67, 1, 0, 0, 0, 424, 426, 7, 7, 0, 0, 425, 427, 7, 8, 0, 0, 426, 425, 1, 0, 0, 0, 426, 427, 1, 0, 0, 0, 427, 429, 1, 0, 0, 0, 428, 430, 3, 60, 28, 0, 429, 428, 1, 0, 0, 0, 430, 431, 1, 0, 0, 0, 431, 429, 1, 0, 0, 0, 431, 432, 1, 0, 0, 0, 432, 69, 1, 0, 0, 0, 433, 438, 5, 34, 0, 0, 434, 437, 3, 64, 30, 0, 435, 437, 3, 66, 31, 0, 436, 434, 1, 0, 0, 0, 436, 435, 1, 0, 0, 0, 437, 440, 1, 0, 0, 0, 438, 436, 1, 0, 0, 0, 438, 439, 1, 0, 0, 0, 439, 441, 1, 0, 0, 0, 440, 438, 1, 0, 0, 0, 441, 463, 5, 34, 0, 0, 442, 443, 5, 34, 0, 0, 443, 444, 5, 34, 0, 0, 444, 445, 5, 34, 0, 0, 445, 449, 1, 0, 0, 0, 446, 448, 8, 1, 0, 0, 447, 446, 1, 0, 0, 0, 448, 451, 1, 0, 0, 0, 449, 450, 1, 0, 0, 0, 449, 447, 1, 0, 0, 0, 450, 452, 1, 0, 0, 0, 451, 449, 1, 0, 0, 0, 452, 453, 5, 34, 0, 0, 453, 454, 5, 34, 0, 0, 454, 455, 5, 34, 0, 0, 455, 457, 1, 0, 0, 0, 456, 458, 5, 34, 0, 0, 457, 456, 1, 0, 0, 0, 457, 458, 1, 0, 0, 0, 458, 460, 1, 0, 0, 0, 459, 461, 5, 34, 0, 0, 460, 459, 1, 0, 0, 0, 460, 461, 1, 0, 0, 0, 461, 463, 1, 0, 0, 0, 462, 433, 1, 0, 0, 0, 462, 442, 1, 0, 0, 0, 463, 71, 1, 0, 0, 0, 464, 466, 3, 60, 28, 0, 465, 464, 1, 0, 0, 0, 466, 467, 1, 0, 0, 0, 467, 465, 1, 0, 0, 0, 467, 468, 1, 0, 0, 0, 468, 73, 1, 0, 0, 0, 469, 471, 3, 60, 28, 0, 470, 469, 1, 0, 0, 0, 471, 472, 1, 0, 0, 0, 472, 470, 1, 0, 0, 0, 472, 473, 1, 0, 0, 0, 473, 474, 1, 0, 0, 0, 474, 478, 3, 88, 42, 0, 475, 477, 3, 60, 28, 0, 476, 475, 1, 0, 0, 0, 477, 480, 1, 0, 0, 0, 478, 476, 1, 0, 0, 0, 478, 479, 1, 0, 0, 0, 479, 512, 1, 0, 0, 0, 480, 478, 1, 0, 0, 0, 481, 483, 3, 88, 42, 0, 482, 484, 3, 60, 28, 0, 483, 482, 1, 0, 0, 0, 484, 485, 1, 0, 0, 0, 485, 483, 1, 0, 0, 0, 485, 486, 1, 0, 0, 0, 486, 512, 1, 0, 0, 0, 487, 489, 3, 60, 28, 0, 488, 487, 1, 0, 0, 0, 489, 490, 1, 0, 0, 0, 490, 488, 1, 0, 0, 0, 490, 491, 1, 0, 0, 0, 491, 499, 1, 0, 0, 0, 492, 496, 3, 88, 42, 0, 493, 495, 3, 60, 28, 0, 494, 493, 1, 0, 0, 0, 495, 498, 1, 0, 0, 0, 496, 494, 1, 0, 0, 0, 496, 497, 1, 0, 0, 0, 497, 500, 1, 0, 0, 0, 498, 496, 1, 0, 0, 0, 499, 492, 1, 0, 0, 0, 499, 500, 1, 0, 0, 0, 500, 501, 1, 0, 0, 0, 501, 502, 3, 68, 32, 0, 502, 512, 1, 0, 0, 0, 503, 505, 3, 88, 42, 0, 504, 506, 3, 60, 28, 0, 505, 504, 1, 0, 0, 0, 506, 507, 1, 0, 0, 0, 507, 505, 1, 0, 0, 0, 507, 508, 1, 0, 0, 0, 508, 509, 1, 0, 0, 0, 509, 510, 3, 68, 32, 0, 510, 512, 1, 0, 0, 0, 511, 470, 1, 0, 0, 0, 511, 481, 1, 0, 0, 0, 511, 488, 1, 0, 0, 0, 511, 503, 1, 0, 0, 0, 512, 75, 1, 0, 0, 0, 513, 514, 5, 98, 0, 0, 514, 515, 5, 121, 0, 0, 515, 77, 1, 0, 0, 0, 516, 517, 5, 97, 0, 0, 517, 518, 5, 110, 0, 0, 518, 519, 5, 100, 0, 0, 519, 79, 1, 0, 0, 0, 520, 521, 5, 97, 0, 0, 521, 522, 5, 115, 0, 0, 522, 523, 5, 99, 0, 0, 523, 81, 1, 0, 0, 0, 524, 525, 5, 61, 0, 0, 525, 83, 1, 0, 0, 0, 526, 527, 5, 44, 0, 0, 527, 85, 1, 0, 0, 0, 528, 529, 5, 100, 0, 0, 529, 530, 5, 101, 0, 0, 530, 531, 5, 115, 0, 0, 531, 532, 5, 99, 0, 0, 532, 87, 1, 0, 0, 0, 533, 534, 5, 46, 0, 0, 534, 89, 1, 0, 0, 0, 535, 536, 5, 102, 0, 0, 536, 537, 5, 97, 0, 0, 537, 538, 5, 108, 0, 0, 538, 539, 5, 115, 0, 0, 539, 540, 5, 101, 0, 0, 540, 91, 1, 0, 0, 0, 541, 542, 5, 102, 0, 0, 542, 543, 5, 105, 0, 0, 543, 544, 5, 114, 0, 0, 544, 545, 5, 115, 0, 0, 545, 546, 5, 116, 0, 0, 546, 93, 1, 0, 0, 0, 547, 548, 5, 108, 0, 0, 548, 549, 5, 97, 0, 0, 549, 550, 5, 115, 0, 0, 550, 551, 5, 116, 0, 0, 551, 95, 1, 0, 0, 0, 552, 553, 5, 40, 0, 0, 553, 97, 1, 0, 0, 0, 554, 555, 5, 105, 0, 0, 555, 556, 5, 110, 0, 0, 556, 99, 1, 0, 0, 0, 557, 558, 5, 105, 0, 0, 558, 559, 5, 115, 0, 0, 559, 101, 1, 0, 0, 0, 560, 561, 5, 108, 0, 0, 561, 562, 5, 105, 0, 0, 562, 563, 5, 107, 0, 0, 563, 564, 5, 101, 0, 0, 564, 103, 1, 0, 0, 0, 565, 566, 5, 110, 0, 0, 566, 567, 5, 111, 0, 0, 567, 568, 5, 116, 0, 0, 568, 105, 1, 0, 0, 0, 569, 570, 5, 110, 0, 0, 570, 571, 5, 117, 0, 0, 571, 572, 5, 108, 0, 0, 572, 573, 5, 108, 0, 0, 573, 107, 1, 0, 0, 0, 574, 575, 5, 110, 0, 0, 575, 576, 5, 117, 0, 0, 576, 577, 5, 108, 0, 0, 577, 578, 5, 108, 0, 0, 578, 579, 5, 115, 0, 0, 579, 109, 1, 0, 0, 0, 580, 581, 5, 111, 0, 0, 581, 582, 5, 114, 0, 0, 582, 111, 1, 0, 0, 0, 583, 584, 5, 63, 0, 0, 584, 113, 1, 0, 0, 0, 585, 586, 5, 114, 0, 0, 586, 587, 5, 108, 0, 0, 587, 588, 5, 105, 0, 0, 588, 589, 5, 107, 0, 0, 589, 590, 5, 101, 0, 0, 590, 115, 1, 0, 0, 0, 591, 592, 5, 41, 0, 0, 592, 117, 1, 0, 0, 0, 593, 594, 5, 116, 0, 0, 594, 595, 5, 114, 0, 0, 595, 596, 5, 117, 0, 0, 596, 597, 5, 101, 0, 0, 597, 119, 1, 0, 0, 0, 598, 599, 5, 105, 0, 0, 599, 600, 5, 110, 0, 0, 600, 601, 5, 102, 0, 0, 601, 602, 5, 111, 0, 0, 602, 121, 1, 0, 0, 0, 603, 604, 5, 102, 0, 0, 604, 605, 5, 117, 0, 0, 605, 606, 5, 110, 0, 0, 606, 607, 5, 99, 0, 0, 607, 608, 5, 116, 0, 0, 608, 609, 5, 105, 0, 0, 609, 610, 5, 111, 0, 0, 610, 611, 5, 110, 0, 0, 611, 612, 5, 115, 0, 0, 612, 123, 1, 0, 0, 0, 613, 614, 5, 61, 0, 0, 614, 615, 5, 61, 0, 0, 615, 125, 1, 0, 0, 0, 616, 617, 5, 33, 0, 0, 617, 618, 5, 61, 0, 0, 618, 127, 1, 0, 0, 0, 619, 620, 5, 60, 0, 0, 620, 129, 1, 0, 0, 0, 621, 622, 5, 60, 0, 0, 622, 623, 5, 61, 0, 0, 623, 131, 1, 0, 0, 0, 624, 625, 5, 62, 0, 0, 625, 133, 1, 0, 0, 0, 626, 627, 5, 62, 0, 0, 627, 628, 5, 61, 0, 0, 628, 135, 1, 0, 0, 0, 629, 630, 5, 43, 0, 0, 630, 137, 1, 0, 0, 0, 631, 632, 5, 45, 0, 0, 632, 139, 1, 0, 0, 0, 633, 634, 5, 42, 0, 0, 634, 141, 1, 0, 0, 0, 635, 636, 5, 47, 0, 0, 636, 143, 1, 0, 0, 0, 637, 638, 5, 37, 0, 0, 638, 145, 1, 0, 0, 0, 639, 640, 5, 91, 0, 0, 640, 641, 1, 0, 0, 0, 641, 642, 6, 71, 0, 0, 642, 643, 6, 71, 0, 0, 643, 147, 1, 0, 0, 0, 644, 645, 5, 93, 0, 0, 645, 646, 1, 0, 0, 0, 646, 647, 6, 72, 7, 0, 647, 648, 6, 72, 7, 0, 648, 149, 1, 0, 0, 0, 649, 655, 3, 62, 29, 0, 650, 654, 3, 62, 29, 0, 651, 654, 3, 60, 28, 0, 652, 654, 5, 95, 0, 0, 653, 650, 1, 0, 0, 0, 653, 651, 1, 0, 0, 0, 653, 652, 1, 0, 0, 0, 654, 657, 1, 0, 0, 0, 655, 653, 1, 0, 0, 0, 655, 656, 1, 0, 0, 0, 656, 667, 1, 0, 0, 0, 657, 655, 1, 0, 0, 0, 658, 662, 7, 9, 0, 0, 659, 663, 3, 62, 29, 0, 660, 663, 3, 60, 28, 0, 661, 663, 5, 95, 0, 0, 662, 659, 1, 0, 0, 0, 662, 660, 1, 0, 0, 0, 662, 661, 1, 0, 0, 0, 663, 664, 1, 0, 0, 0, 664, 662, 1, 0, 0, 0, 664, 665, 1, 0, 0, 0, 665, 667, 1, 0, 0, 0, 666, 649, 1, 0, 0, 0, 666, 658, 1, 0, 0, 0, 667, 151, 1, 0, 0, 0, 668, 674, 5, 96, 0, 0, 669, 673, 8, 10, 0, 0, 670, 671, 5, 96, 0, 0, 671, 673, 5, 96, 0, 0, 672, 669, 1, 0, 0, 0, 672, 670, 1, 0, 0, 0, 673, 676, 1, 0, 0, 0, 674, 672, 1, 0, 0, 0, 674, 675, 1, 0, 0, 0, 675, 677, 1, 0, 0, 0, 676, 674, 1, 0, 0, 0, 677, 678, 5, 96, 0, 0, 678, 153, 1, 0, 0, 0, 679, 680, 3, 42, 19, 0, 680, 681, 1, 0, 0, 0, 681, 682, 6, 75, 3, 0, 682, 155, 1, 0, 0, 0, 683, 684, 3, 44, 20, 0, 684, 685, 1, 0, 0, 0, 685, 686, 6, 76, 3, 0, 686, 157, 1, 0, 0, 0, 687, 688, 3, 46, 21, 0, 688, 689, 1, 0, 0, 0, 689, 690, 6, 77, 3, 0, 690, 159, 1, 0, 0, 0, 691, 692, 5, 124, 0, 0, 692, 693, 1, 0, 0, 0, 693, 694, 6, 78, 6, 0, 694, 695, 6, 78, 7, 0, 695, 161, 1, 0, 0, 0, 696, 697, 5, 91, 0, 0, 697, 698, 1, 0, 0, 0, 698, 699, 6, 79, 4, 0, 699, 700, 6, 79, 1, 0, 700, 701, 6, 79, 1, 0, 701, 163, 1, 0, 0, 0, 702, 703, 5, 93, 0, 0, 703, 704, 1, 0, 0, 0, 704, 705, 6, 80, 7, 0, 705, 706, 6, 80, 7, 0, 706, 707, 6, 80, 8, 0, 707, 165, 1, 0, 0, 0, 708, 709, 5, 44, 0, 0, 709, 710, 1, 0, 0, 0, 710, 711, 6, 81, 9, 0, 711, 167, 1, 0, 0, 0, 712, 713, 5, 61, 0, 0, 713, 714, 1, 0, 0, 0, 714, 715, 6, 82, 10, 0, 715, 169, 1, 0, 0, 0, 716, 717, 5, 97, 0, 0, 717, 718, 5, 115, 0, 0, 718, 171, 1, 0, 0, 0, 719, 720, 5, 109, 0, 0, 720, 721, 5, 101, 0, 0, 721, 722, 5, 116, 0, 0, 722, 723, 5, 97, 0, 0, 723, 724, 5, 100, 0, 0, 724, 725, 5, 97, 0, 0, 725, 726, 5, 116, 0, 0, 726, 727, 5, 97, 0, 0, 727, 173, 1, 0, 0, 0, 728, 729, 5, 111, 0, 0, 729, 730, 5, 110, 0, 0, 730, 175, 1, 0, 0, 0, 731, 732, 5, 119, 0, 0, 732, 733, 5, 105, 0, 0, 733, 734, 5, 116, 0, 0, 734, 735, 5, 104, 0, 0, 735, 177, 1, 0, 0, 0, 736, 738, 3, 180, 88, 0, 737, 736, 1, 0, 0, 0, 738, 739, 1, 0, 0, 0, 739, 737, 1, 0, 0, 0, 739, 740, 1, 0, 0, 0, 740, 179, 1, 0, 0, 0, 741, 743, 8, 11, 0, 0, 742, 741, 1, 0, 0, 0, 743, 744, 1, 0, 0, 0, 744, 742, 1, 0, 0, 0, 744, 745, 1, 0, 0, 0, 745, 749, 1, 0, 0, 0, 746, 747, 5, 47, 0, 0, 747, 749, 8, 12, 0, 0, 748, 742, 1, 0, 0, 0, 748, 746, 1, 0, 0, 0, 749, 181, 1, 0, 0, 0, 750, 751, 3, 152, 74, 0, 751, 183, 1, 0, 0, 0, 752, 753, 3, 42, 19, 0, 753, 754, 1, 0, 0, 0, 754, 755, 6, 90, 3, 0, 755, 185, 1, 0, 0, 0, 756, 757, 3, 44, 20, 0, 757, 758, 1, 0, 0, 0, 758, 759, 6, 91, 3, 0, 759, 187, 1, 0, 0, 0, 760, 761, 3, 46, 21, 0, 761, 762, 1, 0, 0, 0, 762, 763, 6, 92, 3, 0, 763, 189, 1, 0, 0, 0, 38, 0, 1, 2, 3, 346, 356, 360, 363, 372, 374, 385, 426, 431, 436, 438, 449, 457, 460, 462, 467, 472, 478, 485, 490, 496, 499, 507, 511, 653, 655, 662, 664, 666, 672, 674, 739, 744, 748, 11, 5, 2, 0, 5, 3, 0, 5, 1, 0, 0, 1, 0, 7, 65, 0, 5, 0, 0, 7, 26, 0, 4, 0, 0, 7, 66, 0, 7, 34, 0, 7, 33, 0] \ No newline at end of file +[4, 0, 98, 1090, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 4, 18, 451, 8, 18, 11, 18, 12, 18, 452, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 5, 19, 461, 8, 19, 10, 19, 12, 19, 464, 9, 19, 1, 19, 3, 19, 467, 8, 19, 1, 19, 3, 19, 470, 8, 19, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 5, 20, 479, 8, 20, 10, 20, 12, 20, 482, 9, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 21, 4, 21, 490, 8, 21, 11, 21, 12, 21, 491, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 27, 1, 28, 1, 28, 1, 29, 1, 29, 1, 30, 1, 30, 1, 30, 1, 31, 1, 31, 1, 32, 1, 32, 3, 32, 533, 8, 32, 1, 32, 4, 32, 536, 8, 32, 11, 32, 12, 32, 537, 1, 33, 1, 33, 1, 34, 1, 34, 1, 35, 1, 35, 1, 35, 3, 35, 547, 8, 35, 1, 36, 1, 36, 1, 37, 1, 37, 1, 37, 3, 37, 554, 8, 37, 1, 38, 1, 38, 1, 38, 5, 38, 559, 8, 38, 10, 38, 12, 38, 562, 9, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 570, 8, 38, 10, 38, 12, 38, 573, 9, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 3, 38, 580, 8, 38, 1, 38, 3, 38, 583, 8, 38, 3, 38, 585, 8, 38, 1, 39, 4, 39, 588, 8, 39, 11, 39, 12, 39, 589, 1, 40, 4, 40, 593, 8, 40, 11, 40, 12, 40, 594, 1, 40, 1, 40, 5, 40, 599, 8, 40, 10, 40, 12, 40, 602, 9, 40, 1, 40, 1, 40, 4, 40, 606, 8, 40, 11, 40, 12, 40, 607, 1, 40, 4, 40, 611, 8, 40, 11, 40, 12, 40, 612, 1, 40, 1, 40, 5, 40, 617, 8, 40, 10, 40, 12, 40, 620, 9, 40, 3, 40, 622, 8, 40, 1, 40, 1, 40, 1, 40, 1, 40, 4, 40, 628, 8, 40, 11, 40, 12, 40, 629, 1, 40, 1, 40, 3, 40, 634, 8, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 67, 1, 67, 1, 68, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 73, 1, 73, 1, 74, 1, 74, 1, 74, 1, 74, 1, 74, 1, 75, 1, 75, 1, 75, 1, 75, 1, 75, 1, 76, 1, 76, 5, 76, 759, 8, 76, 10, 76, 12, 76, 762, 9, 76, 1, 76, 1, 76, 3, 76, 766, 8, 76, 1, 76, 4, 76, 769, 8, 76, 11, 76, 12, 76, 770, 3, 76, 773, 8, 76, 1, 77, 1, 77, 4, 77, 777, 8, 77, 11, 77, 12, 77, 778, 1, 77, 1, 77, 1, 78, 1, 78, 1, 78, 1, 78, 1, 79, 1, 79, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 86, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 3, 87, 832, 8, 87, 1, 88, 4, 88, 835, 8, 88, 11, 88, 12, 88, 836, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 1, 95, 1, 96, 1, 96, 1, 96, 1, 96, 3, 96, 872, 8, 96, 1, 97, 1, 97, 3, 97, 876, 8, 97, 1, 97, 5, 97, 879, 8, 97, 10, 97, 12, 97, 882, 9, 97, 1, 97, 1, 97, 3, 97, 886, 8, 97, 1, 97, 4, 97, 889, 8, 97, 11, 97, 12, 97, 890, 3, 97, 893, 8, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 107, 1, 107, 1, 107, 1, 107, 1, 108, 1, 108, 1, 108, 1, 108, 1, 109, 1, 109, 1, 109, 1, 109, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 128, 1, 128, 1, 129, 1, 129, 1, 129, 1, 129, 1, 130, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 139, 1, 139, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 2, 480, 571, 0, 143, 10, 1, 12, 2, 14, 3, 16, 4, 18, 5, 20, 6, 22, 7, 24, 8, 26, 9, 28, 10, 30, 11, 32, 12, 34, 13, 36, 14, 38, 15, 40, 16, 42, 17, 44, 18, 46, 19, 48, 20, 50, 21, 52, 22, 54, 0, 56, 0, 58, 23, 60, 24, 62, 25, 64, 26, 66, 0, 68, 0, 70, 0, 72, 0, 74, 0, 76, 0, 78, 0, 80, 0, 82, 0, 84, 0, 86, 27, 88, 28, 90, 29, 92, 30, 94, 31, 96, 32, 98, 33, 100, 34, 102, 35, 104, 36, 106, 37, 108, 38, 110, 39, 112, 40, 114, 41, 116, 42, 118, 43, 120, 44, 122, 45, 124, 46, 126, 47, 128, 48, 130, 49, 132, 50, 134, 51, 136, 52, 138, 53, 140, 54, 142, 55, 144, 56, 146, 57, 148, 58, 150, 59, 152, 60, 154, 61, 156, 62, 158, 63, 160, 64, 162, 65, 164, 66, 166, 67, 168, 68, 170, 69, 172, 0, 174, 0, 176, 0, 178, 0, 180, 0, 182, 70, 184, 0, 186, 71, 188, 0, 190, 72, 192, 73, 194, 74, 196, 0, 198, 0, 200, 0, 202, 0, 204, 75, 206, 0, 208, 76, 210, 77, 212, 78, 214, 0, 216, 0, 218, 0, 220, 0, 222, 79, 224, 0, 226, 0, 228, 80, 230, 81, 232, 82, 234, 0, 236, 83, 238, 84, 240, 0, 242, 0, 244, 85, 246, 86, 248, 87, 250, 0, 252, 0, 254, 0, 256, 0, 258, 0, 260, 0, 262, 0, 264, 88, 266, 89, 268, 90, 270, 0, 272, 0, 274, 0, 276, 0, 278, 91, 280, 92, 282, 93, 284, 0, 286, 94, 288, 95, 290, 96, 292, 97, 294, 98, 10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 5, 0, 34, 34, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 69, 69, 101, 101, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 10, 0, 9, 10, 13, 13, 32, 32, 44, 44, 47, 47, 61, 61, 91, 91, 93, 93, 96, 96, 124, 124, 2, 0, 42, 42, 47, 47, 1112, 0, 10, 1, 0, 0, 0, 0, 12, 1, 0, 0, 0, 0, 14, 1, 0, 0, 0, 0, 16, 1, 0, 0, 0, 0, 18, 1, 0, 0, 0, 0, 20, 1, 0, 0, 0, 0, 22, 1, 0, 0, 0, 0, 24, 1, 0, 0, 0, 0, 26, 1, 0, 0, 0, 0, 28, 1, 0, 0, 0, 0, 30, 1, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 34, 1, 0, 0, 0, 0, 36, 1, 0, 0, 0, 0, 38, 1, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 42, 1, 0, 0, 0, 0, 44, 1, 0, 0, 0, 0, 46, 1, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 50, 1, 0, 0, 0, 0, 52, 1, 0, 0, 0, 1, 54, 1, 0, 0, 0, 1, 56, 1, 0, 0, 0, 1, 58, 1, 0, 0, 0, 1, 60, 1, 0, 0, 0, 1, 62, 1, 0, 0, 0, 2, 64, 1, 0, 0, 0, 2, 86, 1, 0, 0, 0, 2, 88, 1, 0, 0, 0, 2, 90, 1, 0, 0, 0, 2, 92, 1, 0, 0, 0, 2, 94, 1, 0, 0, 0, 2, 96, 1, 0, 0, 0, 2, 98, 1, 0, 0, 0, 2, 100, 1, 0, 0, 0, 2, 102, 1, 0, 0, 0, 2, 104, 1, 0, 0, 0, 2, 106, 1, 0, 0, 0, 2, 108, 1, 0, 0, 0, 2, 110, 1, 0, 0, 0, 2, 112, 1, 0, 0, 0, 2, 114, 1, 0, 0, 0, 2, 116, 1, 0, 0, 0, 2, 118, 1, 0, 0, 0, 2, 120, 1, 0, 0, 0, 2, 122, 1, 0, 0, 0, 2, 124, 1, 0, 0, 0, 2, 126, 1, 0, 0, 0, 2, 128, 1, 0, 0, 0, 2, 130, 1, 0, 0, 0, 2, 132, 1, 0, 0, 0, 2, 134, 1, 0, 0, 0, 2, 136, 1, 0, 0, 0, 2, 138, 1, 0, 0, 0, 2, 140, 1, 0, 0, 0, 2, 142, 1, 0, 0, 0, 2, 144, 1, 0, 0, 0, 2, 146, 1, 0, 0, 0, 2, 148, 1, 0, 0, 0, 2, 150, 1, 0, 0, 0, 2, 152, 1, 0, 0, 0, 2, 154, 1, 0, 0, 0, 2, 156, 1, 0, 0, 0, 2, 158, 1, 0, 0, 0, 2, 160, 1, 0, 0, 0, 2, 162, 1, 0, 0, 0, 2, 164, 1, 0, 0, 0, 2, 166, 1, 0, 0, 0, 2, 168, 1, 0, 0, 0, 2, 170, 1, 0, 0, 0, 3, 172, 1, 0, 0, 0, 3, 174, 1, 0, 0, 0, 3, 176, 1, 0, 0, 0, 3, 178, 1, 0, 0, 0, 3, 180, 1, 0, 0, 0, 3, 182, 1, 0, 0, 0, 3, 186, 1, 0, 0, 0, 3, 188, 1, 0, 0, 0, 3, 190, 1, 0, 0, 0, 3, 192, 1, 0, 0, 0, 3, 194, 1, 0, 0, 0, 4, 196, 1, 0, 0, 0, 4, 198, 1, 0, 0, 0, 4, 200, 1, 0, 0, 0, 4, 204, 1, 0, 0, 0, 4, 206, 1, 0, 0, 0, 4, 208, 1, 0, 0, 0, 4, 210, 1, 0, 0, 0, 4, 212, 1, 0, 0, 0, 5, 214, 1, 0, 0, 0, 5, 216, 1, 0, 0, 0, 5, 218, 1, 0, 0, 0, 5, 220, 1, 0, 0, 0, 5, 222, 1, 0, 0, 0, 5, 224, 1, 0, 0, 0, 5, 226, 1, 0, 0, 0, 5, 228, 1, 0, 0, 0, 5, 230, 1, 0, 0, 0, 5, 232, 1, 0, 0, 0, 6, 234, 1, 0, 0, 0, 6, 236, 1, 0, 0, 0, 6, 238, 1, 0, 0, 0, 6, 240, 1, 0, 0, 0, 6, 242, 1, 0, 0, 0, 6, 244, 1, 0, 0, 0, 6, 246, 1, 0, 0, 0, 6, 248, 1, 0, 0, 0, 7, 250, 1, 0, 0, 0, 7, 252, 1, 0, 0, 0, 7, 254, 1, 0, 0, 0, 7, 256, 1, 0, 0, 0, 7, 258, 1, 0, 0, 0, 7, 260, 1, 0, 0, 0, 7, 262, 1, 0, 0, 0, 7, 264, 1, 0, 0, 0, 7, 266, 1, 0, 0, 0, 7, 268, 1, 0, 0, 0, 8, 270, 1, 0, 0, 0, 8, 272, 1, 0, 0, 0, 8, 274, 1, 0, 0, 0, 8, 276, 1, 0, 0, 0, 8, 278, 1, 0, 0, 0, 8, 280, 1, 0, 0, 0, 8, 282, 1, 0, 0, 0, 9, 284, 1, 0, 0, 0, 9, 286, 1, 0, 0, 0, 9, 288, 1, 0, 0, 0, 9, 290, 1, 0, 0, 0, 9, 292, 1, 0, 0, 0, 9, 294, 1, 0, 0, 0, 10, 296, 1, 0, 0, 0, 12, 306, 1, 0, 0, 0, 14, 313, 1, 0, 0, 0, 16, 322, 1, 0, 0, 0, 18, 329, 1, 0, 0, 0, 20, 339, 1, 0, 0, 0, 22, 346, 1, 0, 0, 0, 24, 353, 1, 0, 0, 0, 26, 367, 1, 0, 0, 0, 28, 374, 1, 0, 0, 0, 30, 382, 1, 0, 0, 0, 32, 394, 1, 0, 0, 0, 34, 404, 1, 0, 0, 0, 36, 413, 1, 0, 0, 0, 38, 419, 1, 0, 0, 0, 40, 426, 1, 0, 0, 0, 42, 433, 1, 0, 0, 0, 44, 441, 1, 0, 0, 0, 46, 450, 1, 0, 0, 0, 48, 456, 1, 0, 0, 0, 50, 473, 1, 0, 0, 0, 52, 489, 1, 0, 0, 0, 54, 495, 1, 0, 0, 0, 56, 500, 1, 0, 0, 0, 58, 505, 1, 0, 0, 0, 60, 509, 1, 0, 0, 0, 62, 513, 1, 0, 0, 0, 64, 517, 1, 0, 0, 0, 66, 521, 1, 0, 0, 0, 68, 523, 1, 0, 0, 0, 70, 525, 1, 0, 0, 0, 72, 528, 1, 0, 0, 0, 74, 530, 1, 0, 0, 0, 76, 539, 1, 0, 0, 0, 78, 541, 1, 0, 0, 0, 80, 546, 1, 0, 0, 0, 82, 548, 1, 0, 0, 0, 84, 553, 1, 0, 0, 0, 86, 584, 1, 0, 0, 0, 88, 587, 1, 0, 0, 0, 90, 633, 1, 0, 0, 0, 92, 635, 1, 0, 0, 0, 94, 638, 1, 0, 0, 0, 96, 642, 1, 0, 0, 0, 98, 646, 1, 0, 0, 0, 100, 648, 1, 0, 0, 0, 102, 650, 1, 0, 0, 0, 104, 655, 1, 0, 0, 0, 106, 657, 1, 0, 0, 0, 108, 663, 1, 0, 0, 0, 110, 669, 1, 0, 0, 0, 112, 674, 1, 0, 0, 0, 114, 676, 1, 0, 0, 0, 116, 679, 1, 0, 0, 0, 118, 682, 1, 0, 0, 0, 120, 687, 1, 0, 0, 0, 122, 691, 1, 0, 0, 0, 124, 696, 1, 0, 0, 0, 126, 702, 1, 0, 0, 0, 128, 705, 1, 0, 0, 0, 130, 707, 1, 0, 0, 0, 132, 713, 1, 0, 0, 0, 134, 715, 1, 0, 0, 0, 136, 720, 1, 0, 0, 0, 138, 723, 1, 0, 0, 0, 140, 726, 1, 0, 0, 0, 142, 728, 1, 0, 0, 0, 144, 731, 1, 0, 0, 0, 146, 733, 1, 0, 0, 0, 148, 736, 1, 0, 0, 0, 150, 738, 1, 0, 0, 0, 152, 740, 1, 0, 0, 0, 154, 742, 1, 0, 0, 0, 156, 744, 1, 0, 0, 0, 158, 746, 1, 0, 0, 0, 160, 751, 1, 0, 0, 0, 162, 772, 1, 0, 0, 0, 164, 774, 1, 0, 0, 0, 166, 782, 1, 0, 0, 0, 168, 786, 1, 0, 0, 0, 170, 790, 1, 0, 0, 0, 172, 794, 1, 0, 0, 0, 174, 799, 1, 0, 0, 0, 176, 805, 1, 0, 0, 0, 178, 811, 1, 0, 0, 0, 180, 815, 1, 0, 0, 0, 182, 819, 1, 0, 0, 0, 184, 831, 1, 0, 0, 0, 186, 834, 1, 0, 0, 0, 188, 838, 1, 0, 0, 0, 190, 842, 1, 0, 0, 0, 192, 846, 1, 0, 0, 0, 194, 850, 1, 0, 0, 0, 196, 854, 1, 0, 0, 0, 198, 859, 1, 0, 0, 0, 200, 863, 1, 0, 0, 0, 202, 871, 1, 0, 0, 0, 204, 892, 1, 0, 0, 0, 206, 894, 1, 0, 0, 0, 208, 898, 1, 0, 0, 0, 210, 902, 1, 0, 0, 0, 212, 906, 1, 0, 0, 0, 214, 910, 1, 0, 0, 0, 216, 915, 1, 0, 0, 0, 218, 919, 1, 0, 0, 0, 220, 923, 1, 0, 0, 0, 222, 927, 1, 0, 0, 0, 224, 930, 1, 0, 0, 0, 226, 934, 1, 0, 0, 0, 228, 938, 1, 0, 0, 0, 230, 942, 1, 0, 0, 0, 232, 946, 1, 0, 0, 0, 234, 950, 1, 0, 0, 0, 236, 955, 1, 0, 0, 0, 238, 960, 1, 0, 0, 0, 240, 967, 1, 0, 0, 0, 242, 971, 1, 0, 0, 0, 244, 975, 1, 0, 0, 0, 246, 979, 1, 0, 0, 0, 248, 983, 1, 0, 0, 0, 250, 987, 1, 0, 0, 0, 252, 993, 1, 0, 0, 0, 254, 997, 1, 0, 0, 0, 256, 1001, 1, 0, 0, 0, 258, 1005, 1, 0, 0, 0, 260, 1009, 1, 0, 0, 0, 262, 1013, 1, 0, 0, 0, 264, 1017, 1, 0, 0, 0, 266, 1021, 1, 0, 0, 0, 268, 1025, 1, 0, 0, 0, 270, 1029, 1, 0, 0, 0, 272, 1034, 1, 0, 0, 0, 274, 1038, 1, 0, 0, 0, 276, 1042, 1, 0, 0, 0, 278, 1046, 1, 0, 0, 0, 280, 1050, 1, 0, 0, 0, 282, 1054, 1, 0, 0, 0, 284, 1058, 1, 0, 0, 0, 286, 1063, 1, 0, 0, 0, 288, 1068, 1, 0, 0, 0, 290, 1078, 1, 0, 0, 0, 292, 1082, 1, 0, 0, 0, 294, 1086, 1, 0, 0, 0, 296, 297, 5, 100, 0, 0, 297, 298, 5, 105, 0, 0, 298, 299, 5, 115, 0, 0, 299, 300, 5, 115, 0, 0, 300, 301, 5, 101, 0, 0, 301, 302, 5, 99, 0, 0, 302, 303, 5, 116, 0, 0, 303, 304, 1, 0, 0, 0, 304, 305, 6, 0, 0, 0, 305, 11, 1, 0, 0, 0, 306, 307, 5, 100, 0, 0, 307, 308, 5, 114, 0, 0, 308, 309, 5, 111, 0, 0, 309, 310, 5, 112, 0, 0, 310, 311, 1, 0, 0, 0, 311, 312, 6, 1, 1, 0, 312, 13, 1, 0, 0, 0, 313, 314, 5, 101, 0, 0, 314, 315, 5, 110, 0, 0, 315, 316, 5, 114, 0, 0, 316, 317, 5, 105, 0, 0, 317, 318, 5, 99, 0, 0, 318, 319, 5, 104, 0, 0, 319, 320, 1, 0, 0, 0, 320, 321, 6, 2, 2, 0, 321, 15, 1, 0, 0, 0, 322, 323, 5, 101, 0, 0, 323, 324, 5, 118, 0, 0, 324, 325, 5, 97, 0, 0, 325, 326, 5, 108, 0, 0, 326, 327, 1, 0, 0, 0, 327, 328, 6, 3, 0, 0, 328, 17, 1, 0, 0, 0, 329, 330, 5, 101, 0, 0, 330, 331, 5, 120, 0, 0, 331, 332, 5, 112, 0, 0, 332, 333, 5, 108, 0, 0, 333, 334, 5, 97, 0, 0, 334, 335, 5, 105, 0, 0, 335, 336, 5, 110, 0, 0, 336, 337, 1, 0, 0, 0, 337, 338, 6, 4, 3, 0, 338, 19, 1, 0, 0, 0, 339, 340, 5, 102, 0, 0, 340, 341, 5, 114, 0, 0, 341, 342, 5, 111, 0, 0, 342, 343, 5, 109, 0, 0, 343, 344, 1, 0, 0, 0, 344, 345, 6, 5, 4, 0, 345, 21, 1, 0, 0, 0, 346, 347, 5, 103, 0, 0, 347, 348, 5, 114, 0, 0, 348, 349, 5, 111, 0, 0, 349, 350, 5, 107, 0, 0, 350, 351, 1, 0, 0, 0, 351, 352, 6, 6, 0, 0, 352, 23, 1, 0, 0, 0, 353, 354, 5, 105, 0, 0, 354, 355, 5, 110, 0, 0, 355, 356, 5, 108, 0, 0, 356, 357, 5, 105, 0, 0, 357, 358, 5, 110, 0, 0, 358, 359, 5, 101, 0, 0, 359, 360, 5, 115, 0, 0, 360, 361, 5, 116, 0, 0, 361, 362, 5, 97, 0, 0, 362, 363, 5, 116, 0, 0, 363, 364, 5, 115, 0, 0, 364, 365, 1, 0, 0, 0, 365, 366, 6, 7, 0, 0, 366, 25, 1, 0, 0, 0, 367, 368, 5, 107, 0, 0, 368, 369, 5, 101, 0, 0, 369, 370, 5, 101, 0, 0, 370, 371, 5, 112, 0, 0, 371, 372, 1, 0, 0, 0, 372, 373, 6, 8, 1, 0, 373, 27, 1, 0, 0, 0, 374, 375, 5, 108, 0, 0, 375, 376, 5, 105, 0, 0, 376, 377, 5, 109, 0, 0, 377, 378, 5, 105, 0, 0, 378, 379, 5, 116, 0, 0, 379, 380, 1, 0, 0, 0, 380, 381, 6, 9, 0, 0, 381, 29, 1, 0, 0, 0, 382, 383, 5, 109, 0, 0, 383, 384, 5, 118, 0, 0, 384, 385, 5, 95, 0, 0, 385, 386, 5, 101, 0, 0, 386, 387, 5, 120, 0, 0, 387, 388, 5, 112, 0, 0, 388, 389, 5, 97, 0, 0, 389, 390, 5, 110, 0, 0, 390, 391, 5, 100, 0, 0, 391, 392, 1, 0, 0, 0, 392, 393, 6, 10, 5, 0, 393, 31, 1, 0, 0, 0, 394, 395, 5, 112, 0, 0, 395, 396, 5, 114, 0, 0, 396, 397, 5, 111, 0, 0, 397, 398, 5, 106, 0, 0, 398, 399, 5, 101, 0, 0, 399, 400, 5, 99, 0, 0, 400, 401, 5, 116, 0, 0, 401, 402, 1, 0, 0, 0, 402, 403, 6, 11, 1, 0, 403, 33, 1, 0, 0, 0, 404, 405, 5, 114, 0, 0, 405, 406, 5, 101, 0, 0, 406, 407, 5, 110, 0, 0, 407, 408, 5, 97, 0, 0, 408, 409, 5, 109, 0, 0, 409, 410, 5, 101, 0, 0, 410, 411, 1, 0, 0, 0, 411, 412, 6, 12, 6, 0, 412, 35, 1, 0, 0, 0, 413, 414, 5, 114, 0, 0, 414, 415, 5, 111, 0, 0, 415, 416, 5, 119, 0, 0, 416, 417, 1, 0, 0, 0, 417, 418, 6, 13, 0, 0, 418, 37, 1, 0, 0, 0, 419, 420, 5, 115, 0, 0, 420, 421, 5, 104, 0, 0, 421, 422, 5, 111, 0, 0, 422, 423, 5, 119, 0, 0, 423, 424, 1, 0, 0, 0, 424, 425, 6, 14, 7, 0, 425, 39, 1, 0, 0, 0, 426, 427, 5, 115, 0, 0, 427, 428, 5, 111, 0, 0, 428, 429, 5, 114, 0, 0, 429, 430, 5, 116, 0, 0, 430, 431, 1, 0, 0, 0, 431, 432, 6, 15, 0, 0, 432, 41, 1, 0, 0, 0, 433, 434, 5, 115, 0, 0, 434, 435, 5, 116, 0, 0, 435, 436, 5, 97, 0, 0, 436, 437, 5, 116, 0, 0, 437, 438, 5, 115, 0, 0, 438, 439, 1, 0, 0, 0, 439, 440, 6, 16, 0, 0, 440, 43, 1, 0, 0, 0, 441, 442, 5, 119, 0, 0, 442, 443, 5, 104, 0, 0, 443, 444, 5, 101, 0, 0, 444, 445, 5, 114, 0, 0, 445, 446, 5, 101, 0, 0, 446, 447, 1, 0, 0, 0, 447, 448, 6, 17, 0, 0, 448, 45, 1, 0, 0, 0, 449, 451, 8, 0, 0, 0, 450, 449, 1, 0, 0, 0, 451, 452, 1, 0, 0, 0, 452, 450, 1, 0, 0, 0, 452, 453, 1, 0, 0, 0, 453, 454, 1, 0, 0, 0, 454, 455, 6, 18, 0, 0, 455, 47, 1, 0, 0, 0, 456, 457, 5, 47, 0, 0, 457, 458, 5, 47, 0, 0, 458, 462, 1, 0, 0, 0, 459, 461, 8, 1, 0, 0, 460, 459, 1, 0, 0, 0, 461, 464, 1, 0, 0, 0, 462, 460, 1, 0, 0, 0, 462, 463, 1, 0, 0, 0, 463, 466, 1, 0, 0, 0, 464, 462, 1, 0, 0, 0, 465, 467, 5, 13, 0, 0, 466, 465, 1, 0, 0, 0, 466, 467, 1, 0, 0, 0, 467, 469, 1, 0, 0, 0, 468, 470, 5, 10, 0, 0, 469, 468, 1, 0, 0, 0, 469, 470, 1, 0, 0, 0, 470, 471, 1, 0, 0, 0, 471, 472, 6, 19, 8, 0, 472, 49, 1, 0, 0, 0, 473, 474, 5, 47, 0, 0, 474, 475, 5, 42, 0, 0, 475, 480, 1, 0, 0, 0, 476, 479, 3, 50, 20, 0, 477, 479, 9, 0, 0, 0, 478, 476, 1, 0, 0, 0, 478, 477, 1, 0, 0, 0, 479, 482, 1, 0, 0, 0, 480, 481, 1, 0, 0, 0, 480, 478, 1, 0, 0, 0, 481, 483, 1, 0, 0, 0, 482, 480, 1, 0, 0, 0, 483, 484, 5, 42, 0, 0, 484, 485, 5, 47, 0, 0, 485, 486, 1, 0, 0, 0, 486, 487, 6, 20, 8, 0, 487, 51, 1, 0, 0, 0, 488, 490, 7, 2, 0, 0, 489, 488, 1, 0, 0, 0, 490, 491, 1, 0, 0, 0, 491, 489, 1, 0, 0, 0, 491, 492, 1, 0, 0, 0, 492, 493, 1, 0, 0, 0, 493, 494, 6, 21, 8, 0, 494, 53, 1, 0, 0, 0, 495, 496, 3, 158, 74, 0, 496, 497, 1, 0, 0, 0, 497, 498, 6, 22, 9, 0, 498, 499, 6, 22, 10, 0, 499, 55, 1, 0, 0, 0, 500, 501, 3, 64, 27, 0, 501, 502, 1, 0, 0, 0, 502, 503, 6, 23, 11, 0, 503, 504, 6, 23, 12, 0, 504, 57, 1, 0, 0, 0, 505, 506, 3, 52, 21, 0, 506, 507, 1, 0, 0, 0, 507, 508, 6, 24, 8, 0, 508, 59, 1, 0, 0, 0, 509, 510, 3, 48, 19, 0, 510, 511, 1, 0, 0, 0, 511, 512, 6, 25, 8, 0, 512, 61, 1, 0, 0, 0, 513, 514, 3, 50, 20, 0, 514, 515, 1, 0, 0, 0, 515, 516, 6, 26, 8, 0, 516, 63, 1, 0, 0, 0, 517, 518, 5, 124, 0, 0, 518, 519, 1, 0, 0, 0, 519, 520, 6, 27, 12, 0, 520, 65, 1, 0, 0, 0, 521, 522, 7, 3, 0, 0, 522, 67, 1, 0, 0, 0, 523, 524, 7, 4, 0, 0, 524, 69, 1, 0, 0, 0, 525, 526, 5, 92, 0, 0, 526, 527, 7, 5, 0, 0, 527, 71, 1, 0, 0, 0, 528, 529, 8, 6, 0, 0, 529, 73, 1, 0, 0, 0, 530, 532, 7, 7, 0, 0, 531, 533, 7, 8, 0, 0, 532, 531, 1, 0, 0, 0, 532, 533, 1, 0, 0, 0, 533, 535, 1, 0, 0, 0, 534, 536, 3, 66, 28, 0, 535, 534, 1, 0, 0, 0, 536, 537, 1, 0, 0, 0, 537, 535, 1, 0, 0, 0, 537, 538, 1, 0, 0, 0, 538, 75, 1, 0, 0, 0, 539, 540, 5, 64, 0, 0, 540, 77, 1, 0, 0, 0, 541, 542, 5, 96, 0, 0, 542, 79, 1, 0, 0, 0, 543, 547, 8, 9, 0, 0, 544, 545, 5, 96, 0, 0, 545, 547, 5, 96, 0, 0, 546, 543, 1, 0, 0, 0, 546, 544, 1, 0, 0, 0, 547, 81, 1, 0, 0, 0, 548, 549, 5, 95, 0, 0, 549, 83, 1, 0, 0, 0, 550, 554, 3, 68, 29, 0, 551, 554, 3, 66, 28, 0, 552, 554, 3, 82, 36, 0, 553, 550, 1, 0, 0, 0, 553, 551, 1, 0, 0, 0, 553, 552, 1, 0, 0, 0, 554, 85, 1, 0, 0, 0, 555, 560, 5, 34, 0, 0, 556, 559, 3, 70, 30, 0, 557, 559, 3, 72, 31, 0, 558, 556, 1, 0, 0, 0, 558, 557, 1, 0, 0, 0, 559, 562, 1, 0, 0, 0, 560, 558, 1, 0, 0, 0, 560, 561, 1, 0, 0, 0, 561, 563, 1, 0, 0, 0, 562, 560, 1, 0, 0, 0, 563, 585, 5, 34, 0, 0, 564, 565, 5, 34, 0, 0, 565, 566, 5, 34, 0, 0, 566, 567, 5, 34, 0, 0, 567, 571, 1, 0, 0, 0, 568, 570, 8, 1, 0, 0, 569, 568, 1, 0, 0, 0, 570, 573, 1, 0, 0, 0, 571, 572, 1, 0, 0, 0, 571, 569, 1, 0, 0, 0, 572, 574, 1, 0, 0, 0, 573, 571, 1, 0, 0, 0, 574, 575, 5, 34, 0, 0, 575, 576, 5, 34, 0, 0, 576, 577, 5, 34, 0, 0, 577, 579, 1, 0, 0, 0, 578, 580, 5, 34, 0, 0, 579, 578, 1, 0, 0, 0, 579, 580, 1, 0, 0, 0, 580, 582, 1, 0, 0, 0, 581, 583, 5, 34, 0, 0, 582, 581, 1, 0, 0, 0, 582, 583, 1, 0, 0, 0, 583, 585, 1, 0, 0, 0, 584, 555, 1, 0, 0, 0, 584, 564, 1, 0, 0, 0, 585, 87, 1, 0, 0, 0, 586, 588, 3, 66, 28, 0, 587, 586, 1, 0, 0, 0, 588, 589, 1, 0, 0, 0, 589, 587, 1, 0, 0, 0, 589, 590, 1, 0, 0, 0, 590, 89, 1, 0, 0, 0, 591, 593, 3, 66, 28, 0, 592, 591, 1, 0, 0, 0, 593, 594, 1, 0, 0, 0, 594, 592, 1, 0, 0, 0, 594, 595, 1, 0, 0, 0, 595, 596, 1, 0, 0, 0, 596, 600, 3, 104, 47, 0, 597, 599, 3, 66, 28, 0, 598, 597, 1, 0, 0, 0, 599, 602, 1, 0, 0, 0, 600, 598, 1, 0, 0, 0, 600, 601, 1, 0, 0, 0, 601, 634, 1, 0, 0, 0, 602, 600, 1, 0, 0, 0, 603, 605, 3, 104, 47, 0, 604, 606, 3, 66, 28, 0, 605, 604, 1, 0, 0, 0, 606, 607, 1, 0, 0, 0, 607, 605, 1, 0, 0, 0, 607, 608, 1, 0, 0, 0, 608, 634, 1, 0, 0, 0, 609, 611, 3, 66, 28, 0, 610, 609, 1, 0, 0, 0, 611, 612, 1, 0, 0, 0, 612, 610, 1, 0, 0, 0, 612, 613, 1, 0, 0, 0, 613, 621, 1, 0, 0, 0, 614, 618, 3, 104, 47, 0, 615, 617, 3, 66, 28, 0, 616, 615, 1, 0, 0, 0, 617, 620, 1, 0, 0, 0, 618, 616, 1, 0, 0, 0, 618, 619, 1, 0, 0, 0, 619, 622, 1, 0, 0, 0, 620, 618, 1, 0, 0, 0, 621, 614, 1, 0, 0, 0, 621, 622, 1, 0, 0, 0, 622, 623, 1, 0, 0, 0, 623, 624, 3, 74, 32, 0, 624, 634, 1, 0, 0, 0, 625, 627, 3, 104, 47, 0, 626, 628, 3, 66, 28, 0, 627, 626, 1, 0, 0, 0, 628, 629, 1, 0, 0, 0, 629, 627, 1, 0, 0, 0, 629, 630, 1, 0, 0, 0, 630, 631, 1, 0, 0, 0, 631, 632, 3, 74, 32, 0, 632, 634, 1, 0, 0, 0, 633, 592, 1, 0, 0, 0, 633, 603, 1, 0, 0, 0, 633, 610, 1, 0, 0, 0, 633, 625, 1, 0, 0, 0, 634, 91, 1, 0, 0, 0, 635, 636, 5, 98, 0, 0, 636, 637, 5, 121, 0, 0, 637, 93, 1, 0, 0, 0, 638, 639, 5, 97, 0, 0, 639, 640, 5, 110, 0, 0, 640, 641, 5, 100, 0, 0, 641, 95, 1, 0, 0, 0, 642, 643, 5, 97, 0, 0, 643, 644, 5, 115, 0, 0, 644, 645, 5, 99, 0, 0, 645, 97, 1, 0, 0, 0, 646, 647, 5, 61, 0, 0, 647, 99, 1, 0, 0, 0, 648, 649, 5, 44, 0, 0, 649, 101, 1, 0, 0, 0, 650, 651, 5, 100, 0, 0, 651, 652, 5, 101, 0, 0, 652, 653, 5, 115, 0, 0, 653, 654, 5, 99, 0, 0, 654, 103, 1, 0, 0, 0, 655, 656, 5, 46, 0, 0, 656, 105, 1, 0, 0, 0, 657, 658, 5, 102, 0, 0, 658, 659, 5, 97, 0, 0, 659, 660, 5, 108, 0, 0, 660, 661, 5, 115, 0, 0, 661, 662, 5, 101, 0, 0, 662, 107, 1, 0, 0, 0, 663, 664, 5, 102, 0, 0, 664, 665, 5, 105, 0, 0, 665, 666, 5, 114, 0, 0, 666, 667, 5, 115, 0, 0, 667, 668, 5, 116, 0, 0, 668, 109, 1, 0, 0, 0, 669, 670, 5, 108, 0, 0, 670, 671, 5, 97, 0, 0, 671, 672, 5, 115, 0, 0, 672, 673, 5, 116, 0, 0, 673, 111, 1, 0, 0, 0, 674, 675, 5, 40, 0, 0, 675, 113, 1, 0, 0, 0, 676, 677, 5, 105, 0, 0, 677, 678, 5, 110, 0, 0, 678, 115, 1, 0, 0, 0, 679, 680, 5, 105, 0, 0, 680, 681, 5, 115, 0, 0, 681, 117, 1, 0, 0, 0, 682, 683, 5, 108, 0, 0, 683, 684, 5, 105, 0, 0, 684, 685, 5, 107, 0, 0, 685, 686, 5, 101, 0, 0, 686, 119, 1, 0, 0, 0, 687, 688, 5, 110, 0, 0, 688, 689, 5, 111, 0, 0, 689, 690, 5, 116, 0, 0, 690, 121, 1, 0, 0, 0, 691, 692, 5, 110, 0, 0, 692, 693, 5, 117, 0, 0, 693, 694, 5, 108, 0, 0, 694, 695, 5, 108, 0, 0, 695, 123, 1, 0, 0, 0, 696, 697, 5, 110, 0, 0, 697, 698, 5, 117, 0, 0, 698, 699, 5, 108, 0, 0, 699, 700, 5, 108, 0, 0, 700, 701, 5, 115, 0, 0, 701, 125, 1, 0, 0, 0, 702, 703, 5, 111, 0, 0, 703, 704, 5, 114, 0, 0, 704, 127, 1, 0, 0, 0, 705, 706, 5, 63, 0, 0, 706, 129, 1, 0, 0, 0, 707, 708, 5, 114, 0, 0, 708, 709, 5, 108, 0, 0, 709, 710, 5, 105, 0, 0, 710, 711, 5, 107, 0, 0, 711, 712, 5, 101, 0, 0, 712, 131, 1, 0, 0, 0, 713, 714, 5, 41, 0, 0, 714, 133, 1, 0, 0, 0, 715, 716, 5, 116, 0, 0, 716, 717, 5, 114, 0, 0, 717, 718, 5, 117, 0, 0, 718, 719, 5, 101, 0, 0, 719, 135, 1, 0, 0, 0, 720, 721, 5, 61, 0, 0, 721, 722, 5, 61, 0, 0, 722, 137, 1, 0, 0, 0, 723, 724, 5, 33, 0, 0, 724, 725, 5, 61, 0, 0, 725, 139, 1, 0, 0, 0, 726, 727, 5, 60, 0, 0, 727, 141, 1, 0, 0, 0, 728, 729, 5, 60, 0, 0, 729, 730, 5, 61, 0, 0, 730, 143, 1, 0, 0, 0, 731, 732, 5, 62, 0, 0, 732, 145, 1, 0, 0, 0, 733, 734, 5, 62, 0, 0, 734, 735, 5, 61, 0, 0, 735, 147, 1, 0, 0, 0, 736, 737, 5, 43, 0, 0, 737, 149, 1, 0, 0, 0, 738, 739, 5, 45, 0, 0, 739, 151, 1, 0, 0, 0, 740, 741, 5, 42, 0, 0, 741, 153, 1, 0, 0, 0, 742, 743, 5, 47, 0, 0, 743, 155, 1, 0, 0, 0, 744, 745, 5, 37, 0, 0, 745, 157, 1, 0, 0, 0, 746, 747, 5, 91, 0, 0, 747, 748, 1, 0, 0, 0, 748, 749, 6, 74, 0, 0, 749, 750, 6, 74, 0, 0, 750, 159, 1, 0, 0, 0, 751, 752, 5, 93, 0, 0, 752, 753, 1, 0, 0, 0, 753, 754, 6, 75, 12, 0, 754, 755, 6, 75, 12, 0, 755, 161, 1, 0, 0, 0, 756, 760, 3, 68, 29, 0, 757, 759, 3, 84, 37, 0, 758, 757, 1, 0, 0, 0, 759, 762, 1, 0, 0, 0, 760, 758, 1, 0, 0, 0, 760, 761, 1, 0, 0, 0, 761, 773, 1, 0, 0, 0, 762, 760, 1, 0, 0, 0, 763, 766, 3, 82, 36, 0, 764, 766, 3, 76, 33, 0, 765, 763, 1, 0, 0, 0, 765, 764, 1, 0, 0, 0, 766, 768, 1, 0, 0, 0, 767, 769, 3, 84, 37, 0, 768, 767, 1, 0, 0, 0, 769, 770, 1, 0, 0, 0, 770, 768, 1, 0, 0, 0, 770, 771, 1, 0, 0, 0, 771, 773, 1, 0, 0, 0, 772, 756, 1, 0, 0, 0, 772, 765, 1, 0, 0, 0, 773, 163, 1, 0, 0, 0, 774, 776, 3, 78, 34, 0, 775, 777, 3, 80, 35, 0, 776, 775, 1, 0, 0, 0, 777, 778, 1, 0, 0, 0, 778, 776, 1, 0, 0, 0, 778, 779, 1, 0, 0, 0, 779, 780, 1, 0, 0, 0, 780, 781, 3, 78, 34, 0, 781, 165, 1, 0, 0, 0, 782, 783, 3, 48, 19, 0, 783, 784, 1, 0, 0, 0, 784, 785, 6, 78, 8, 0, 785, 167, 1, 0, 0, 0, 786, 787, 3, 50, 20, 0, 787, 788, 1, 0, 0, 0, 788, 789, 6, 79, 8, 0, 789, 169, 1, 0, 0, 0, 790, 791, 3, 52, 21, 0, 791, 792, 1, 0, 0, 0, 792, 793, 6, 80, 8, 0, 793, 171, 1, 0, 0, 0, 794, 795, 3, 64, 27, 0, 795, 796, 1, 0, 0, 0, 796, 797, 6, 81, 11, 0, 797, 798, 6, 81, 12, 0, 798, 173, 1, 0, 0, 0, 799, 800, 3, 158, 74, 0, 800, 801, 1, 0, 0, 0, 801, 802, 6, 82, 9, 0, 802, 803, 6, 82, 4, 0, 803, 804, 6, 82, 4, 0, 804, 175, 1, 0, 0, 0, 805, 806, 3, 160, 75, 0, 806, 807, 1, 0, 0, 0, 807, 808, 6, 83, 13, 0, 808, 809, 6, 83, 12, 0, 809, 810, 6, 83, 12, 0, 810, 177, 1, 0, 0, 0, 811, 812, 3, 100, 45, 0, 812, 813, 1, 0, 0, 0, 813, 814, 6, 84, 14, 0, 814, 179, 1, 0, 0, 0, 815, 816, 3, 98, 44, 0, 816, 817, 1, 0, 0, 0, 817, 818, 6, 85, 15, 0, 818, 181, 1, 0, 0, 0, 819, 820, 5, 109, 0, 0, 820, 821, 5, 101, 0, 0, 821, 822, 5, 116, 0, 0, 822, 823, 5, 97, 0, 0, 823, 824, 5, 100, 0, 0, 824, 825, 5, 97, 0, 0, 825, 826, 5, 116, 0, 0, 826, 827, 5, 97, 0, 0, 827, 183, 1, 0, 0, 0, 828, 832, 8, 10, 0, 0, 829, 830, 5, 47, 0, 0, 830, 832, 8, 11, 0, 0, 831, 828, 1, 0, 0, 0, 831, 829, 1, 0, 0, 0, 832, 185, 1, 0, 0, 0, 833, 835, 3, 184, 87, 0, 834, 833, 1, 0, 0, 0, 835, 836, 1, 0, 0, 0, 836, 834, 1, 0, 0, 0, 836, 837, 1, 0, 0, 0, 837, 187, 1, 0, 0, 0, 838, 839, 3, 164, 77, 0, 839, 840, 1, 0, 0, 0, 840, 841, 6, 89, 16, 0, 841, 189, 1, 0, 0, 0, 842, 843, 3, 48, 19, 0, 843, 844, 1, 0, 0, 0, 844, 845, 6, 90, 8, 0, 845, 191, 1, 0, 0, 0, 846, 847, 3, 50, 20, 0, 847, 848, 1, 0, 0, 0, 848, 849, 6, 91, 8, 0, 849, 193, 1, 0, 0, 0, 850, 851, 3, 52, 21, 0, 851, 852, 1, 0, 0, 0, 852, 853, 6, 92, 8, 0, 853, 195, 1, 0, 0, 0, 854, 855, 3, 64, 27, 0, 855, 856, 1, 0, 0, 0, 856, 857, 6, 93, 11, 0, 857, 858, 6, 93, 12, 0, 858, 197, 1, 0, 0, 0, 859, 860, 3, 104, 47, 0, 860, 861, 1, 0, 0, 0, 861, 862, 6, 94, 17, 0, 862, 199, 1, 0, 0, 0, 863, 864, 3, 100, 45, 0, 864, 865, 1, 0, 0, 0, 865, 866, 6, 95, 14, 0, 866, 201, 1, 0, 0, 0, 867, 872, 3, 68, 29, 0, 868, 872, 3, 66, 28, 0, 869, 872, 3, 82, 36, 0, 870, 872, 3, 152, 71, 0, 871, 867, 1, 0, 0, 0, 871, 868, 1, 0, 0, 0, 871, 869, 1, 0, 0, 0, 871, 870, 1, 0, 0, 0, 872, 203, 1, 0, 0, 0, 873, 876, 3, 68, 29, 0, 874, 876, 3, 152, 71, 0, 875, 873, 1, 0, 0, 0, 875, 874, 1, 0, 0, 0, 876, 880, 1, 0, 0, 0, 877, 879, 3, 202, 96, 0, 878, 877, 1, 0, 0, 0, 879, 882, 1, 0, 0, 0, 880, 878, 1, 0, 0, 0, 880, 881, 1, 0, 0, 0, 881, 893, 1, 0, 0, 0, 882, 880, 1, 0, 0, 0, 883, 886, 3, 82, 36, 0, 884, 886, 3, 76, 33, 0, 885, 883, 1, 0, 0, 0, 885, 884, 1, 0, 0, 0, 886, 888, 1, 0, 0, 0, 887, 889, 3, 202, 96, 0, 888, 887, 1, 0, 0, 0, 889, 890, 1, 0, 0, 0, 890, 888, 1, 0, 0, 0, 890, 891, 1, 0, 0, 0, 891, 893, 1, 0, 0, 0, 892, 875, 1, 0, 0, 0, 892, 885, 1, 0, 0, 0, 893, 205, 1, 0, 0, 0, 894, 895, 3, 164, 77, 0, 895, 896, 1, 0, 0, 0, 896, 897, 6, 98, 16, 0, 897, 207, 1, 0, 0, 0, 898, 899, 3, 48, 19, 0, 899, 900, 1, 0, 0, 0, 900, 901, 6, 99, 8, 0, 901, 209, 1, 0, 0, 0, 902, 903, 3, 50, 20, 0, 903, 904, 1, 0, 0, 0, 904, 905, 6, 100, 8, 0, 905, 211, 1, 0, 0, 0, 906, 907, 3, 52, 21, 0, 907, 908, 1, 0, 0, 0, 908, 909, 6, 101, 8, 0, 909, 213, 1, 0, 0, 0, 910, 911, 3, 64, 27, 0, 911, 912, 1, 0, 0, 0, 912, 913, 6, 102, 11, 0, 913, 914, 6, 102, 12, 0, 914, 215, 1, 0, 0, 0, 915, 916, 3, 98, 44, 0, 916, 917, 1, 0, 0, 0, 917, 918, 6, 103, 15, 0, 918, 217, 1, 0, 0, 0, 919, 920, 3, 100, 45, 0, 920, 921, 1, 0, 0, 0, 921, 922, 6, 104, 14, 0, 922, 219, 1, 0, 0, 0, 923, 924, 3, 104, 47, 0, 924, 925, 1, 0, 0, 0, 925, 926, 6, 105, 17, 0, 926, 221, 1, 0, 0, 0, 927, 928, 5, 97, 0, 0, 928, 929, 5, 115, 0, 0, 929, 223, 1, 0, 0, 0, 930, 931, 3, 164, 77, 0, 931, 932, 1, 0, 0, 0, 932, 933, 6, 107, 16, 0, 933, 225, 1, 0, 0, 0, 934, 935, 3, 204, 97, 0, 935, 936, 1, 0, 0, 0, 936, 937, 6, 108, 18, 0, 937, 227, 1, 0, 0, 0, 938, 939, 3, 48, 19, 0, 939, 940, 1, 0, 0, 0, 940, 941, 6, 109, 8, 0, 941, 229, 1, 0, 0, 0, 942, 943, 3, 50, 20, 0, 943, 944, 1, 0, 0, 0, 944, 945, 6, 110, 8, 0, 945, 231, 1, 0, 0, 0, 946, 947, 3, 52, 21, 0, 947, 948, 1, 0, 0, 0, 948, 949, 6, 111, 8, 0, 949, 233, 1, 0, 0, 0, 950, 951, 3, 64, 27, 0, 951, 952, 1, 0, 0, 0, 952, 953, 6, 112, 11, 0, 953, 954, 6, 112, 12, 0, 954, 235, 1, 0, 0, 0, 955, 956, 5, 111, 0, 0, 956, 957, 5, 110, 0, 0, 957, 958, 1, 0, 0, 0, 958, 959, 6, 113, 19, 0, 959, 237, 1, 0, 0, 0, 960, 961, 5, 119, 0, 0, 961, 962, 5, 105, 0, 0, 962, 963, 5, 116, 0, 0, 963, 964, 5, 104, 0, 0, 964, 965, 1, 0, 0, 0, 965, 966, 6, 114, 19, 0, 966, 239, 1, 0, 0, 0, 967, 968, 3, 186, 88, 0, 968, 969, 1, 0, 0, 0, 969, 970, 6, 115, 20, 0, 970, 241, 1, 0, 0, 0, 971, 972, 3, 164, 77, 0, 972, 973, 1, 0, 0, 0, 973, 974, 6, 116, 16, 0, 974, 243, 1, 0, 0, 0, 975, 976, 3, 48, 19, 0, 976, 977, 1, 0, 0, 0, 977, 978, 6, 117, 8, 0, 978, 245, 1, 0, 0, 0, 979, 980, 3, 50, 20, 0, 980, 981, 1, 0, 0, 0, 981, 982, 6, 118, 8, 0, 982, 247, 1, 0, 0, 0, 983, 984, 3, 52, 21, 0, 984, 985, 1, 0, 0, 0, 985, 986, 6, 119, 8, 0, 986, 249, 1, 0, 0, 0, 987, 988, 3, 64, 27, 0, 988, 989, 1, 0, 0, 0, 989, 990, 6, 120, 11, 0, 990, 991, 6, 120, 12, 0, 991, 992, 6, 120, 12, 0, 992, 251, 1, 0, 0, 0, 993, 994, 3, 98, 44, 0, 994, 995, 1, 0, 0, 0, 995, 996, 6, 121, 15, 0, 996, 253, 1, 0, 0, 0, 997, 998, 3, 100, 45, 0, 998, 999, 1, 0, 0, 0, 999, 1000, 6, 122, 14, 0, 1000, 255, 1, 0, 0, 0, 1001, 1002, 3, 104, 47, 0, 1002, 1003, 1, 0, 0, 0, 1003, 1004, 6, 123, 17, 0, 1004, 257, 1, 0, 0, 0, 1005, 1006, 3, 238, 114, 0, 1006, 1007, 1, 0, 0, 0, 1007, 1008, 6, 124, 21, 0, 1008, 259, 1, 0, 0, 0, 1009, 1010, 3, 204, 97, 0, 1010, 1011, 1, 0, 0, 0, 1011, 1012, 6, 125, 18, 0, 1012, 261, 1, 0, 0, 0, 1013, 1014, 3, 164, 77, 0, 1014, 1015, 1, 0, 0, 0, 1015, 1016, 6, 126, 16, 0, 1016, 263, 1, 0, 0, 0, 1017, 1018, 3, 48, 19, 0, 1018, 1019, 1, 0, 0, 0, 1019, 1020, 6, 127, 8, 0, 1020, 265, 1, 0, 0, 0, 1021, 1022, 3, 50, 20, 0, 1022, 1023, 1, 0, 0, 0, 1023, 1024, 6, 128, 8, 0, 1024, 267, 1, 0, 0, 0, 1025, 1026, 3, 52, 21, 0, 1026, 1027, 1, 0, 0, 0, 1027, 1028, 6, 129, 8, 0, 1028, 269, 1, 0, 0, 0, 1029, 1030, 3, 64, 27, 0, 1030, 1031, 1, 0, 0, 0, 1031, 1032, 6, 130, 11, 0, 1032, 1033, 6, 130, 12, 0, 1033, 271, 1, 0, 0, 0, 1034, 1035, 3, 104, 47, 0, 1035, 1036, 1, 0, 0, 0, 1036, 1037, 6, 131, 17, 0, 1037, 273, 1, 0, 0, 0, 1038, 1039, 3, 164, 77, 0, 1039, 1040, 1, 0, 0, 0, 1040, 1041, 6, 132, 16, 0, 1041, 275, 1, 0, 0, 0, 1042, 1043, 3, 162, 76, 0, 1043, 1044, 1, 0, 0, 0, 1044, 1045, 6, 133, 22, 0, 1045, 277, 1, 0, 0, 0, 1046, 1047, 3, 48, 19, 0, 1047, 1048, 1, 0, 0, 0, 1048, 1049, 6, 134, 8, 0, 1049, 279, 1, 0, 0, 0, 1050, 1051, 3, 50, 20, 0, 1051, 1052, 1, 0, 0, 0, 1052, 1053, 6, 135, 8, 0, 1053, 281, 1, 0, 0, 0, 1054, 1055, 3, 52, 21, 0, 1055, 1056, 1, 0, 0, 0, 1056, 1057, 6, 136, 8, 0, 1057, 283, 1, 0, 0, 0, 1058, 1059, 3, 64, 27, 0, 1059, 1060, 1, 0, 0, 0, 1060, 1061, 6, 137, 11, 0, 1061, 1062, 6, 137, 12, 0, 1062, 285, 1, 0, 0, 0, 1063, 1064, 5, 105, 0, 0, 1064, 1065, 5, 110, 0, 0, 1065, 1066, 5, 102, 0, 0, 1066, 1067, 5, 111, 0, 0, 1067, 287, 1, 0, 0, 0, 1068, 1069, 5, 102, 0, 0, 1069, 1070, 5, 117, 0, 0, 1070, 1071, 5, 110, 0, 0, 1071, 1072, 5, 99, 0, 0, 1072, 1073, 5, 116, 0, 0, 1073, 1074, 5, 105, 0, 0, 1074, 1075, 5, 111, 0, 0, 1075, 1076, 5, 110, 0, 0, 1076, 1077, 5, 115, 0, 0, 1077, 289, 1, 0, 0, 0, 1078, 1079, 3, 48, 19, 0, 1079, 1080, 1, 0, 0, 0, 1080, 1081, 6, 140, 8, 0, 1081, 291, 1, 0, 0, 0, 1082, 1083, 3, 50, 20, 0, 1083, 1084, 1, 0, 0, 0, 1084, 1085, 6, 141, 8, 0, 1085, 293, 1, 0, 0, 0, 1086, 1087, 3, 52, 21, 0, 1087, 1088, 1, 0, 0, 0, 1088, 1089, 6, 142, 8, 0, 1089, 295, 1, 0, 0, 0, 49, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 452, 462, 466, 469, 478, 480, 491, 532, 537, 546, 553, 558, 560, 571, 579, 582, 584, 589, 594, 600, 607, 612, 618, 621, 629, 633, 760, 765, 770, 772, 778, 831, 836, 871, 875, 880, 885, 890, 892, 23, 5, 2, 0, 5, 4, 0, 5, 6, 0, 5, 1, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 0, 1, 0, 7, 63, 0, 5, 0, 0, 7, 26, 0, 4, 0, 0, 7, 64, 0, 7, 34, 0, 7, 33, 0, 7, 66, 0, 7, 36, 0, 7, 75, 0, 5, 7, 0, 7, 71, 0, 7, 84, 0, 7, 65, 0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java index 5a01cfa11b3fd..8946172327bcc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java @@ -24,20 +24,28 @@ public class EsqlBaseLexer extends Lexer { PIPE=26, STRING=27, INTEGER_LITERAL=28, DECIMAL_LITERAL=29, BY=30, AND=31, ASC=32, ASSIGN=33, COMMA=34, DESC=35, DOT=36, FALSE=37, FIRST=38, LAST=39, LP=40, IN=41, IS=42, LIKE=43, NOT=44, NULL=45, NULLS=46, OR=47, PARAM=48, - RLIKE=49, RP=50, TRUE=51, INFO=52, FUNCTIONS=53, EQ=54, NEQ=55, LT=56, - LTE=57, GT=58, GTE=59, PLUS=60, MINUS=61, ASTERISK=62, SLASH=63, PERCENT=64, - OPENING_BRACKET=65, CLOSING_BRACKET=66, UNQUOTED_IDENTIFIER=67, QUOTED_IDENTIFIER=68, - EXPR_LINE_COMMENT=69, EXPR_MULTILINE_COMMENT=70, EXPR_WS=71, AS=72, METADATA=73, - ON=74, WITH=75, SRC_UNQUOTED_IDENTIFIER=76, SRC_QUOTED_IDENTIFIER=77, - SRC_LINE_COMMENT=78, SRC_MULTILINE_COMMENT=79, SRC_WS=80, EXPLAIN_PIPE=81; + RLIKE=49, RP=50, TRUE=51, EQ=52, NEQ=53, LT=54, LTE=55, GT=56, GTE=57, + PLUS=58, MINUS=59, ASTERISK=60, SLASH=61, PERCENT=62, OPENING_BRACKET=63, + CLOSING_BRACKET=64, UNQUOTED_IDENTIFIER=65, QUOTED_IDENTIFIER=66, EXPR_LINE_COMMENT=67, + EXPR_MULTILINE_COMMENT=68, EXPR_WS=69, METADATA=70, FROM_UNQUOTED_IDENTIFIER=71, + FROM_LINE_COMMENT=72, FROM_MULTILINE_COMMENT=73, FROM_WS=74, PROJECT_UNQUOTED_IDENTIFIER=75, + PROJECT_LINE_COMMENT=76, PROJECT_MULTILINE_COMMENT=77, PROJECT_WS=78, + AS=79, RENAME_LINE_COMMENT=80, RENAME_MULTILINE_COMMENT=81, RENAME_WS=82, + ON=83, WITH=84, ENRICH_LINE_COMMENT=85, ENRICH_MULTILINE_COMMENT=86, ENRICH_WS=87, + ENRICH_FIELD_LINE_COMMENT=88, ENRICH_FIELD_MULTILINE_COMMENT=89, ENRICH_FIELD_WS=90, + MVEXPAND_LINE_COMMENT=91, MVEXPAND_MULTILINE_COMMENT=92, MVEXPAND_WS=93, + INFO=94, FUNCTIONS=95, SHOW_LINE_COMMENT=96, SHOW_MULTILINE_COMMENT=97, + SHOW_WS=98; public static final int - EXPLAIN_MODE=1, EXPRESSION=2, SOURCE_IDENTIFIERS=3; + EXPLAIN_MODE=1, EXPRESSION_MODE=2, FROM_MODE=3, PROJECT_MODE=4, RENAME_MODE=5, + ENRICH_MODE=6, ENRICH_FIELD_MODE=7, MVEXPAND_MODE=8, SHOW_MODE=9; public static String[] channelNames = { "DEFAULT_TOKEN_CHANNEL", "HIDDEN" }; public static String[] modeNames = { - "DEFAULT_MODE", "EXPLAIN_MODE", "EXPRESSION", "SOURCE_IDENTIFIERS" + "DEFAULT_MODE", "EXPLAIN_MODE", "EXPRESSION_MODE", "FROM_MODE", "PROJECT_MODE", + "RENAME_MODE", "ENRICH_MODE", "ENRICH_FIELD_MODE", "MVEXPAND_MODE", "SHOW_MODE" }; private static String[] makeRuleNames() { @@ -47,16 +55,30 @@ private static String[] makeRuleNames() { "STATS", "WHERE", "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "EXPLAIN_OPENING_BRACKET", "EXPLAIN_PIPE", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", "PIPE", "DIGIT", "LETTER", "ESCAPE_SEQUENCE", - "UNESCAPED_CHARS", "EXPONENT", "STRING", "INTEGER_LITERAL", "DECIMAL_LITERAL", + "UNESCAPED_CHARS", "EXPONENT", "ASPERAND", "BACKQUOTE", "BACKQUOTE_BLOCK", + "UNDERSCORE", "UNQUOTED_ID_BODY", "STRING", "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "COMMA", "DESC", "DOT", "FALSE", "FIRST", "LAST", "LP", "IN", "IS", "LIKE", "NOT", "NULL", "NULLS", "OR", "PARAM", - "RLIKE", "RP", "TRUE", "INFO", "FUNCTIONS", "EQ", "NEQ", "LT", "LTE", - "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", "PERCENT", "OPENING_BRACKET", - "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", - "EXPR_MULTILINE_COMMENT", "EXPR_WS", "SRC_PIPE", "SRC_OPENING_BRACKET", - "SRC_CLOSING_BRACKET", "SRC_COMMA", "SRC_ASSIGN", "AS", "METADATA", "ON", - "WITH", "SRC_UNQUOTED_IDENTIFIER", "SRC_UNQUOTED_IDENTIFIER_PART", "SRC_QUOTED_IDENTIFIER", - "SRC_LINE_COMMENT", "SRC_MULTILINE_COMMENT", "SRC_WS" + "RLIKE", "RP", "TRUE", "EQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", + "MINUS", "ASTERISK", "SLASH", "PERCENT", "OPENING_BRACKET", "CLOSING_BRACKET", + "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", + "EXPR_WS", "FROM_PIPE", "FROM_OPENING_BRACKET", "FROM_CLOSING_BRACKET", + "FROM_COMMA", "FROM_ASSIGN", "METADATA", "FROM_UNQUOTED_IDENTIFIER_PART", + "FROM_UNQUOTED_IDENTIFIER", "FROM_QUOTED_IDENTIFIER", "FROM_LINE_COMMENT", + "FROM_MULTILINE_COMMENT", "FROM_WS", "PROJECT_PIPE", "PROJECT_DOT", "PROJECT_COMMA", + "UNQUOTED_ID_BODY_WITH_PATTERN", "PROJECT_UNQUOTED_IDENTIFIER", "PROJECT_QUOTED_IDENTIFIER", + "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", "PROJECT_WS", "RENAME_PIPE", + "RENAME_ASSIGN", "RENAME_COMMA", "RENAME_DOT", "AS", "RENAME_QUOTED_IDENTIFIER", + "RENAME_UNQUOTED_IDENTIFIER", "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", + "RENAME_WS", "ENRICH_PIPE", "ON", "WITH", "ENRICH_POLICY_UNQUOTED_IDENTIFIER", + "ENRICH_QUOTED_IDENTIFIER", "ENRICH_LINE_COMMENT", "ENRICH_MULTILINE_COMMENT", + "ENRICH_WS", "ENRICH_FIELD_PIPE", "ENRICH_FIELD_ASSIGN", "ENRICH_FIELD_COMMA", + "ENRICH_FIELD_DOT", "ENRICH_FIELD_WITH", "ENRICH_FIELD_UNQUOTED_IDENTIFIER", + "ENRICH_FIELD_QUOTED_IDENTIFIER", "ENRICH_FIELD_LINE_COMMENT", "ENRICH_FIELD_MULTILINE_COMMENT", + "ENRICH_FIELD_WS", "MVEXPAND_PIPE", "MVEXPAND_DOT", "MVEXPAND_QUOTED_IDENTIFIER", + "MVEXPAND_UNQUOTED_IDENTIFIER", "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", + "MVEXPAND_WS", "SHOW_PIPE", "INFO", "FUNCTIONS", "SHOW_LINE_COMMENT", + "SHOW_MULTILINE_COMMENT", "SHOW_WS" }; } public static final String[] ruleNames = makeRuleNames(); @@ -66,12 +88,14 @@ private static String[] makeLiteralNames() { null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", "'grok'", "'inlinestats'", "'keep'", "'limit'", "'mv_expand'", "'project'", "'rename'", "'row'", "'show'", "'sort'", "'stats'", "'where'", null, - null, null, null, null, null, null, null, null, null, null, "'by'", "'and'", - "'asc'", null, null, "'desc'", "'.'", "'false'", "'first'", "'last'", - "'('", "'in'", "'is'", "'like'", "'not'", "'null'", "'nulls'", "'or'", - "'?'", "'rlike'", "')'", "'true'", "'info'", "'functions'", "'=='", "'!='", - "'<'", "'<='", "'>'", "'>='", "'+'", "'-'", "'*'", "'/'", "'%'", null, - "']'", null, null, null, null, null, "'as'", "'metadata'", "'on'", "'with'" + null, null, null, null, null, null, "'|'", null, null, null, "'by'", + "'and'", "'asc'", "'='", "','", "'desc'", "'.'", "'false'", "'first'", + "'last'", "'('", "'in'", "'is'", "'like'", "'not'", "'null'", "'nulls'", + "'or'", "'?'", "'rlike'", "')'", "'true'", "'=='", "'!='", "'<'", "'<='", + "'>'", "'>='", "'+'", "'-'", "'*'", "'/'", "'%'", null, "']'", null, + null, null, null, null, "'metadata'", null, null, null, null, null, null, + null, null, "'as'", null, null, null, "'on'", "'with'", null, null, null, + null, null, null, null, null, null, "'info'", "'functions'" }; } private static final String[] _LITERAL_NAMES = makeLiteralNames(); @@ -84,12 +108,17 @@ private static String[] makeSymbolicNames() { "PIPE", "STRING", "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "COMMA", "DESC", "DOT", "FALSE", "FIRST", "LAST", "LP", "IN", "IS", "LIKE", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", - "TRUE", "INFO", "FUNCTIONS", "EQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", - "MINUS", "ASTERISK", "SLASH", "PERCENT", "OPENING_BRACKET", "CLOSING_BRACKET", - "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", - "EXPR_WS", "AS", "METADATA", "ON", "WITH", "SRC_UNQUOTED_IDENTIFIER", - "SRC_QUOTED_IDENTIFIER", "SRC_LINE_COMMENT", "SRC_MULTILINE_COMMENT", - "SRC_WS", "EXPLAIN_PIPE" + "TRUE", "EQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", + "SLASH", "PERCENT", "OPENING_BRACKET", "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", + "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", + "METADATA", "FROM_UNQUOTED_IDENTIFIER", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", + "FROM_WS", "PROJECT_UNQUOTED_IDENTIFIER", "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", + "PROJECT_WS", "AS", "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", + "RENAME_WS", "ON", "WITH", "ENRICH_LINE_COMMENT", "ENRICH_MULTILINE_COMMENT", + "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", "ENRICH_FIELD_MULTILINE_COMMENT", + "ENRICH_FIELD_WS", "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", + "MVEXPAND_WS", "INFO", "FUNCTIONS", "SHOW_LINE_COMMENT", "SHOW_MULTILINE_COMMENT", + "SHOW_WS" }; } private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames(); @@ -152,482 +181,683 @@ public EsqlBaseLexer(CharStream input) { public ATN getATN() { return _ATN; } public static final String _serializedATN = - "\u0004\u0000Q\u02fc\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ - "\u0006\uffff\uffff\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+ - "\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002"+ - "\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007\u0002"+ - "\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b\u0002"+ - "\f\u0007\f\u0002\r\u0007\r\u0002\u000e\u0007\u000e\u0002\u000f\u0007\u000f"+ - "\u0002\u0010\u0007\u0010\u0002\u0011\u0007\u0011\u0002\u0012\u0007\u0012"+ - "\u0002\u0013\u0007\u0013\u0002\u0014\u0007\u0014\u0002\u0015\u0007\u0015"+ - "\u0002\u0016\u0007\u0016\u0002\u0017\u0007\u0017\u0002\u0018\u0007\u0018"+ - "\u0002\u0019\u0007\u0019\u0002\u001a\u0007\u001a\u0002\u001b\u0007\u001b"+ - "\u0002\u001c\u0007\u001c\u0002\u001d\u0007\u001d\u0002\u001e\u0007\u001e"+ - "\u0002\u001f\u0007\u001f\u0002 \u0007 \u0002!\u0007!\u0002\"\u0007\"\u0002"+ - "#\u0007#\u0002$\u0007$\u0002%\u0007%\u0002&\u0007&\u0002\'\u0007\'\u0002"+ - "(\u0007(\u0002)\u0007)\u0002*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002"+ - "-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u00021\u00071\u0002"+ - "2\u00072\u00023\u00073\u00024\u00074\u00025\u00075\u00026\u00076\u0002"+ - "7\u00077\u00028\u00078\u00029\u00079\u0002:\u0007:\u0002;\u0007;\u0002"+ - "<\u0007<\u0002=\u0007=\u0002>\u0007>\u0002?\u0007?\u0002@\u0007@\u0002"+ - "A\u0007A\u0002B\u0007B\u0002C\u0007C\u0002D\u0007D\u0002E\u0007E\u0002"+ - "F\u0007F\u0002G\u0007G\u0002H\u0007H\u0002I\u0007I\u0002J\u0007J\u0002"+ - "K\u0007K\u0002L\u0007L\u0002M\u0007M\u0002N\u0007N\u0002O\u0007O\u0002"+ - "P\u0007P\u0002Q\u0007Q\u0002R\u0007R\u0002S\u0007S\u0002T\u0007T\u0002"+ - "U\u0007U\u0002V\u0007V\u0002W\u0007W\u0002X\u0007X\u0002Y\u0007Y\u0002"+ - "Z\u0007Z\u0002[\u0007[\u0002\\\u0007\\\u0001\u0000\u0001\u0000\u0001\u0000"+ - "\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000"+ - "\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ - "\u0001\u0001\u0001\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002"+ - "\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0003"+ - "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ - "\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004"+ - "\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005"+ - "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0006"+ - "\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006"+ - "\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007"+ - "\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007"+ - "\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001"+ - "\b\u0001\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001"+ - "\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+ - "\n\u0001\n\u0001\n\u0001\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"+ - "\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"+ - "\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001"+ - "\f\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\u000e\u0001\u000e"+ - "\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f"+ - "\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f"+ + "\u0004\u0000b\u0442\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ + "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ + "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0002\u0000\u0007"+ + "\u0000\u0002\u0001\u0007\u0001\u0002\u0002\u0007\u0002\u0002\u0003\u0007"+ + "\u0003\u0002\u0004\u0007\u0004\u0002\u0005\u0007\u0005\u0002\u0006\u0007"+ + "\u0006\u0002\u0007\u0007\u0007\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n"+ + "\u0007\n\u0002\u000b\u0007\u000b\u0002\f\u0007\f\u0002\r\u0007\r\u0002"+ + "\u000e\u0007\u000e\u0002\u000f\u0007\u000f\u0002\u0010\u0007\u0010\u0002"+ + "\u0011\u0007\u0011\u0002\u0012\u0007\u0012\u0002\u0013\u0007\u0013\u0002"+ + "\u0014\u0007\u0014\u0002\u0015\u0007\u0015\u0002\u0016\u0007\u0016\u0002"+ + "\u0017\u0007\u0017\u0002\u0018\u0007\u0018\u0002\u0019\u0007\u0019\u0002"+ + "\u001a\u0007\u001a\u0002\u001b\u0007\u001b\u0002\u001c\u0007\u001c\u0002"+ + "\u001d\u0007\u001d\u0002\u001e\u0007\u001e\u0002\u001f\u0007\u001f\u0002"+ + " \u0007 \u0002!\u0007!\u0002\"\u0007\"\u0002#\u0007#\u0002$\u0007$\u0002"+ + "%\u0007%\u0002&\u0007&\u0002\'\u0007\'\u0002(\u0007(\u0002)\u0007)\u0002"+ + "*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002-\u0007-\u0002.\u0007.\u0002"+ + "/\u0007/\u00020\u00070\u00021\u00071\u00022\u00072\u00023\u00073\u0002"+ + "4\u00074\u00025\u00075\u00026\u00076\u00027\u00077\u00028\u00078\u0002"+ + "9\u00079\u0002:\u0007:\u0002;\u0007;\u0002<\u0007<\u0002=\u0007=\u0002"+ + ">\u0007>\u0002?\u0007?\u0002@\u0007@\u0002A\u0007A\u0002B\u0007B\u0002"+ + "C\u0007C\u0002D\u0007D\u0002E\u0007E\u0002F\u0007F\u0002G\u0007G\u0002"+ + "H\u0007H\u0002I\u0007I\u0002J\u0007J\u0002K\u0007K\u0002L\u0007L\u0002"+ + "M\u0007M\u0002N\u0007N\u0002O\u0007O\u0002P\u0007P\u0002Q\u0007Q\u0002"+ + "R\u0007R\u0002S\u0007S\u0002T\u0007T\u0002U\u0007U\u0002V\u0007V\u0002"+ + "W\u0007W\u0002X\u0007X\u0002Y\u0007Y\u0002Z\u0007Z\u0002[\u0007[\u0002"+ + "\\\u0007\\\u0002]\u0007]\u0002^\u0007^\u0002_\u0007_\u0002`\u0007`\u0002"+ + "a\u0007a\u0002b\u0007b\u0002c\u0007c\u0002d\u0007d\u0002e\u0007e\u0002"+ + "f\u0007f\u0002g\u0007g\u0002h\u0007h\u0002i\u0007i\u0002j\u0007j\u0002"+ + "k\u0007k\u0002l\u0007l\u0002m\u0007m\u0002n\u0007n\u0002o\u0007o\u0002"+ + "p\u0007p\u0002q\u0007q\u0002r\u0007r\u0002s\u0007s\u0002t\u0007t\u0002"+ + "u\u0007u\u0002v\u0007v\u0002w\u0007w\u0002x\u0007x\u0002y\u0007y\u0002"+ + "z\u0007z\u0002{\u0007{\u0002|\u0007|\u0002}\u0007}\u0002~\u0007~\u0002"+ + "\u007f\u0007\u007f\u0002\u0080\u0007\u0080\u0002\u0081\u0007\u0081\u0002"+ + "\u0082\u0007\u0082\u0002\u0083\u0007\u0083\u0002\u0084\u0007\u0084\u0002"+ + "\u0085\u0007\u0085\u0002\u0086\u0007\u0086\u0002\u0087\u0007\u0087\u0002"+ + "\u0088\u0007\u0088\u0002\u0089\u0007\u0089\u0002\u008a\u0007\u008a\u0002"+ + "\u008b\u0007\u008b\u0002\u008c\u0007\u008c\u0002\u008d\u0007\u008d\u0002"+ + "\u008e\u0007\u008e\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001"+ + "\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001"+ + "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ + "\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+ + "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0003\u0001\u0003\u0001"+ + "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0004\u0001"+ + "\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001"+ + "\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0006\u0001\u0006\u0001"+ + "\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0007\u0001"+ + "\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+ + "\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+ + "\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\t"+ + "\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\n\u0001"+ + "\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+ + "\n\u0001\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"+ + "\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001"+ + "\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\r\u0001"+ + "\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e"+ + "\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f"+ + "\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u0010"+ "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010"+ - "\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011"+ - "\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0012\u0004\u0012"+ - "\u0159\b\u0012\u000b\u0012\f\u0012\u015a\u0001\u0012\u0001\u0012\u0001"+ - "\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0005\u0013\u0163\b\u0013\n"+ - "\u0013\f\u0013\u0166\t\u0013\u0001\u0013\u0003\u0013\u0169\b\u0013\u0001"+ - "\u0013\u0003\u0013\u016c\b\u0013\u0001\u0013\u0001\u0013\u0001\u0014\u0001"+ - "\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0005\u0014\u0175\b\u0014\n"+ - "\u0014\f\u0014\u0178\t\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001"+ - "\u0014\u0001\u0014\u0001\u0015\u0004\u0015\u0180\b\u0015\u000b\u0015\f"+ - "\u0015\u0181\u0001\u0015\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0016"+ - "\u0001\u0016\u0001\u0016\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017"+ - "\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0019"+ - "\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u001a\u0001\u001a\u0001\u001a"+ - "\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001c"+ - "\u0001\u001c\u0001\u001d\u0001\u001d\u0001\u001e\u0001\u001e\u0001\u001e"+ - "\u0001\u001f\u0001\u001f\u0001 \u0001 \u0003 \u01ab\b \u0001 \u0004 \u01ae"+ - "\b \u000b \f \u01af\u0001!\u0001!\u0001!\u0005!\u01b5\b!\n!\f!\u01b8\t"+ - "!\u0001!\u0001!\u0001!\u0001!\u0001!\u0001!\u0005!\u01c0\b!\n!\f!\u01c3"+ - "\t!\u0001!\u0001!\u0001!\u0001!\u0001!\u0003!\u01ca\b!\u0001!\u0003!\u01cd"+ - "\b!\u0003!\u01cf\b!\u0001\"\u0004\"\u01d2\b\"\u000b\"\f\"\u01d3\u0001"+ - "#\u0004#\u01d7\b#\u000b#\f#\u01d8\u0001#\u0001#\u0005#\u01dd\b#\n#\f#"+ - "\u01e0\t#\u0001#\u0001#\u0004#\u01e4\b#\u000b#\f#\u01e5\u0001#\u0004#"+ - "\u01e9\b#\u000b#\f#\u01ea\u0001#\u0001#\u0005#\u01ef\b#\n#\f#\u01f2\t"+ - "#\u0003#\u01f4\b#\u0001#\u0001#\u0001#\u0001#\u0004#\u01fa\b#\u000b#\f"+ - "#\u01fb\u0001#\u0001#\u0003#\u0200\b#\u0001$\u0001$\u0001$\u0001%\u0001"+ - "%\u0001%\u0001%\u0001&\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001(\u0001"+ - "(\u0001)\u0001)\u0001)\u0001)\u0001)\u0001*\u0001*\u0001+\u0001+\u0001"+ - "+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001,\u0001,\u0001,\u0001,\u0001"+ - "-\u0001-\u0001-\u0001-\u0001-\u0001.\u0001.\u0001/\u0001/\u0001/\u0001"+ - "0\u00010\u00010\u00011\u00011\u00011\u00011\u00011\u00012\u00012\u0001"+ - "2\u00012\u00013\u00013\u00013\u00013\u00013\u00014\u00014\u00014\u0001"+ - "4\u00014\u00014\u00015\u00015\u00015\u00016\u00016\u00017\u00017\u0001"+ - "7\u00017\u00017\u00017\u00018\u00018\u00019\u00019\u00019\u00019\u0001"+ - "9\u0001:\u0001:\u0001:\u0001:\u0001:\u0001;\u0001;\u0001;\u0001;\u0001"+ - ";\u0001;\u0001;\u0001;\u0001;\u0001;\u0001<\u0001<\u0001<\u0001=\u0001"+ - "=\u0001=\u0001>\u0001>\u0001?\u0001?\u0001?\u0001@\u0001@\u0001A\u0001"+ - "A\u0001A\u0001B\u0001B\u0001C\u0001C\u0001D\u0001D\u0001E\u0001E\u0001"+ - "F\u0001F\u0001G\u0001G\u0001G\u0001G\u0001G\u0001H\u0001H\u0001H\u0001"+ - "H\u0001H\u0001I\u0001I\u0001I\u0001I\u0005I\u028e\bI\nI\fI\u0291\tI\u0001"+ - "I\u0001I\u0001I\u0001I\u0004I\u0297\bI\u000bI\fI\u0298\u0003I\u029b\b"+ - "I\u0001J\u0001J\u0001J\u0001J\u0005J\u02a1\bJ\nJ\fJ\u02a4\tJ\u0001J\u0001"+ - "J\u0001K\u0001K\u0001K\u0001K\u0001L\u0001L\u0001L\u0001L\u0001M\u0001"+ - "M\u0001M\u0001M\u0001N\u0001N\u0001N\u0001N\u0001N\u0001O\u0001O\u0001"+ - "O\u0001O\u0001O\u0001O\u0001P\u0001P\u0001P\u0001P\u0001P\u0001P\u0001"+ - "Q\u0001Q\u0001Q\u0001Q\u0001R\u0001R\u0001R\u0001R\u0001S\u0001S\u0001"+ - "S\u0001T\u0001T\u0001T\u0001T\u0001T\u0001T\u0001T\u0001T\u0001T\u0001"+ - "U\u0001U\u0001U\u0001V\u0001V\u0001V\u0001V\u0001V\u0001W\u0004W\u02e2"+ - "\bW\u000bW\fW\u02e3\u0001X\u0004X\u02e7\bX\u000bX\fX\u02e8\u0001X\u0001"+ - "X\u0003X\u02ed\bX\u0001Y\u0001Y\u0001Z\u0001Z\u0001Z\u0001Z\u0001[\u0001"+ - "[\u0001[\u0001[\u0001\\\u0001\\\u0001\\\u0001\\\u0002\u0176\u01c1\u0000"+ - "]\u0004\u0001\u0006\u0002\b\u0003\n\u0004\f\u0005\u000e\u0006\u0010\u0007"+ - "\u0012\b\u0014\t\u0016\n\u0018\u000b\u001a\f\u001c\r\u001e\u000e \u000f"+ - "\"\u0010$\u0011&\u0012(\u0013*\u0014,\u0015.\u00160\u00002Q4\u00176\u0018"+ - "8\u0019:\u001a<\u0000>\u0000@\u0000B\u0000D\u0000F\u001bH\u001cJ\u001d"+ - "L\u001eN\u001fP R!T\"V#X$Z%\\&^\'`(b)d*f+h,j-l.n/p0r1t2v3x4z5|6~7\u0080"+ - "8\u00829\u0084:\u0086;\u0088<\u008a=\u008c>\u008e?\u0090@\u0092A\u0094"+ - "B\u0096C\u0098D\u009aE\u009cF\u009eG\u00a0\u0000\u00a2\u0000\u00a4\u0000"+ - "\u00a6\u0000\u00a8\u0000\u00aaH\u00acI\u00aeJ\u00b0K\u00b2L\u00b4\u0000"+ - "\u00b6M\u00b8N\u00baO\u00bcP\u0004\u0000\u0001\u0002\u0003\r\u0006\u0000"+ - "\t\n\r\r //[[]]\u0002\u0000\n\n\r\r\u0003\u0000\t\n\r\r \u0001\u0000"+ - "09\u0002\u0000AZaz\u0005\u0000\"\"\\\\nnrrtt\u0004\u0000\n\n\r\r\"\"\\"+ - "\\\u0002\u0000EEee\u0002\u0000++--\u0002\u0000@@__\u0001\u0000``\n\u0000"+ - "\t\n\r\r ,,//==[[]]``||\u0002\u0000**//\u0318\u0000\u0004\u0001\u0000"+ - "\u0000\u0000\u0000\u0006\u0001\u0000\u0000\u0000\u0000\b\u0001\u0000\u0000"+ - "\u0000\u0000\n\u0001\u0000\u0000\u0000\u0000\f\u0001\u0000\u0000\u0000"+ - "\u0000\u000e\u0001\u0000\u0000\u0000\u0000\u0010\u0001\u0000\u0000\u0000"+ - "\u0000\u0012\u0001\u0000\u0000\u0000\u0000\u0014\u0001\u0000\u0000\u0000"+ - "\u0000\u0016\u0001\u0000\u0000\u0000\u0000\u0018\u0001\u0000\u0000\u0000"+ - "\u0000\u001a\u0001\u0000\u0000\u0000\u0000\u001c\u0001\u0000\u0000\u0000"+ - "\u0000\u001e\u0001\u0000\u0000\u0000\u0000 \u0001\u0000\u0000\u0000\u0000"+ - "\"\u0001\u0000\u0000\u0000\u0000$\u0001\u0000\u0000\u0000\u0000&\u0001"+ - "\u0000\u0000\u0000\u0000(\u0001\u0000\u0000\u0000\u0000*\u0001\u0000\u0000"+ - "\u0000\u0000,\u0001\u0000\u0000\u0000\u0000.\u0001\u0000\u0000\u0000\u0001"+ - "0\u0001\u0000\u0000\u0000\u00012\u0001\u0000\u0000\u0000\u00014\u0001"+ - "\u0000\u0000\u0000\u00016\u0001\u0000\u0000\u0000\u00018\u0001\u0000\u0000"+ - "\u0000\u0002:\u0001\u0000\u0000\u0000\u0002F\u0001\u0000\u0000\u0000\u0002"+ - "H\u0001\u0000\u0000\u0000\u0002J\u0001\u0000\u0000\u0000\u0002L\u0001"+ - "\u0000\u0000\u0000\u0002N\u0001\u0000\u0000\u0000\u0002P\u0001\u0000\u0000"+ - "\u0000\u0002R\u0001\u0000\u0000\u0000\u0002T\u0001\u0000\u0000\u0000\u0002"+ - "V\u0001\u0000\u0000\u0000\u0002X\u0001\u0000\u0000\u0000\u0002Z\u0001"+ - "\u0000\u0000\u0000\u0002\\\u0001\u0000\u0000\u0000\u0002^\u0001\u0000"+ - "\u0000\u0000\u0002`\u0001\u0000\u0000\u0000\u0002b\u0001\u0000\u0000\u0000"+ - "\u0002d\u0001\u0000\u0000\u0000\u0002f\u0001\u0000\u0000\u0000\u0002h"+ - "\u0001\u0000\u0000\u0000\u0002j\u0001\u0000\u0000\u0000\u0002l\u0001\u0000"+ - "\u0000\u0000\u0002n\u0001\u0000\u0000\u0000\u0002p\u0001\u0000\u0000\u0000"+ - "\u0002r\u0001\u0000\u0000\u0000\u0002t\u0001\u0000\u0000\u0000\u0002v"+ - "\u0001\u0000\u0000\u0000\u0002x\u0001\u0000\u0000\u0000\u0002z\u0001\u0000"+ - "\u0000\u0000\u0002|\u0001\u0000\u0000\u0000\u0002~\u0001\u0000\u0000\u0000"+ - "\u0002\u0080\u0001\u0000\u0000\u0000\u0002\u0082\u0001\u0000\u0000\u0000"+ - "\u0002\u0084\u0001\u0000\u0000\u0000\u0002\u0086\u0001\u0000\u0000\u0000"+ - "\u0002\u0088\u0001\u0000\u0000\u0000\u0002\u008a\u0001\u0000\u0000\u0000"+ - "\u0002\u008c\u0001\u0000\u0000\u0000\u0002\u008e\u0001\u0000\u0000\u0000"+ - "\u0002\u0090\u0001\u0000\u0000\u0000\u0002\u0092\u0001\u0000\u0000\u0000"+ - "\u0002\u0094\u0001\u0000\u0000\u0000\u0002\u0096\u0001\u0000\u0000\u0000"+ - "\u0002\u0098\u0001\u0000\u0000\u0000\u0002\u009a\u0001\u0000\u0000\u0000"+ - "\u0002\u009c\u0001\u0000\u0000\u0000\u0002\u009e\u0001\u0000\u0000\u0000"+ - "\u0003\u00a0\u0001\u0000\u0000\u0000\u0003\u00a2\u0001\u0000\u0000\u0000"+ - "\u0003\u00a4\u0001\u0000\u0000\u0000\u0003\u00a6\u0001\u0000\u0000\u0000"+ - "\u0003\u00a8\u0001\u0000\u0000\u0000\u0003\u00aa\u0001\u0000\u0000\u0000"+ - "\u0003\u00ac\u0001\u0000\u0000\u0000\u0003\u00ae\u0001\u0000\u0000\u0000"+ - "\u0003\u00b0\u0001\u0000\u0000\u0000\u0003\u00b2\u0001\u0000\u0000\u0000"+ - "\u0003\u00b6\u0001\u0000\u0000\u0000\u0003\u00b8\u0001\u0000\u0000\u0000"+ - "\u0003\u00ba\u0001\u0000\u0000\u0000\u0003\u00bc\u0001\u0000\u0000\u0000"+ - "\u0004\u00be\u0001\u0000\u0000\u0000\u0006\u00c8\u0001\u0000\u0000\u0000"+ - "\b\u00cf\u0001\u0000\u0000\u0000\n\u00d8\u0001\u0000\u0000\u0000\f\u00df"+ - "\u0001\u0000\u0000\u0000\u000e\u00e9\u0001\u0000\u0000\u0000\u0010\u00f0"+ - "\u0001\u0000\u0000\u0000\u0012\u00f7\u0001\u0000\u0000\u0000\u0014\u0105"+ - "\u0001\u0000\u0000\u0000\u0016\u010c\u0001\u0000\u0000\u0000\u0018\u0114"+ - "\u0001\u0000\u0000\u0000\u001a\u0120\u0001\u0000\u0000\u0000\u001c\u012a"+ - "\u0001\u0000\u0000\u0000\u001e\u0133\u0001\u0000\u0000\u0000 \u0139\u0001"+ - "\u0000\u0000\u0000\"\u0140\u0001\u0000\u0000\u0000$\u0147\u0001\u0000"+ - "\u0000\u0000&\u014f\u0001\u0000\u0000\u0000(\u0158\u0001\u0000\u0000\u0000"+ - "*\u015e\u0001\u0000\u0000\u0000,\u016f\u0001\u0000\u0000\u0000.\u017f"+ - "\u0001\u0000\u0000\u00000\u0185\u0001\u0000\u0000\u00002\u018a\u0001\u0000"+ - "\u0000\u00004\u018f\u0001\u0000\u0000\u00006\u0193\u0001\u0000\u0000\u0000"+ - "8\u0197\u0001\u0000\u0000\u0000:\u019b\u0001\u0000\u0000\u0000<\u019f"+ - "\u0001\u0000\u0000\u0000>\u01a1\u0001\u0000\u0000\u0000@\u01a3\u0001\u0000"+ - "\u0000\u0000B\u01a6\u0001\u0000\u0000\u0000D\u01a8\u0001\u0000\u0000\u0000"+ - "F\u01ce\u0001\u0000\u0000\u0000H\u01d1\u0001\u0000\u0000\u0000J\u01ff"+ - "\u0001\u0000\u0000\u0000L\u0201\u0001\u0000\u0000\u0000N\u0204\u0001\u0000"+ - "\u0000\u0000P\u0208\u0001\u0000\u0000\u0000R\u020c\u0001\u0000\u0000\u0000"+ - "T\u020e\u0001\u0000\u0000\u0000V\u0210\u0001\u0000\u0000\u0000X\u0215"+ - "\u0001\u0000\u0000\u0000Z\u0217\u0001\u0000\u0000\u0000\\\u021d\u0001"+ - "\u0000\u0000\u0000^\u0223\u0001\u0000\u0000\u0000`\u0228\u0001\u0000\u0000"+ - "\u0000b\u022a\u0001\u0000\u0000\u0000d\u022d\u0001\u0000\u0000\u0000f"+ - "\u0230\u0001\u0000\u0000\u0000h\u0235\u0001\u0000\u0000\u0000j\u0239\u0001"+ - "\u0000\u0000\u0000l\u023e\u0001\u0000\u0000\u0000n\u0244\u0001\u0000\u0000"+ - "\u0000p\u0247\u0001\u0000\u0000\u0000r\u0249\u0001\u0000\u0000\u0000t"+ - "\u024f\u0001\u0000\u0000\u0000v\u0251\u0001\u0000\u0000\u0000x\u0256\u0001"+ - "\u0000\u0000\u0000z\u025b\u0001\u0000\u0000\u0000|\u0265\u0001\u0000\u0000"+ - "\u0000~\u0268\u0001\u0000\u0000\u0000\u0080\u026b\u0001\u0000\u0000\u0000"+ - "\u0082\u026d\u0001\u0000\u0000\u0000\u0084\u0270\u0001\u0000\u0000\u0000"+ - "\u0086\u0272\u0001\u0000\u0000\u0000\u0088\u0275\u0001\u0000\u0000\u0000"+ - "\u008a\u0277\u0001\u0000\u0000\u0000\u008c\u0279\u0001\u0000\u0000\u0000"+ - "\u008e\u027b\u0001\u0000\u0000\u0000\u0090\u027d\u0001\u0000\u0000\u0000"+ - "\u0092\u027f\u0001\u0000\u0000\u0000\u0094\u0284\u0001\u0000\u0000\u0000"+ - "\u0096\u029a\u0001\u0000\u0000\u0000\u0098\u029c\u0001\u0000\u0000\u0000"+ - "\u009a\u02a7\u0001\u0000\u0000\u0000\u009c\u02ab\u0001\u0000\u0000\u0000"+ - "\u009e\u02af\u0001\u0000\u0000\u0000\u00a0\u02b3\u0001\u0000\u0000\u0000"+ - "\u00a2\u02b8\u0001\u0000\u0000\u0000\u00a4\u02be\u0001\u0000\u0000\u0000"+ - "\u00a6\u02c4\u0001\u0000\u0000\u0000\u00a8\u02c8\u0001\u0000\u0000\u0000"+ - "\u00aa\u02cc\u0001\u0000\u0000\u0000\u00ac\u02cf\u0001\u0000\u0000\u0000"+ - "\u00ae\u02d8\u0001\u0000\u0000\u0000\u00b0\u02db\u0001\u0000\u0000\u0000"+ - "\u00b2\u02e1\u0001\u0000\u0000\u0000\u00b4\u02ec\u0001\u0000\u0000\u0000"+ - "\u00b6\u02ee\u0001\u0000\u0000\u0000\u00b8\u02f0\u0001\u0000\u0000\u0000"+ - "\u00ba\u02f4\u0001\u0000\u0000\u0000\u00bc\u02f8\u0001\u0000\u0000\u0000"+ - "\u00be\u00bf\u0005d\u0000\u0000\u00bf\u00c0\u0005i\u0000\u0000\u00c0\u00c1"+ - "\u0005s\u0000\u0000\u00c1\u00c2\u0005s\u0000\u0000\u00c2\u00c3\u0005e"+ - "\u0000\u0000\u00c3\u00c4\u0005c\u0000\u0000\u00c4\u00c5\u0005t\u0000\u0000"+ - "\u00c5\u00c6\u0001\u0000\u0000\u0000\u00c6\u00c7\u0006\u0000\u0000\u0000"+ - "\u00c7\u0005\u0001\u0000\u0000\u0000\u00c8\u00c9\u0005d\u0000\u0000\u00c9"+ - "\u00ca\u0005r\u0000\u0000\u00ca\u00cb\u0005o\u0000\u0000\u00cb\u00cc\u0005"+ - "p\u0000\u0000\u00cc\u00cd\u0001\u0000\u0000\u0000\u00cd\u00ce\u0006\u0001"+ - "\u0001\u0000\u00ce\u0007\u0001\u0000\u0000\u0000\u00cf\u00d0\u0005e\u0000"+ - "\u0000\u00d0\u00d1\u0005n\u0000\u0000\u00d1\u00d2\u0005r\u0000\u0000\u00d2"+ - "\u00d3\u0005i\u0000\u0000\u00d3\u00d4\u0005c\u0000\u0000\u00d4\u00d5\u0005"+ - "h\u0000\u0000\u00d5\u00d6\u0001\u0000\u0000\u0000\u00d6\u00d7\u0006\u0002"+ - "\u0001\u0000\u00d7\t\u0001\u0000\u0000\u0000\u00d8\u00d9\u0005e\u0000"+ - "\u0000\u00d9\u00da\u0005v\u0000\u0000\u00da\u00db\u0005a\u0000\u0000\u00db"+ - "\u00dc\u0005l\u0000\u0000\u00dc\u00dd\u0001\u0000\u0000\u0000\u00dd\u00de"+ - "\u0006\u0003\u0000\u0000\u00de\u000b\u0001\u0000\u0000\u0000\u00df\u00e0"+ - "\u0005e\u0000\u0000\u00e0\u00e1\u0005x\u0000\u0000\u00e1\u00e2\u0005p"+ - "\u0000\u0000\u00e2\u00e3\u0005l\u0000\u0000\u00e3\u00e4\u0005a\u0000\u0000"+ - "\u00e4\u00e5\u0005i\u0000\u0000\u00e5\u00e6\u0005n\u0000\u0000\u00e6\u00e7"+ - "\u0001\u0000\u0000\u0000\u00e7\u00e8\u0006\u0004\u0002\u0000\u00e8\r\u0001"+ - "\u0000\u0000\u0000\u00e9\u00ea\u0005f\u0000\u0000\u00ea\u00eb\u0005r\u0000"+ - "\u0000\u00eb\u00ec\u0005o\u0000\u0000\u00ec\u00ed\u0005m\u0000\u0000\u00ed"+ - "\u00ee\u0001\u0000\u0000\u0000\u00ee\u00ef\u0006\u0005\u0001\u0000\u00ef"+ - "\u000f\u0001\u0000\u0000\u0000\u00f0\u00f1\u0005g\u0000\u0000\u00f1\u00f2"+ - "\u0005r\u0000\u0000\u00f2\u00f3\u0005o\u0000\u0000\u00f3\u00f4\u0005k"+ - "\u0000\u0000\u00f4\u00f5\u0001\u0000\u0000\u0000\u00f5\u00f6\u0006\u0006"+ - "\u0000\u0000\u00f6\u0011\u0001\u0000\u0000\u0000\u00f7\u00f8\u0005i\u0000"+ - "\u0000\u00f8\u00f9\u0005n\u0000\u0000\u00f9\u00fa\u0005l\u0000\u0000\u00fa"+ - "\u00fb\u0005i\u0000\u0000\u00fb\u00fc\u0005n\u0000\u0000\u00fc\u00fd\u0005"+ - "e\u0000\u0000\u00fd\u00fe\u0005s\u0000\u0000\u00fe\u00ff\u0005t\u0000"+ - "\u0000\u00ff\u0100\u0005a\u0000\u0000\u0100\u0101\u0005t\u0000\u0000\u0101"+ - "\u0102\u0005s\u0000\u0000\u0102\u0103\u0001\u0000\u0000\u0000\u0103\u0104"+ - "\u0006\u0007\u0000\u0000\u0104\u0013\u0001\u0000\u0000\u0000\u0105\u0106"+ - "\u0005k\u0000\u0000\u0106\u0107\u0005e\u0000\u0000\u0107\u0108\u0005e"+ - "\u0000\u0000\u0108\u0109\u0005p\u0000\u0000\u0109\u010a\u0001\u0000\u0000"+ - "\u0000\u010a\u010b\u0006\b\u0001\u0000\u010b\u0015\u0001\u0000\u0000\u0000"+ - "\u010c\u010d\u0005l\u0000\u0000\u010d\u010e\u0005i\u0000\u0000\u010e\u010f"+ - "\u0005m\u0000\u0000\u010f\u0110\u0005i\u0000\u0000\u0110\u0111\u0005t"+ - "\u0000\u0000\u0111\u0112\u0001\u0000\u0000\u0000\u0112\u0113\u0006\t\u0000"+ - "\u0000\u0113\u0017\u0001\u0000\u0000\u0000\u0114\u0115\u0005m\u0000\u0000"+ - "\u0115\u0116\u0005v\u0000\u0000\u0116\u0117\u0005_\u0000\u0000\u0117\u0118"+ - "\u0005e\u0000\u0000\u0118\u0119\u0005x\u0000\u0000\u0119\u011a\u0005p"+ - "\u0000\u0000\u011a\u011b\u0005a\u0000\u0000\u011b\u011c\u0005n\u0000\u0000"+ - "\u011c\u011d\u0005d\u0000\u0000\u011d\u011e\u0001\u0000\u0000\u0000\u011e"+ - "\u011f\u0006\n\u0001\u0000\u011f\u0019\u0001\u0000\u0000\u0000\u0120\u0121"+ - "\u0005p\u0000\u0000\u0121\u0122\u0005r\u0000\u0000\u0122\u0123\u0005o"+ - "\u0000\u0000\u0123\u0124\u0005j\u0000\u0000\u0124\u0125\u0005e\u0000\u0000"+ - "\u0125\u0126\u0005c\u0000\u0000\u0126\u0127\u0005t\u0000\u0000\u0127\u0128"+ - "\u0001\u0000\u0000\u0000\u0128\u0129\u0006\u000b\u0001\u0000\u0129\u001b"+ - "\u0001\u0000\u0000\u0000\u012a\u012b\u0005r\u0000\u0000\u012b\u012c\u0005"+ - "e\u0000\u0000\u012c\u012d\u0005n\u0000\u0000\u012d\u012e\u0005a\u0000"+ - "\u0000\u012e\u012f\u0005m\u0000\u0000\u012f\u0130\u0005e\u0000\u0000\u0130"+ - "\u0131\u0001\u0000\u0000\u0000\u0131\u0132\u0006\f\u0001\u0000\u0132\u001d"+ - "\u0001\u0000\u0000\u0000\u0133\u0134\u0005r\u0000\u0000\u0134\u0135\u0005"+ - "o\u0000\u0000\u0135\u0136\u0005w\u0000\u0000\u0136\u0137\u0001\u0000\u0000"+ - "\u0000\u0137\u0138\u0006\r\u0000\u0000\u0138\u001f\u0001\u0000\u0000\u0000"+ - "\u0139\u013a\u0005s\u0000\u0000\u013a\u013b\u0005h\u0000\u0000\u013b\u013c"+ - "\u0005o\u0000\u0000\u013c\u013d\u0005w\u0000\u0000\u013d\u013e\u0001\u0000"+ - "\u0000\u0000\u013e\u013f\u0006\u000e\u0000\u0000\u013f!\u0001\u0000\u0000"+ - "\u0000\u0140\u0141\u0005s\u0000\u0000\u0141\u0142\u0005o\u0000\u0000\u0142"+ - "\u0143\u0005r\u0000\u0000\u0143\u0144\u0005t\u0000\u0000\u0144\u0145\u0001"+ - "\u0000\u0000\u0000\u0145\u0146\u0006\u000f\u0000\u0000\u0146#\u0001\u0000"+ - "\u0000\u0000\u0147\u0148\u0005s\u0000\u0000\u0148\u0149\u0005t\u0000\u0000"+ - "\u0149\u014a\u0005a\u0000\u0000\u014a\u014b\u0005t\u0000\u0000\u014b\u014c"+ - "\u0005s\u0000\u0000\u014c\u014d\u0001\u0000\u0000\u0000\u014d\u014e\u0006"+ - "\u0010\u0000\u0000\u014e%\u0001\u0000\u0000\u0000\u014f\u0150\u0005w\u0000"+ - "\u0000\u0150\u0151\u0005h\u0000\u0000\u0151\u0152\u0005e\u0000\u0000\u0152"+ - "\u0153\u0005r\u0000\u0000\u0153\u0154\u0005e\u0000\u0000\u0154\u0155\u0001"+ - "\u0000\u0000\u0000\u0155\u0156\u0006\u0011\u0000\u0000\u0156\'\u0001\u0000"+ - "\u0000\u0000\u0157\u0159\b\u0000\u0000\u0000\u0158\u0157\u0001\u0000\u0000"+ - "\u0000\u0159\u015a\u0001\u0000\u0000\u0000\u015a\u0158\u0001\u0000\u0000"+ - "\u0000\u015a\u015b\u0001\u0000\u0000\u0000\u015b\u015c\u0001\u0000\u0000"+ - "\u0000\u015c\u015d\u0006\u0012\u0000\u0000\u015d)\u0001\u0000\u0000\u0000"+ - "\u015e\u015f\u0005/\u0000\u0000\u015f\u0160\u0005/\u0000\u0000\u0160\u0164"+ - "\u0001\u0000\u0000\u0000\u0161\u0163\b\u0001\u0000\u0000\u0162\u0161\u0001"+ - "\u0000\u0000\u0000\u0163\u0166\u0001\u0000\u0000\u0000\u0164\u0162\u0001"+ - "\u0000\u0000\u0000\u0164\u0165\u0001\u0000\u0000\u0000\u0165\u0168\u0001"+ - "\u0000\u0000\u0000\u0166\u0164\u0001\u0000\u0000\u0000\u0167\u0169\u0005"+ - "\r\u0000\u0000\u0168\u0167\u0001\u0000\u0000\u0000\u0168\u0169\u0001\u0000"+ - "\u0000\u0000\u0169\u016b\u0001\u0000\u0000\u0000\u016a\u016c\u0005\n\u0000"+ - "\u0000\u016b\u016a\u0001\u0000\u0000\u0000\u016b\u016c\u0001\u0000\u0000"+ - "\u0000\u016c\u016d\u0001\u0000\u0000\u0000\u016d\u016e\u0006\u0013\u0003"+ - "\u0000\u016e+\u0001\u0000\u0000\u0000\u016f\u0170\u0005/\u0000\u0000\u0170"+ - "\u0171\u0005*\u0000\u0000\u0171\u0176\u0001\u0000\u0000\u0000\u0172\u0175"+ - "\u0003,\u0014\u0000\u0173\u0175\t\u0000\u0000\u0000\u0174\u0172\u0001"+ - "\u0000\u0000\u0000\u0174\u0173\u0001\u0000\u0000\u0000\u0175\u0178\u0001"+ - "\u0000\u0000\u0000\u0176\u0177\u0001\u0000\u0000\u0000\u0176\u0174\u0001"+ - "\u0000\u0000\u0000\u0177\u0179\u0001\u0000\u0000\u0000\u0178\u0176\u0001"+ - "\u0000\u0000\u0000\u0179\u017a\u0005*\u0000\u0000\u017a\u017b\u0005/\u0000"+ - "\u0000\u017b\u017c\u0001\u0000\u0000\u0000\u017c\u017d\u0006\u0014\u0003"+ - "\u0000\u017d-\u0001\u0000\u0000\u0000\u017e\u0180\u0007\u0002\u0000\u0000"+ - "\u017f\u017e\u0001\u0000\u0000\u0000\u0180\u0181\u0001\u0000\u0000\u0000"+ - "\u0181\u017f\u0001\u0000\u0000\u0000\u0181\u0182\u0001\u0000\u0000\u0000"+ - "\u0182\u0183\u0001\u0000\u0000\u0000\u0183\u0184\u0006\u0015\u0003\u0000"+ - "\u0184/\u0001\u0000\u0000\u0000\u0185\u0186\u0005[\u0000\u0000\u0186\u0187"+ - "\u0001\u0000\u0000\u0000\u0187\u0188\u0006\u0016\u0004\u0000\u0188\u0189"+ - "\u0006\u0016\u0005\u0000\u01891\u0001\u0000\u0000\u0000\u018a\u018b\u0005"+ - "|\u0000\u0000\u018b\u018c\u0001\u0000\u0000\u0000\u018c\u018d\u0006\u0017"+ - "\u0006\u0000\u018d\u018e\u0006\u0017\u0007\u0000\u018e3\u0001\u0000\u0000"+ - "\u0000\u018f\u0190\u0003.\u0015\u0000\u0190\u0191\u0001\u0000\u0000\u0000"+ - "\u0191\u0192\u0006\u0018\u0003\u0000\u01925\u0001\u0000\u0000\u0000\u0193"+ - "\u0194\u0003*\u0013\u0000\u0194\u0195\u0001\u0000\u0000\u0000\u0195\u0196"+ - "\u0006\u0019\u0003\u0000\u01967\u0001\u0000\u0000\u0000\u0197\u0198\u0003"+ - ",\u0014\u0000\u0198\u0199\u0001\u0000\u0000\u0000\u0199\u019a\u0006\u001a"+ - "\u0003\u0000\u019a9\u0001\u0000\u0000\u0000\u019b\u019c\u0005|\u0000\u0000"+ - "\u019c\u019d\u0001\u0000\u0000\u0000\u019d\u019e\u0006\u001b\u0007\u0000"+ - "\u019e;\u0001\u0000\u0000\u0000\u019f\u01a0\u0007\u0003\u0000\u0000\u01a0"+ - "=\u0001\u0000\u0000\u0000\u01a1\u01a2\u0007\u0004\u0000\u0000\u01a2?\u0001"+ - "\u0000\u0000\u0000\u01a3\u01a4\u0005\\\u0000\u0000\u01a4\u01a5\u0007\u0005"+ - "\u0000\u0000\u01a5A\u0001\u0000\u0000\u0000\u01a6\u01a7\b\u0006\u0000"+ - "\u0000\u01a7C\u0001\u0000\u0000\u0000\u01a8\u01aa\u0007\u0007\u0000\u0000"+ - "\u01a9\u01ab\u0007\b\u0000\u0000\u01aa\u01a9\u0001\u0000\u0000\u0000\u01aa"+ - "\u01ab\u0001\u0000\u0000\u0000\u01ab\u01ad\u0001\u0000\u0000\u0000\u01ac"+ - "\u01ae\u0003<\u001c\u0000\u01ad\u01ac\u0001\u0000\u0000\u0000\u01ae\u01af"+ - "\u0001\u0000\u0000\u0000\u01af\u01ad\u0001\u0000\u0000\u0000\u01af\u01b0"+ - "\u0001\u0000\u0000\u0000\u01b0E\u0001\u0000\u0000\u0000\u01b1\u01b6\u0005"+ - "\"\u0000\u0000\u01b2\u01b5\u0003@\u001e\u0000\u01b3\u01b5\u0003B\u001f"+ - "\u0000\u01b4\u01b2\u0001\u0000\u0000\u0000\u01b4\u01b3\u0001\u0000\u0000"+ - "\u0000\u01b5\u01b8\u0001\u0000\u0000\u0000\u01b6\u01b4\u0001\u0000\u0000"+ - "\u0000\u01b6\u01b7\u0001\u0000\u0000\u0000\u01b7\u01b9\u0001\u0000\u0000"+ - "\u0000\u01b8\u01b6\u0001\u0000\u0000\u0000\u01b9\u01cf\u0005\"\u0000\u0000"+ - "\u01ba\u01bb\u0005\"\u0000\u0000\u01bb\u01bc\u0005\"\u0000\u0000\u01bc"+ - "\u01bd\u0005\"\u0000\u0000\u01bd\u01c1\u0001\u0000\u0000\u0000\u01be\u01c0"+ - "\b\u0001\u0000\u0000\u01bf\u01be\u0001\u0000\u0000\u0000\u01c0\u01c3\u0001"+ - "\u0000\u0000\u0000\u01c1\u01c2\u0001\u0000\u0000\u0000\u01c1\u01bf\u0001"+ - "\u0000\u0000\u0000\u01c2\u01c4\u0001\u0000\u0000\u0000\u01c3\u01c1\u0001"+ - "\u0000\u0000\u0000\u01c4\u01c5\u0005\"\u0000\u0000\u01c5\u01c6\u0005\""+ - "\u0000\u0000\u01c6\u01c7\u0005\"\u0000\u0000\u01c7\u01c9\u0001\u0000\u0000"+ - "\u0000\u01c8\u01ca\u0005\"\u0000\u0000\u01c9\u01c8\u0001\u0000\u0000\u0000"+ - "\u01c9\u01ca\u0001\u0000\u0000\u0000\u01ca\u01cc\u0001\u0000\u0000\u0000"+ - "\u01cb\u01cd\u0005\"\u0000\u0000\u01cc\u01cb\u0001\u0000\u0000\u0000\u01cc"+ - "\u01cd\u0001\u0000\u0000\u0000\u01cd\u01cf\u0001\u0000\u0000\u0000\u01ce"+ - "\u01b1\u0001\u0000\u0000\u0000\u01ce\u01ba\u0001\u0000\u0000\u0000\u01cf"+ - "G\u0001\u0000\u0000\u0000\u01d0\u01d2\u0003<\u001c\u0000\u01d1\u01d0\u0001"+ - "\u0000\u0000\u0000\u01d2\u01d3\u0001\u0000\u0000\u0000\u01d3\u01d1\u0001"+ - "\u0000\u0000\u0000\u01d3\u01d4\u0001\u0000\u0000\u0000\u01d4I\u0001\u0000"+ - "\u0000\u0000\u01d5\u01d7\u0003<\u001c\u0000\u01d6\u01d5\u0001\u0000\u0000"+ - "\u0000\u01d7\u01d8\u0001\u0000\u0000\u0000\u01d8\u01d6\u0001\u0000\u0000"+ - "\u0000\u01d8\u01d9\u0001\u0000\u0000\u0000\u01d9\u01da\u0001\u0000\u0000"+ - "\u0000\u01da\u01de\u0003X*\u0000\u01db\u01dd\u0003<\u001c\u0000\u01dc"+ - "\u01db\u0001\u0000\u0000\u0000\u01dd\u01e0\u0001\u0000\u0000\u0000\u01de"+ - "\u01dc\u0001\u0000\u0000\u0000\u01de\u01df\u0001\u0000\u0000\u0000\u01df"+ - "\u0200\u0001\u0000\u0000\u0000\u01e0\u01de\u0001\u0000\u0000\u0000\u01e1"+ - "\u01e3\u0003X*\u0000\u01e2\u01e4\u0003<\u001c\u0000\u01e3\u01e2\u0001"+ - "\u0000\u0000\u0000\u01e4\u01e5\u0001\u0000\u0000\u0000\u01e5\u01e3\u0001"+ - "\u0000\u0000\u0000\u01e5\u01e6\u0001\u0000\u0000\u0000\u01e6\u0200\u0001"+ - "\u0000\u0000\u0000\u01e7\u01e9\u0003<\u001c\u0000\u01e8\u01e7\u0001\u0000"+ - "\u0000\u0000\u01e9\u01ea\u0001\u0000\u0000\u0000\u01ea\u01e8\u0001\u0000"+ - "\u0000\u0000\u01ea\u01eb\u0001\u0000\u0000\u0000\u01eb\u01f3\u0001\u0000"+ - "\u0000\u0000\u01ec\u01f0\u0003X*\u0000\u01ed\u01ef\u0003<\u001c\u0000"+ - "\u01ee\u01ed\u0001\u0000\u0000\u0000\u01ef\u01f2\u0001\u0000\u0000\u0000"+ - "\u01f0\u01ee\u0001\u0000\u0000\u0000\u01f0\u01f1\u0001\u0000\u0000\u0000"+ - "\u01f1\u01f4\u0001\u0000\u0000\u0000\u01f2\u01f0\u0001\u0000\u0000\u0000"+ - "\u01f3\u01ec\u0001\u0000\u0000\u0000\u01f3\u01f4\u0001\u0000\u0000\u0000"+ - "\u01f4\u01f5\u0001\u0000\u0000\u0000\u01f5\u01f6\u0003D \u0000\u01f6\u0200"+ - "\u0001\u0000\u0000\u0000\u01f7\u01f9\u0003X*\u0000\u01f8\u01fa\u0003<"+ - "\u001c\u0000\u01f9\u01f8\u0001\u0000\u0000\u0000\u01fa\u01fb\u0001\u0000"+ - "\u0000\u0000\u01fb\u01f9\u0001\u0000\u0000\u0000\u01fb\u01fc\u0001\u0000"+ - "\u0000\u0000\u01fc\u01fd\u0001\u0000\u0000\u0000\u01fd\u01fe\u0003D \u0000"+ - "\u01fe\u0200\u0001\u0000\u0000\u0000\u01ff\u01d6\u0001\u0000\u0000\u0000"+ - "\u01ff\u01e1\u0001\u0000\u0000\u0000\u01ff\u01e8\u0001\u0000\u0000\u0000"+ - "\u01ff\u01f7\u0001\u0000\u0000\u0000\u0200K\u0001\u0000\u0000\u0000\u0201"+ - "\u0202\u0005b\u0000\u0000\u0202\u0203\u0005y\u0000\u0000\u0203M\u0001"+ - "\u0000\u0000\u0000\u0204\u0205\u0005a\u0000\u0000\u0205\u0206\u0005n\u0000"+ - "\u0000\u0206\u0207\u0005d\u0000\u0000\u0207O\u0001\u0000\u0000\u0000\u0208"+ - "\u0209\u0005a\u0000\u0000\u0209\u020a\u0005s\u0000\u0000\u020a\u020b\u0005"+ - "c\u0000\u0000\u020bQ\u0001\u0000\u0000\u0000\u020c\u020d\u0005=\u0000"+ - "\u0000\u020dS\u0001\u0000\u0000\u0000\u020e\u020f\u0005,\u0000\u0000\u020f"+ - "U\u0001\u0000\u0000\u0000\u0210\u0211\u0005d\u0000\u0000\u0211\u0212\u0005"+ - "e\u0000\u0000\u0212\u0213\u0005s\u0000\u0000\u0213\u0214\u0005c\u0000"+ - "\u0000\u0214W\u0001\u0000\u0000\u0000\u0215\u0216\u0005.\u0000\u0000\u0216"+ - "Y\u0001\u0000\u0000\u0000\u0217\u0218\u0005f\u0000\u0000\u0218\u0219\u0005"+ - "a\u0000\u0000\u0219\u021a\u0005l\u0000\u0000\u021a\u021b\u0005s\u0000"+ - "\u0000\u021b\u021c\u0005e\u0000\u0000\u021c[\u0001\u0000\u0000\u0000\u021d"+ - "\u021e\u0005f\u0000\u0000\u021e\u021f\u0005i\u0000\u0000\u021f\u0220\u0005"+ - "r\u0000\u0000\u0220\u0221\u0005s\u0000\u0000\u0221\u0222\u0005t\u0000"+ - "\u0000\u0222]\u0001\u0000\u0000\u0000\u0223\u0224\u0005l\u0000\u0000\u0224"+ - "\u0225\u0005a\u0000\u0000\u0225\u0226\u0005s\u0000\u0000\u0226\u0227\u0005"+ - "t\u0000\u0000\u0227_\u0001\u0000\u0000\u0000\u0228\u0229\u0005(\u0000"+ - "\u0000\u0229a\u0001\u0000\u0000\u0000\u022a\u022b\u0005i\u0000\u0000\u022b"+ - "\u022c\u0005n\u0000\u0000\u022cc\u0001\u0000\u0000\u0000\u022d\u022e\u0005"+ - "i\u0000\u0000\u022e\u022f\u0005s\u0000\u0000\u022fe\u0001\u0000\u0000"+ - "\u0000\u0230\u0231\u0005l\u0000\u0000\u0231\u0232\u0005i\u0000\u0000\u0232"+ - "\u0233\u0005k\u0000\u0000\u0233\u0234\u0005e\u0000\u0000\u0234g\u0001"+ - "\u0000\u0000\u0000\u0235\u0236\u0005n\u0000\u0000\u0236\u0237\u0005o\u0000"+ - "\u0000\u0237\u0238\u0005t\u0000\u0000\u0238i\u0001\u0000\u0000\u0000\u0239"+ - "\u023a\u0005n\u0000\u0000\u023a\u023b\u0005u\u0000\u0000\u023b\u023c\u0005"+ - "l\u0000\u0000\u023c\u023d\u0005l\u0000\u0000\u023dk\u0001\u0000\u0000"+ - "\u0000\u023e\u023f\u0005n\u0000\u0000\u023f\u0240\u0005u\u0000\u0000\u0240"+ - "\u0241\u0005l\u0000\u0000\u0241\u0242\u0005l\u0000\u0000\u0242\u0243\u0005"+ - "s\u0000\u0000\u0243m\u0001\u0000\u0000\u0000\u0244\u0245\u0005o\u0000"+ - "\u0000\u0245\u0246\u0005r\u0000\u0000\u0246o\u0001\u0000\u0000\u0000\u0247"+ - "\u0248\u0005?\u0000\u0000\u0248q\u0001\u0000\u0000\u0000\u0249\u024a\u0005"+ - "r\u0000\u0000\u024a\u024b\u0005l\u0000\u0000\u024b\u024c\u0005i\u0000"+ - "\u0000\u024c\u024d\u0005k\u0000\u0000\u024d\u024e\u0005e\u0000\u0000\u024e"+ - "s\u0001\u0000\u0000\u0000\u024f\u0250\u0005)\u0000\u0000\u0250u\u0001"+ - "\u0000\u0000\u0000\u0251\u0252\u0005t\u0000\u0000\u0252\u0253\u0005r\u0000"+ - "\u0000\u0253\u0254\u0005u\u0000\u0000\u0254\u0255\u0005e\u0000\u0000\u0255"+ - "w\u0001\u0000\u0000\u0000\u0256\u0257\u0005i\u0000\u0000\u0257\u0258\u0005"+ - "n\u0000\u0000\u0258\u0259\u0005f\u0000\u0000\u0259\u025a\u0005o\u0000"+ - "\u0000\u025ay\u0001\u0000\u0000\u0000\u025b\u025c\u0005f\u0000\u0000\u025c"+ - "\u025d\u0005u\u0000\u0000\u025d\u025e\u0005n\u0000\u0000\u025e\u025f\u0005"+ - "c\u0000\u0000\u025f\u0260\u0005t\u0000\u0000\u0260\u0261\u0005i\u0000"+ - "\u0000\u0261\u0262\u0005o\u0000\u0000\u0262\u0263\u0005n\u0000\u0000\u0263"+ - "\u0264\u0005s\u0000\u0000\u0264{\u0001\u0000\u0000\u0000\u0265\u0266\u0005"+ - "=\u0000\u0000\u0266\u0267\u0005=\u0000\u0000\u0267}\u0001\u0000\u0000"+ - "\u0000\u0268\u0269\u0005!\u0000\u0000\u0269\u026a\u0005=\u0000\u0000\u026a"+ - "\u007f\u0001\u0000\u0000\u0000\u026b\u026c\u0005<\u0000\u0000\u026c\u0081"+ - "\u0001\u0000\u0000\u0000\u026d\u026e\u0005<\u0000\u0000\u026e\u026f\u0005"+ - "=\u0000\u0000\u026f\u0083\u0001\u0000\u0000\u0000\u0270\u0271\u0005>\u0000"+ - "\u0000\u0271\u0085\u0001\u0000\u0000\u0000\u0272\u0273\u0005>\u0000\u0000"+ - "\u0273\u0274\u0005=\u0000\u0000\u0274\u0087\u0001\u0000\u0000\u0000\u0275"+ - "\u0276\u0005+\u0000\u0000\u0276\u0089\u0001\u0000\u0000\u0000\u0277\u0278"+ - "\u0005-\u0000\u0000\u0278\u008b\u0001\u0000\u0000\u0000\u0279\u027a\u0005"+ - "*\u0000\u0000\u027a\u008d\u0001\u0000\u0000\u0000\u027b\u027c\u0005/\u0000"+ - "\u0000\u027c\u008f\u0001\u0000\u0000\u0000\u027d\u027e\u0005%\u0000\u0000"+ - "\u027e\u0091\u0001\u0000\u0000\u0000\u027f\u0280\u0005[\u0000\u0000\u0280"+ - "\u0281\u0001\u0000\u0000\u0000\u0281\u0282\u0006G\u0000\u0000\u0282\u0283"+ - "\u0006G\u0000\u0000\u0283\u0093\u0001\u0000\u0000\u0000\u0284\u0285\u0005"+ - "]\u0000\u0000\u0285\u0286\u0001\u0000\u0000\u0000\u0286\u0287\u0006H\u0007"+ - "\u0000\u0287\u0288\u0006H\u0007\u0000\u0288\u0095\u0001\u0000\u0000\u0000"+ - "\u0289\u028f\u0003>\u001d\u0000\u028a\u028e\u0003>\u001d\u0000\u028b\u028e"+ - "\u0003<\u001c\u0000\u028c\u028e\u0005_\u0000\u0000\u028d\u028a\u0001\u0000"+ - "\u0000\u0000\u028d\u028b\u0001\u0000\u0000\u0000\u028d\u028c\u0001\u0000"+ - "\u0000\u0000\u028e\u0291\u0001\u0000\u0000\u0000\u028f\u028d\u0001\u0000"+ - "\u0000\u0000\u028f\u0290\u0001\u0000\u0000\u0000\u0290\u029b\u0001\u0000"+ - "\u0000\u0000\u0291\u028f\u0001\u0000\u0000\u0000\u0292\u0296\u0007\t\u0000"+ - "\u0000\u0293\u0297\u0003>\u001d\u0000\u0294\u0297\u0003<\u001c\u0000\u0295"+ - "\u0297\u0005_\u0000\u0000\u0296\u0293\u0001\u0000\u0000\u0000\u0296\u0294"+ - "\u0001\u0000\u0000\u0000\u0296\u0295\u0001\u0000\u0000\u0000\u0297\u0298"+ - "\u0001\u0000\u0000\u0000\u0298\u0296\u0001\u0000\u0000\u0000\u0298\u0299"+ - "\u0001\u0000\u0000\u0000\u0299\u029b\u0001\u0000\u0000\u0000\u029a\u0289"+ - "\u0001\u0000\u0000\u0000\u029a\u0292\u0001\u0000\u0000\u0000\u029b\u0097"+ - "\u0001\u0000\u0000\u0000\u029c\u02a2\u0005`\u0000\u0000\u029d\u02a1\b"+ - "\n\u0000\u0000\u029e\u029f\u0005`\u0000\u0000\u029f\u02a1\u0005`\u0000"+ - "\u0000\u02a0\u029d\u0001\u0000\u0000\u0000\u02a0\u029e\u0001\u0000\u0000"+ - "\u0000\u02a1\u02a4\u0001\u0000\u0000\u0000\u02a2\u02a0\u0001\u0000\u0000"+ - "\u0000\u02a2\u02a3\u0001\u0000\u0000\u0000\u02a3\u02a5\u0001\u0000\u0000"+ - "\u0000\u02a4\u02a2\u0001\u0000\u0000\u0000\u02a5\u02a6\u0005`\u0000\u0000"+ - "\u02a6\u0099\u0001\u0000\u0000\u0000\u02a7\u02a8\u0003*\u0013\u0000\u02a8"+ - "\u02a9\u0001\u0000\u0000\u0000\u02a9\u02aa\u0006K\u0003\u0000\u02aa\u009b"+ - "\u0001\u0000\u0000\u0000\u02ab\u02ac\u0003,\u0014\u0000\u02ac\u02ad\u0001"+ - "\u0000\u0000\u0000\u02ad\u02ae\u0006L\u0003\u0000\u02ae\u009d\u0001\u0000"+ - "\u0000\u0000\u02af\u02b0\u0003.\u0015\u0000\u02b0\u02b1\u0001\u0000\u0000"+ - "\u0000\u02b1\u02b2\u0006M\u0003\u0000\u02b2\u009f\u0001\u0000\u0000\u0000"+ - "\u02b3\u02b4\u0005|\u0000\u0000\u02b4\u02b5\u0001\u0000\u0000\u0000\u02b5"+ - "\u02b6\u0006N\u0006\u0000\u02b6\u02b7\u0006N\u0007\u0000\u02b7\u00a1\u0001"+ - "\u0000\u0000\u0000\u02b8\u02b9\u0005[\u0000\u0000\u02b9\u02ba\u0001\u0000"+ - "\u0000\u0000\u02ba\u02bb\u0006O\u0004\u0000\u02bb\u02bc\u0006O\u0001\u0000"+ - "\u02bc\u02bd\u0006O\u0001\u0000\u02bd\u00a3\u0001\u0000\u0000\u0000\u02be"+ - "\u02bf\u0005]\u0000\u0000\u02bf\u02c0\u0001\u0000\u0000\u0000\u02c0\u02c1"+ - "\u0006P\u0007\u0000\u02c1\u02c2\u0006P\u0007\u0000\u02c2\u02c3\u0006P"+ - "\b\u0000\u02c3\u00a5\u0001\u0000\u0000\u0000\u02c4\u02c5\u0005,\u0000"+ - "\u0000\u02c5\u02c6\u0001\u0000\u0000\u0000\u02c6\u02c7\u0006Q\t\u0000"+ - "\u02c7\u00a7\u0001\u0000\u0000\u0000\u02c8\u02c9\u0005=\u0000\u0000\u02c9"+ - "\u02ca\u0001\u0000\u0000\u0000\u02ca\u02cb\u0006R\n\u0000\u02cb\u00a9"+ - "\u0001\u0000\u0000\u0000\u02cc\u02cd\u0005a\u0000\u0000\u02cd\u02ce\u0005"+ - "s\u0000\u0000\u02ce\u00ab\u0001\u0000\u0000\u0000\u02cf\u02d0\u0005m\u0000"+ - "\u0000\u02d0\u02d1\u0005e\u0000\u0000\u02d1\u02d2\u0005t\u0000\u0000\u02d2"+ - "\u02d3\u0005a\u0000\u0000\u02d3\u02d4\u0005d\u0000\u0000\u02d4\u02d5\u0005"+ - "a\u0000\u0000\u02d5\u02d6\u0005t\u0000\u0000\u02d6\u02d7\u0005a\u0000"+ - "\u0000\u02d7\u00ad\u0001\u0000\u0000\u0000\u02d8\u02d9\u0005o\u0000\u0000"+ - "\u02d9\u02da\u0005n\u0000\u0000\u02da\u00af\u0001\u0000\u0000\u0000\u02db"+ - "\u02dc\u0005w\u0000\u0000\u02dc\u02dd\u0005i\u0000\u0000\u02dd\u02de\u0005"+ - "t\u0000\u0000\u02de\u02df\u0005h\u0000\u0000\u02df\u00b1\u0001\u0000\u0000"+ - "\u0000\u02e0\u02e2\u0003\u00b4X\u0000\u02e1\u02e0\u0001\u0000\u0000\u0000"+ - "\u02e2\u02e3\u0001\u0000\u0000\u0000\u02e3\u02e1\u0001\u0000\u0000\u0000"+ - "\u02e3\u02e4\u0001\u0000\u0000\u0000\u02e4\u00b3\u0001\u0000\u0000\u0000"+ - "\u02e5\u02e7\b\u000b\u0000\u0000\u02e6\u02e5\u0001\u0000\u0000\u0000\u02e7"+ - "\u02e8\u0001\u0000\u0000\u0000\u02e8\u02e6\u0001\u0000\u0000\u0000\u02e8"+ - "\u02e9\u0001\u0000\u0000\u0000\u02e9\u02ed\u0001\u0000\u0000\u0000\u02ea"+ - "\u02eb\u0005/\u0000\u0000\u02eb\u02ed\b\f\u0000\u0000\u02ec\u02e6\u0001"+ - "\u0000\u0000\u0000\u02ec\u02ea\u0001\u0000\u0000\u0000\u02ed\u00b5\u0001"+ - "\u0000\u0000\u0000\u02ee\u02ef\u0003\u0098J\u0000\u02ef\u00b7\u0001\u0000"+ - "\u0000\u0000\u02f0\u02f1\u0003*\u0013\u0000\u02f1\u02f2\u0001\u0000\u0000"+ - "\u0000\u02f2\u02f3\u0006Z\u0003\u0000\u02f3\u00b9\u0001\u0000\u0000\u0000"+ - "\u02f4\u02f5\u0003,\u0014\u0000\u02f5\u02f6\u0001\u0000\u0000\u0000\u02f6"+ - "\u02f7\u0006[\u0003\u0000\u02f7\u00bb\u0001\u0000\u0000\u0000\u02f8\u02f9"+ - "\u0003.\u0015\u0000\u02f9\u02fa\u0001\u0000\u0000\u0000\u02fa\u02fb\u0006"+ - "\\\u0003\u0000\u02fb\u00bd\u0001\u0000\u0000\u0000&\u0000\u0001\u0002"+ - "\u0003\u015a\u0164\u0168\u016b\u0174\u0176\u0181\u01aa\u01af\u01b4\u01b6"+ - "\u01c1\u01c9\u01cc\u01ce\u01d3\u01d8\u01de\u01e5\u01ea\u01f0\u01f3\u01fb"+ - "\u01ff\u028d\u028f\u0296\u0298\u029a\u02a0\u02a2\u02e3\u02e8\u02ec\u000b"+ - "\u0005\u0002\u0000\u0005\u0003\u0000\u0005\u0001\u0000\u0000\u0001\u0000"+ - "\u0007A\u0000\u0005\u0000\u0000\u0007\u001a\u0000\u0004\u0000\u0000\u0007"+ - "B\u0000\u0007\"\u0000\u0007!\u0000"; + "\u0001\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011"+ + "\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0012\u0004\u0012\u01c3\b\u0012"+ + "\u000b\u0012\f\u0012\u01c4\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013"+ + "\u0001\u0013\u0001\u0013\u0005\u0013\u01cd\b\u0013\n\u0013\f\u0013\u01d0"+ + "\t\u0013\u0001\u0013\u0003\u0013\u01d3\b\u0013\u0001\u0013\u0003\u0013"+ + "\u01d6\b\u0013\u0001\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001\u0014"+ + "\u0001\u0014\u0001\u0014\u0005\u0014\u01df\b\u0014\n\u0014\f\u0014\u01e2"+ + "\t\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001"+ + "\u0015\u0004\u0015\u01ea\b\u0015\u000b\u0015\f\u0015\u01eb\u0001\u0015"+ + "\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016"+ + "\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0018"+ + "\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0019\u0001\u0019\u0001\u0019"+ + "\u0001\u0019\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001b"+ + "\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001c\u0001\u001c\u0001\u001d"+ + "\u0001\u001d\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001f\u0001\u001f"+ + "\u0001 \u0001 \u0003 \u0215\b \u0001 \u0004 \u0218\b \u000b \f \u0219"+ + "\u0001!\u0001!\u0001\"\u0001\"\u0001#\u0001#\u0001#\u0003#\u0223\b#\u0001"+ + "$\u0001$\u0001%\u0001%\u0001%\u0003%\u022a\b%\u0001&\u0001&\u0001&\u0005"+ + "&\u022f\b&\n&\f&\u0232\t&\u0001&\u0001&\u0001&\u0001&\u0001&\u0001&\u0005"+ + "&\u023a\b&\n&\f&\u023d\t&\u0001&\u0001&\u0001&\u0001&\u0001&\u0003&\u0244"+ + "\b&\u0001&\u0003&\u0247\b&\u0003&\u0249\b&\u0001\'\u0004\'\u024c\b\'\u000b"+ + "\'\f\'\u024d\u0001(\u0004(\u0251\b(\u000b(\f(\u0252\u0001(\u0001(\u0005"+ + "(\u0257\b(\n(\f(\u025a\t(\u0001(\u0001(\u0004(\u025e\b(\u000b(\f(\u025f"+ + "\u0001(\u0004(\u0263\b(\u000b(\f(\u0264\u0001(\u0001(\u0005(\u0269\b("+ + "\n(\f(\u026c\t(\u0003(\u026e\b(\u0001(\u0001(\u0001(\u0001(\u0004(\u0274"+ + "\b(\u000b(\f(\u0275\u0001(\u0001(\u0003(\u027a\b(\u0001)\u0001)\u0001"+ + ")\u0001*\u0001*\u0001*\u0001*\u0001+\u0001+\u0001+\u0001+\u0001,\u0001"+ + ",\u0001-\u0001-\u0001.\u0001.\u0001.\u0001.\u0001.\u0001/\u0001/\u0001"+ + "0\u00010\u00010\u00010\u00010\u00010\u00011\u00011\u00011\u00011\u0001"+ + "1\u00011\u00012\u00012\u00012\u00012\u00012\u00013\u00013\u00014\u0001"+ + "4\u00014\u00015\u00015\u00015\u00016\u00016\u00016\u00016\u00016\u0001"+ + "7\u00017\u00017\u00017\u00018\u00018\u00018\u00018\u00018\u00019\u0001"+ + "9\u00019\u00019\u00019\u00019\u0001:\u0001:\u0001:\u0001;\u0001;\u0001"+ + "<\u0001<\u0001<\u0001<\u0001<\u0001<\u0001=\u0001=\u0001>\u0001>\u0001"+ + ">\u0001>\u0001>\u0001?\u0001?\u0001?\u0001@\u0001@\u0001@\u0001A\u0001"+ + "A\u0001B\u0001B\u0001B\u0001C\u0001C\u0001D\u0001D\u0001D\u0001E\u0001"+ + "E\u0001F\u0001F\u0001G\u0001G\u0001H\u0001H\u0001I\u0001I\u0001J\u0001"+ + "J\u0001J\u0001J\u0001J\u0001K\u0001K\u0001K\u0001K\u0001K\u0001L\u0001"+ + "L\u0005L\u02f7\bL\nL\fL\u02fa\tL\u0001L\u0001L\u0003L\u02fe\bL\u0001L"+ + "\u0004L\u0301\bL\u000bL\fL\u0302\u0003L\u0305\bL\u0001M\u0001M\u0004M"+ + "\u0309\bM\u000bM\fM\u030a\u0001M\u0001M\u0001N\u0001N\u0001N\u0001N\u0001"+ + "O\u0001O\u0001O\u0001O\u0001P\u0001P\u0001P\u0001P\u0001Q\u0001Q\u0001"+ + "Q\u0001Q\u0001Q\u0001R\u0001R\u0001R\u0001R\u0001R\u0001R\u0001S\u0001"+ + "S\u0001S\u0001S\u0001S\u0001S\u0001T\u0001T\u0001T\u0001T\u0001U\u0001"+ + "U\u0001U\u0001U\u0001V\u0001V\u0001V\u0001V\u0001V\u0001V\u0001V\u0001"+ + "V\u0001V\u0001W\u0001W\u0001W\u0003W\u0340\bW\u0001X\u0004X\u0343\bX\u000b"+ + "X\fX\u0344\u0001Y\u0001Y\u0001Y\u0001Y\u0001Z\u0001Z\u0001Z\u0001Z\u0001"+ + "[\u0001[\u0001[\u0001[\u0001\\\u0001\\\u0001\\\u0001\\\u0001]\u0001]\u0001"+ + "]\u0001]\u0001]\u0001^\u0001^\u0001^\u0001^\u0001_\u0001_\u0001_\u0001"+ + "_\u0001`\u0001`\u0001`\u0001`\u0003`\u0368\b`\u0001a\u0001a\u0003a\u036c"+ + "\ba\u0001a\u0005a\u036f\ba\na\fa\u0372\ta\u0001a\u0001a\u0003a\u0376\b"+ + "a\u0001a\u0004a\u0379\ba\u000ba\fa\u037a\u0003a\u037d\ba\u0001b\u0001"+ + "b\u0001b\u0001b\u0001c\u0001c\u0001c\u0001c\u0001d\u0001d\u0001d\u0001"+ + "d\u0001e\u0001e\u0001e\u0001e\u0001f\u0001f\u0001f\u0001f\u0001f\u0001"+ + "g\u0001g\u0001g\u0001g\u0001h\u0001h\u0001h\u0001h\u0001i\u0001i\u0001"+ + "i\u0001i\u0001j\u0001j\u0001j\u0001k\u0001k\u0001k\u0001k\u0001l\u0001"+ + "l\u0001l\u0001l\u0001m\u0001m\u0001m\u0001m\u0001n\u0001n\u0001n\u0001"+ + "n\u0001o\u0001o\u0001o\u0001o\u0001p\u0001p\u0001p\u0001p\u0001p\u0001"+ + "q\u0001q\u0001q\u0001q\u0001q\u0001r\u0001r\u0001r\u0001r\u0001r\u0001"+ + "r\u0001r\u0001s\u0001s\u0001s\u0001s\u0001t\u0001t\u0001t\u0001t\u0001"+ + "u\u0001u\u0001u\u0001u\u0001v\u0001v\u0001v\u0001v\u0001w\u0001w\u0001"+ + "w\u0001w\u0001x\u0001x\u0001x\u0001x\u0001x\u0001x\u0001y\u0001y\u0001"+ + "y\u0001y\u0001z\u0001z\u0001z\u0001z\u0001{\u0001{\u0001{\u0001{\u0001"+ + "|\u0001|\u0001|\u0001|\u0001}\u0001}\u0001}\u0001}\u0001~\u0001~\u0001"+ + "~\u0001~\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u0080\u0001"+ + "\u0080\u0001\u0080\u0001\u0080\u0001\u0081\u0001\u0081\u0001\u0081\u0001"+ + "\u0081\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0082\u0001"+ + "\u0083\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0084\u0001\u0084\u0001"+ + "\u0084\u0001\u0084\u0001\u0085\u0001\u0085\u0001\u0085\u0001\u0085\u0001"+ + "\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0087\u0001\u0087\u0001"+ + "\u0087\u0001\u0087\u0001\u0088\u0001\u0088\u0001\u0088\u0001\u0088\u0001"+ + "\u0089\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u008a\u0001"+ + "\u008a\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008b\u0001\u008b\u0001"+ + "\u008b\u0001\u008b\u0001\u008b\u0001\u008b\u0001\u008b\u0001\u008b\u0001"+ + "\u008b\u0001\u008b\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008c\u0001"+ + "\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008e\u0001\u008e\u0001"+ + "\u008e\u0001\u008e\u0002\u01e0\u023b\u0000\u008f\n\u0001\f\u0002\u000e"+ + "\u0003\u0010\u0004\u0012\u0005\u0014\u0006\u0016\u0007\u0018\b\u001a\t"+ + "\u001c\n\u001e\u000b \f\"\r$\u000e&\u000f(\u0010*\u0011,\u0012.\u0013"+ + "0\u00142\u00154\u00166\u00008\u0000:\u0017<\u0018>\u0019@\u001aB\u0000"+ + "D\u0000F\u0000H\u0000J\u0000L\u0000N\u0000P\u0000R\u0000T\u0000V\u001b"+ + "X\u001cZ\u001d\\\u001e^\u001f` b!d\"f#h$j%l&n\'p(r)t*v+x,z-|.~/\u0080"+ + "0\u00821\u00842\u00863\u00884\u008a5\u008c6\u008e7\u00908\u00929\u0094"+ + ":\u0096;\u0098<\u009a=\u009c>\u009e?\u00a0@\u00a2A\u00a4B\u00a6C\u00a8"+ + "D\u00aaE\u00ac\u0000\u00ae\u0000\u00b0\u0000\u00b2\u0000\u00b4\u0000\u00b6"+ + "F\u00b8\u0000\u00baG\u00bc\u0000\u00beH\u00c0I\u00c2J\u00c4\u0000\u00c6"+ + "\u0000\u00c8\u0000\u00ca\u0000\u00ccK\u00ce\u0000\u00d0L\u00d2M\u00d4"+ + "N\u00d6\u0000\u00d8\u0000\u00da\u0000\u00dc\u0000\u00deO\u00e0\u0000\u00e2"+ + "\u0000\u00e4P\u00e6Q\u00e8R\u00ea\u0000\u00ecS\u00eeT\u00f0\u0000\u00f2"+ + "\u0000\u00f4U\u00f6V\u00f8W\u00fa\u0000\u00fc\u0000\u00fe\u0000\u0100"+ + "\u0000\u0102\u0000\u0104\u0000\u0106\u0000\u0108X\u010aY\u010cZ\u010e"+ + "\u0000\u0110\u0000\u0112\u0000\u0114\u0000\u0116[\u0118\\\u011a]\u011c"+ + "\u0000\u011e^\u0120_\u0122`\u0124a\u0126b\n\u0000\u0001\u0002\u0003\u0004"+ + "\u0005\u0006\u0007\b\t\f\u0006\u0000\t\n\r\r //[[]]\u0002\u0000\n\n\r"+ + "\r\u0003\u0000\t\n\r\r \u0001\u000009\u0002\u0000AZaz\u0005\u0000\"\""+ + "\\\\nnrrtt\u0004\u0000\n\n\r\r\"\"\\\\\u0002\u0000EEee\u0002\u0000++-"+ + "-\u0001\u0000``\n\u0000\t\n\r\r ,,//==[[]]``||\u0002\u0000**//\u0458"+ + "\u0000\n\u0001\u0000\u0000\u0000\u0000\f\u0001\u0000\u0000\u0000\u0000"+ + "\u000e\u0001\u0000\u0000\u0000\u0000\u0010\u0001\u0000\u0000\u0000\u0000"+ + "\u0012\u0001\u0000\u0000\u0000\u0000\u0014\u0001\u0000\u0000\u0000\u0000"+ + "\u0016\u0001\u0000\u0000\u0000\u0000\u0018\u0001\u0000\u0000\u0000\u0000"+ + "\u001a\u0001\u0000\u0000\u0000\u0000\u001c\u0001\u0000\u0000\u0000\u0000"+ + "\u001e\u0001\u0000\u0000\u0000\u0000 \u0001\u0000\u0000\u0000\u0000\""+ + "\u0001\u0000\u0000\u0000\u0000$\u0001\u0000\u0000\u0000\u0000&\u0001\u0000"+ + "\u0000\u0000\u0000(\u0001\u0000\u0000\u0000\u0000*\u0001\u0000\u0000\u0000"+ + "\u0000,\u0001\u0000\u0000\u0000\u0000.\u0001\u0000\u0000\u0000\u00000"+ + "\u0001\u0000\u0000\u0000\u00002\u0001\u0000\u0000\u0000\u00004\u0001\u0000"+ + "\u0000\u0000\u00016\u0001\u0000\u0000\u0000\u00018\u0001\u0000\u0000\u0000"+ + "\u0001:\u0001\u0000\u0000\u0000\u0001<\u0001\u0000\u0000\u0000\u0001>"+ + "\u0001\u0000\u0000\u0000\u0002@\u0001\u0000\u0000\u0000\u0002V\u0001\u0000"+ + "\u0000\u0000\u0002X\u0001\u0000\u0000\u0000\u0002Z\u0001\u0000\u0000\u0000"+ + "\u0002\\\u0001\u0000\u0000\u0000\u0002^\u0001\u0000\u0000\u0000\u0002"+ + "`\u0001\u0000\u0000\u0000\u0002b\u0001\u0000\u0000\u0000\u0002d\u0001"+ + "\u0000\u0000\u0000\u0002f\u0001\u0000\u0000\u0000\u0002h\u0001\u0000\u0000"+ + "\u0000\u0002j\u0001\u0000\u0000\u0000\u0002l\u0001\u0000\u0000\u0000\u0002"+ + "n\u0001\u0000\u0000\u0000\u0002p\u0001\u0000\u0000\u0000\u0002r\u0001"+ + "\u0000\u0000\u0000\u0002t\u0001\u0000\u0000\u0000\u0002v\u0001\u0000\u0000"+ + "\u0000\u0002x\u0001\u0000\u0000\u0000\u0002z\u0001\u0000\u0000\u0000\u0002"+ + "|\u0001\u0000\u0000\u0000\u0002~\u0001\u0000\u0000\u0000\u0002\u0080\u0001"+ + "\u0000\u0000\u0000\u0002\u0082\u0001\u0000\u0000\u0000\u0002\u0084\u0001"+ + "\u0000\u0000\u0000\u0002\u0086\u0001\u0000\u0000\u0000\u0002\u0088\u0001"+ + "\u0000\u0000\u0000\u0002\u008a\u0001\u0000\u0000\u0000\u0002\u008c\u0001"+ + "\u0000\u0000\u0000\u0002\u008e\u0001\u0000\u0000\u0000\u0002\u0090\u0001"+ + "\u0000\u0000\u0000\u0002\u0092\u0001\u0000\u0000\u0000\u0002\u0094\u0001"+ + "\u0000\u0000\u0000\u0002\u0096\u0001\u0000\u0000\u0000\u0002\u0098\u0001"+ + "\u0000\u0000\u0000\u0002\u009a\u0001\u0000\u0000\u0000\u0002\u009c\u0001"+ + "\u0000\u0000\u0000\u0002\u009e\u0001\u0000\u0000\u0000\u0002\u00a0\u0001"+ + "\u0000\u0000\u0000\u0002\u00a2\u0001\u0000\u0000\u0000\u0002\u00a4\u0001"+ + "\u0000\u0000\u0000\u0002\u00a6\u0001\u0000\u0000\u0000\u0002\u00a8\u0001"+ + "\u0000\u0000\u0000\u0002\u00aa\u0001\u0000\u0000\u0000\u0003\u00ac\u0001"+ + "\u0000\u0000\u0000\u0003\u00ae\u0001\u0000\u0000\u0000\u0003\u00b0\u0001"+ + "\u0000\u0000\u0000\u0003\u00b2\u0001\u0000\u0000\u0000\u0003\u00b4\u0001"+ + "\u0000\u0000\u0000\u0003\u00b6\u0001\u0000\u0000\u0000\u0003\u00ba\u0001"+ + "\u0000\u0000\u0000\u0003\u00bc\u0001\u0000\u0000\u0000\u0003\u00be\u0001"+ + "\u0000\u0000\u0000\u0003\u00c0\u0001\u0000\u0000\u0000\u0003\u00c2\u0001"+ + "\u0000\u0000\u0000\u0004\u00c4\u0001\u0000\u0000\u0000\u0004\u00c6\u0001"+ + "\u0000\u0000\u0000\u0004\u00c8\u0001\u0000\u0000\u0000\u0004\u00cc\u0001"+ + "\u0000\u0000\u0000\u0004\u00ce\u0001\u0000\u0000\u0000\u0004\u00d0\u0001"+ + "\u0000\u0000\u0000\u0004\u00d2\u0001\u0000\u0000\u0000\u0004\u00d4\u0001"+ + "\u0000\u0000\u0000\u0005\u00d6\u0001\u0000\u0000\u0000\u0005\u00d8\u0001"+ + "\u0000\u0000\u0000\u0005\u00da\u0001\u0000\u0000\u0000\u0005\u00dc\u0001"+ + "\u0000\u0000\u0000\u0005\u00de\u0001\u0000\u0000\u0000\u0005\u00e0\u0001"+ + "\u0000\u0000\u0000\u0005\u00e2\u0001\u0000\u0000\u0000\u0005\u00e4\u0001"+ + "\u0000\u0000\u0000\u0005\u00e6\u0001\u0000\u0000\u0000\u0005\u00e8\u0001"+ + "\u0000\u0000\u0000\u0006\u00ea\u0001\u0000\u0000\u0000\u0006\u00ec\u0001"+ + "\u0000\u0000\u0000\u0006\u00ee\u0001\u0000\u0000\u0000\u0006\u00f0\u0001"+ + "\u0000\u0000\u0000\u0006\u00f2\u0001\u0000\u0000\u0000\u0006\u00f4\u0001"+ + "\u0000\u0000\u0000\u0006\u00f6\u0001\u0000\u0000\u0000\u0006\u00f8\u0001"+ + "\u0000\u0000\u0000\u0007\u00fa\u0001\u0000\u0000\u0000\u0007\u00fc\u0001"+ + "\u0000\u0000\u0000\u0007\u00fe\u0001\u0000\u0000\u0000\u0007\u0100\u0001"+ + "\u0000\u0000\u0000\u0007\u0102\u0001\u0000\u0000\u0000\u0007\u0104\u0001"+ + "\u0000\u0000\u0000\u0007\u0106\u0001\u0000\u0000\u0000\u0007\u0108\u0001"+ + "\u0000\u0000\u0000\u0007\u010a\u0001\u0000\u0000\u0000\u0007\u010c\u0001"+ + "\u0000\u0000\u0000\b\u010e\u0001\u0000\u0000\u0000\b\u0110\u0001\u0000"+ + "\u0000\u0000\b\u0112\u0001\u0000\u0000\u0000\b\u0114\u0001\u0000\u0000"+ + "\u0000\b\u0116\u0001\u0000\u0000\u0000\b\u0118\u0001\u0000\u0000\u0000"+ + "\b\u011a\u0001\u0000\u0000\u0000\t\u011c\u0001\u0000\u0000\u0000\t\u011e"+ + "\u0001\u0000\u0000\u0000\t\u0120\u0001\u0000\u0000\u0000\t\u0122\u0001"+ + "\u0000\u0000\u0000\t\u0124\u0001\u0000\u0000\u0000\t\u0126\u0001\u0000"+ + "\u0000\u0000\n\u0128\u0001\u0000\u0000\u0000\f\u0132\u0001\u0000\u0000"+ + "\u0000\u000e\u0139\u0001\u0000\u0000\u0000\u0010\u0142\u0001\u0000\u0000"+ + "\u0000\u0012\u0149\u0001\u0000\u0000\u0000\u0014\u0153\u0001\u0000\u0000"+ + "\u0000\u0016\u015a\u0001\u0000\u0000\u0000\u0018\u0161\u0001\u0000\u0000"+ + "\u0000\u001a\u016f\u0001\u0000\u0000\u0000\u001c\u0176\u0001\u0000\u0000"+ + "\u0000\u001e\u017e\u0001\u0000\u0000\u0000 \u018a\u0001\u0000\u0000\u0000"+ + "\"\u0194\u0001\u0000\u0000\u0000$\u019d\u0001\u0000\u0000\u0000&\u01a3"+ + "\u0001\u0000\u0000\u0000(\u01aa\u0001\u0000\u0000\u0000*\u01b1\u0001\u0000"+ + "\u0000\u0000,\u01b9\u0001\u0000\u0000\u0000.\u01c2\u0001\u0000\u0000\u0000"+ + "0\u01c8\u0001\u0000\u0000\u00002\u01d9\u0001\u0000\u0000\u00004\u01e9"+ + "\u0001\u0000\u0000\u00006\u01ef\u0001\u0000\u0000\u00008\u01f4\u0001\u0000"+ + "\u0000\u0000:\u01f9\u0001\u0000\u0000\u0000<\u01fd\u0001\u0000\u0000\u0000"+ + ">\u0201\u0001\u0000\u0000\u0000@\u0205\u0001\u0000\u0000\u0000B\u0209"+ + "\u0001\u0000\u0000\u0000D\u020b\u0001\u0000\u0000\u0000F\u020d\u0001\u0000"+ + "\u0000\u0000H\u0210\u0001\u0000\u0000\u0000J\u0212\u0001\u0000\u0000\u0000"+ + "L\u021b\u0001\u0000\u0000\u0000N\u021d\u0001\u0000\u0000\u0000P\u0222"+ + "\u0001\u0000\u0000\u0000R\u0224\u0001\u0000\u0000\u0000T\u0229\u0001\u0000"+ + "\u0000\u0000V\u0248\u0001\u0000\u0000\u0000X\u024b\u0001\u0000\u0000\u0000"+ + "Z\u0279\u0001\u0000\u0000\u0000\\\u027b\u0001\u0000\u0000\u0000^\u027e"+ + "\u0001\u0000\u0000\u0000`\u0282\u0001\u0000\u0000\u0000b\u0286\u0001\u0000"+ + "\u0000\u0000d\u0288\u0001\u0000\u0000\u0000f\u028a\u0001\u0000\u0000\u0000"+ + "h\u028f\u0001\u0000\u0000\u0000j\u0291\u0001\u0000\u0000\u0000l\u0297"+ + "\u0001\u0000\u0000\u0000n\u029d\u0001\u0000\u0000\u0000p\u02a2\u0001\u0000"+ + "\u0000\u0000r\u02a4\u0001\u0000\u0000\u0000t\u02a7\u0001\u0000\u0000\u0000"+ + "v\u02aa\u0001\u0000\u0000\u0000x\u02af\u0001\u0000\u0000\u0000z\u02b3"+ + "\u0001\u0000\u0000\u0000|\u02b8\u0001\u0000\u0000\u0000~\u02be\u0001\u0000"+ + "\u0000\u0000\u0080\u02c1\u0001\u0000\u0000\u0000\u0082\u02c3\u0001\u0000"+ + "\u0000\u0000\u0084\u02c9\u0001\u0000\u0000\u0000\u0086\u02cb\u0001\u0000"+ + "\u0000\u0000\u0088\u02d0\u0001\u0000\u0000\u0000\u008a\u02d3\u0001\u0000"+ + "\u0000\u0000\u008c\u02d6\u0001\u0000\u0000\u0000\u008e\u02d8\u0001\u0000"+ + "\u0000\u0000\u0090\u02db\u0001\u0000\u0000\u0000\u0092\u02dd\u0001\u0000"+ + "\u0000\u0000\u0094\u02e0\u0001\u0000\u0000\u0000\u0096\u02e2\u0001\u0000"+ + "\u0000\u0000\u0098\u02e4\u0001\u0000\u0000\u0000\u009a\u02e6\u0001\u0000"+ + "\u0000\u0000\u009c\u02e8\u0001\u0000\u0000\u0000\u009e\u02ea\u0001\u0000"+ + "\u0000\u0000\u00a0\u02ef\u0001\u0000\u0000\u0000\u00a2\u0304\u0001\u0000"+ + "\u0000\u0000\u00a4\u0306\u0001\u0000\u0000\u0000\u00a6\u030e\u0001\u0000"+ + "\u0000\u0000\u00a8\u0312\u0001\u0000\u0000\u0000\u00aa\u0316\u0001\u0000"+ + "\u0000\u0000\u00ac\u031a\u0001\u0000\u0000\u0000\u00ae\u031f\u0001\u0000"+ + "\u0000\u0000\u00b0\u0325\u0001\u0000\u0000\u0000\u00b2\u032b\u0001\u0000"+ + "\u0000\u0000\u00b4\u032f\u0001\u0000\u0000\u0000\u00b6\u0333\u0001\u0000"+ + "\u0000\u0000\u00b8\u033f\u0001\u0000\u0000\u0000\u00ba\u0342\u0001\u0000"+ + "\u0000\u0000\u00bc\u0346\u0001\u0000\u0000\u0000\u00be\u034a\u0001\u0000"+ + "\u0000\u0000\u00c0\u034e\u0001\u0000\u0000\u0000\u00c2\u0352\u0001\u0000"+ + "\u0000\u0000\u00c4\u0356\u0001\u0000\u0000\u0000\u00c6\u035b\u0001\u0000"+ + "\u0000\u0000\u00c8\u035f\u0001\u0000\u0000\u0000\u00ca\u0367\u0001\u0000"+ + "\u0000\u0000\u00cc\u037c\u0001\u0000\u0000\u0000\u00ce\u037e\u0001\u0000"+ + "\u0000\u0000\u00d0\u0382\u0001\u0000\u0000\u0000\u00d2\u0386\u0001\u0000"+ + "\u0000\u0000\u00d4\u038a\u0001\u0000\u0000\u0000\u00d6\u038e\u0001\u0000"+ + "\u0000\u0000\u00d8\u0393\u0001\u0000\u0000\u0000\u00da\u0397\u0001\u0000"+ + "\u0000\u0000\u00dc\u039b\u0001\u0000\u0000\u0000\u00de\u039f\u0001\u0000"+ + "\u0000\u0000\u00e0\u03a2\u0001\u0000\u0000\u0000\u00e2\u03a6\u0001\u0000"+ + "\u0000\u0000\u00e4\u03aa\u0001\u0000\u0000\u0000\u00e6\u03ae\u0001\u0000"+ + "\u0000\u0000\u00e8\u03b2\u0001\u0000\u0000\u0000\u00ea\u03b6\u0001\u0000"+ + "\u0000\u0000\u00ec\u03bb\u0001\u0000\u0000\u0000\u00ee\u03c0\u0001\u0000"+ + "\u0000\u0000\u00f0\u03c7\u0001\u0000\u0000\u0000\u00f2\u03cb\u0001\u0000"+ + "\u0000\u0000\u00f4\u03cf\u0001\u0000\u0000\u0000\u00f6\u03d3\u0001\u0000"+ + "\u0000\u0000\u00f8\u03d7\u0001\u0000\u0000\u0000\u00fa\u03db\u0001\u0000"+ + "\u0000\u0000\u00fc\u03e1\u0001\u0000\u0000\u0000\u00fe\u03e5\u0001\u0000"+ + "\u0000\u0000\u0100\u03e9\u0001\u0000\u0000\u0000\u0102\u03ed\u0001\u0000"+ + "\u0000\u0000\u0104\u03f1\u0001\u0000\u0000\u0000\u0106\u03f5\u0001\u0000"+ + "\u0000\u0000\u0108\u03f9\u0001\u0000\u0000\u0000\u010a\u03fd\u0001\u0000"+ + "\u0000\u0000\u010c\u0401\u0001\u0000\u0000\u0000\u010e\u0405\u0001\u0000"+ + "\u0000\u0000\u0110\u040a\u0001\u0000\u0000\u0000\u0112\u040e\u0001\u0000"+ + "\u0000\u0000\u0114\u0412\u0001\u0000\u0000\u0000\u0116\u0416\u0001\u0000"+ + "\u0000\u0000\u0118\u041a\u0001\u0000\u0000\u0000\u011a\u041e\u0001\u0000"+ + "\u0000\u0000\u011c\u0422\u0001\u0000\u0000\u0000\u011e\u0427\u0001\u0000"+ + "\u0000\u0000\u0120\u042c\u0001\u0000\u0000\u0000\u0122\u0436\u0001\u0000"+ + "\u0000\u0000\u0124\u043a\u0001\u0000\u0000\u0000\u0126\u043e\u0001\u0000"+ + "\u0000\u0000\u0128\u0129\u0005d\u0000\u0000\u0129\u012a\u0005i\u0000\u0000"+ + "\u012a\u012b\u0005s\u0000\u0000\u012b\u012c\u0005s\u0000\u0000\u012c\u012d"+ + "\u0005e\u0000\u0000\u012d\u012e\u0005c\u0000\u0000\u012e\u012f\u0005t"+ + "\u0000\u0000\u012f\u0130\u0001\u0000\u0000\u0000\u0130\u0131\u0006\u0000"+ + "\u0000\u0000\u0131\u000b\u0001\u0000\u0000\u0000\u0132\u0133\u0005d\u0000"+ + "\u0000\u0133\u0134\u0005r\u0000\u0000\u0134\u0135\u0005o\u0000\u0000\u0135"+ + "\u0136\u0005p\u0000\u0000\u0136\u0137\u0001\u0000\u0000\u0000\u0137\u0138"+ + "\u0006\u0001\u0001\u0000\u0138\r\u0001\u0000\u0000\u0000\u0139\u013a\u0005"+ + "e\u0000\u0000\u013a\u013b\u0005n\u0000\u0000\u013b\u013c\u0005r\u0000"+ + "\u0000\u013c\u013d\u0005i\u0000\u0000\u013d\u013e\u0005c\u0000\u0000\u013e"+ + "\u013f\u0005h\u0000\u0000\u013f\u0140\u0001\u0000\u0000\u0000\u0140\u0141"+ + "\u0006\u0002\u0002\u0000\u0141\u000f\u0001\u0000\u0000\u0000\u0142\u0143"+ + "\u0005e\u0000\u0000\u0143\u0144\u0005v\u0000\u0000\u0144\u0145\u0005a"+ + "\u0000\u0000\u0145\u0146\u0005l\u0000\u0000\u0146\u0147\u0001\u0000\u0000"+ + "\u0000\u0147\u0148\u0006\u0003\u0000\u0000\u0148\u0011\u0001\u0000\u0000"+ + "\u0000\u0149\u014a\u0005e\u0000\u0000\u014a\u014b\u0005x\u0000\u0000\u014b"+ + "\u014c\u0005p\u0000\u0000\u014c\u014d\u0005l\u0000\u0000\u014d\u014e\u0005"+ + "a\u0000\u0000\u014e\u014f\u0005i\u0000\u0000\u014f\u0150\u0005n\u0000"+ + "\u0000\u0150\u0151\u0001\u0000\u0000\u0000\u0151\u0152\u0006\u0004\u0003"+ + "\u0000\u0152\u0013\u0001\u0000\u0000\u0000\u0153\u0154\u0005f\u0000\u0000"+ + "\u0154\u0155\u0005r\u0000\u0000\u0155\u0156\u0005o\u0000\u0000\u0156\u0157"+ + "\u0005m\u0000\u0000\u0157\u0158\u0001\u0000\u0000\u0000\u0158\u0159\u0006"+ + "\u0005\u0004\u0000\u0159\u0015\u0001\u0000\u0000\u0000\u015a\u015b\u0005"+ + "g\u0000\u0000\u015b\u015c\u0005r\u0000\u0000\u015c\u015d\u0005o\u0000"+ + "\u0000\u015d\u015e\u0005k\u0000\u0000\u015e\u015f\u0001\u0000\u0000\u0000"+ + "\u015f\u0160\u0006\u0006\u0000\u0000\u0160\u0017\u0001\u0000\u0000\u0000"+ + "\u0161\u0162\u0005i\u0000\u0000\u0162\u0163\u0005n\u0000\u0000\u0163\u0164"+ + "\u0005l\u0000\u0000\u0164\u0165\u0005i\u0000\u0000\u0165\u0166\u0005n"+ + "\u0000\u0000\u0166\u0167\u0005e\u0000\u0000\u0167\u0168\u0005s\u0000\u0000"+ + "\u0168\u0169\u0005t\u0000\u0000\u0169\u016a\u0005a\u0000\u0000\u016a\u016b"+ + "\u0005t\u0000\u0000\u016b\u016c\u0005s\u0000\u0000\u016c\u016d\u0001\u0000"+ + "\u0000\u0000\u016d\u016e\u0006\u0007\u0000\u0000\u016e\u0019\u0001\u0000"+ + "\u0000\u0000\u016f\u0170\u0005k\u0000\u0000\u0170\u0171\u0005e\u0000\u0000"+ + "\u0171\u0172\u0005e\u0000\u0000\u0172\u0173\u0005p\u0000\u0000\u0173\u0174"+ + "\u0001\u0000\u0000\u0000\u0174\u0175\u0006\b\u0001\u0000\u0175\u001b\u0001"+ + "\u0000\u0000\u0000\u0176\u0177\u0005l\u0000\u0000\u0177\u0178\u0005i\u0000"+ + "\u0000\u0178\u0179\u0005m\u0000\u0000\u0179\u017a\u0005i\u0000\u0000\u017a"+ + "\u017b\u0005t\u0000\u0000\u017b\u017c\u0001\u0000\u0000\u0000\u017c\u017d"+ + "\u0006\t\u0000\u0000\u017d\u001d\u0001\u0000\u0000\u0000\u017e\u017f\u0005"+ + "m\u0000\u0000\u017f\u0180\u0005v\u0000\u0000\u0180\u0181\u0005_\u0000"+ + "\u0000\u0181\u0182\u0005e\u0000\u0000\u0182\u0183\u0005x\u0000\u0000\u0183"+ + "\u0184\u0005p\u0000\u0000\u0184\u0185\u0005a\u0000\u0000\u0185\u0186\u0005"+ + "n\u0000\u0000\u0186\u0187\u0005d\u0000\u0000\u0187\u0188\u0001\u0000\u0000"+ + "\u0000\u0188\u0189\u0006\n\u0005\u0000\u0189\u001f\u0001\u0000\u0000\u0000"+ + "\u018a\u018b\u0005p\u0000\u0000\u018b\u018c\u0005r\u0000\u0000\u018c\u018d"+ + "\u0005o\u0000\u0000\u018d\u018e\u0005j\u0000\u0000\u018e\u018f\u0005e"+ + "\u0000\u0000\u018f\u0190\u0005c\u0000\u0000\u0190\u0191\u0005t\u0000\u0000"+ + "\u0191\u0192\u0001\u0000\u0000\u0000\u0192\u0193\u0006\u000b\u0001\u0000"+ + "\u0193!\u0001\u0000\u0000\u0000\u0194\u0195\u0005r\u0000\u0000\u0195\u0196"+ + "\u0005e\u0000\u0000\u0196\u0197\u0005n\u0000\u0000\u0197\u0198\u0005a"+ + "\u0000\u0000\u0198\u0199\u0005m\u0000\u0000\u0199\u019a\u0005e\u0000\u0000"+ + "\u019a\u019b\u0001\u0000\u0000\u0000\u019b\u019c\u0006\f\u0006\u0000\u019c"+ + "#\u0001\u0000\u0000\u0000\u019d\u019e\u0005r\u0000\u0000\u019e\u019f\u0005"+ + "o\u0000\u0000\u019f\u01a0\u0005w\u0000\u0000\u01a0\u01a1\u0001\u0000\u0000"+ + "\u0000\u01a1\u01a2\u0006\r\u0000\u0000\u01a2%\u0001\u0000\u0000\u0000"+ + "\u01a3\u01a4\u0005s\u0000\u0000\u01a4\u01a5\u0005h\u0000\u0000\u01a5\u01a6"+ + "\u0005o\u0000\u0000\u01a6\u01a7\u0005w\u0000\u0000\u01a7\u01a8\u0001\u0000"+ + "\u0000\u0000\u01a8\u01a9\u0006\u000e\u0007\u0000\u01a9\'\u0001\u0000\u0000"+ + "\u0000\u01aa\u01ab\u0005s\u0000\u0000\u01ab\u01ac\u0005o\u0000\u0000\u01ac"+ + "\u01ad\u0005r\u0000\u0000\u01ad\u01ae\u0005t\u0000\u0000\u01ae\u01af\u0001"+ + "\u0000\u0000\u0000\u01af\u01b0\u0006\u000f\u0000\u0000\u01b0)\u0001\u0000"+ + "\u0000\u0000\u01b1\u01b2\u0005s\u0000\u0000\u01b2\u01b3\u0005t\u0000\u0000"+ + "\u01b3\u01b4\u0005a\u0000\u0000\u01b4\u01b5\u0005t\u0000\u0000\u01b5\u01b6"+ + "\u0005s\u0000\u0000\u01b6\u01b7\u0001\u0000\u0000\u0000\u01b7\u01b8\u0006"+ + "\u0010\u0000\u0000\u01b8+\u0001\u0000\u0000\u0000\u01b9\u01ba\u0005w\u0000"+ + "\u0000\u01ba\u01bb\u0005h\u0000\u0000\u01bb\u01bc\u0005e\u0000\u0000\u01bc"+ + "\u01bd\u0005r\u0000\u0000\u01bd\u01be\u0005e\u0000\u0000\u01be\u01bf\u0001"+ + "\u0000\u0000\u0000\u01bf\u01c0\u0006\u0011\u0000\u0000\u01c0-\u0001\u0000"+ + "\u0000\u0000\u01c1\u01c3\b\u0000\u0000\u0000\u01c2\u01c1\u0001\u0000\u0000"+ + "\u0000\u01c3\u01c4\u0001\u0000\u0000\u0000\u01c4\u01c2\u0001\u0000\u0000"+ + "\u0000\u01c4\u01c5\u0001\u0000\u0000\u0000\u01c5\u01c6\u0001\u0000\u0000"+ + "\u0000\u01c6\u01c7\u0006\u0012\u0000\u0000\u01c7/\u0001\u0000\u0000\u0000"+ + "\u01c8\u01c9\u0005/\u0000\u0000\u01c9\u01ca\u0005/\u0000\u0000\u01ca\u01ce"+ + "\u0001\u0000\u0000\u0000\u01cb\u01cd\b\u0001\u0000\u0000\u01cc\u01cb\u0001"+ + "\u0000\u0000\u0000\u01cd\u01d0\u0001\u0000\u0000\u0000\u01ce\u01cc\u0001"+ + "\u0000\u0000\u0000\u01ce\u01cf\u0001\u0000\u0000\u0000\u01cf\u01d2\u0001"+ + "\u0000\u0000\u0000\u01d0\u01ce\u0001\u0000\u0000\u0000\u01d1\u01d3\u0005"+ + "\r\u0000\u0000\u01d2\u01d1\u0001\u0000\u0000\u0000\u01d2\u01d3\u0001\u0000"+ + "\u0000\u0000\u01d3\u01d5\u0001\u0000\u0000\u0000\u01d4\u01d6\u0005\n\u0000"+ + "\u0000\u01d5\u01d4\u0001\u0000\u0000\u0000\u01d5\u01d6\u0001\u0000\u0000"+ + "\u0000\u01d6\u01d7\u0001\u0000\u0000\u0000\u01d7\u01d8\u0006\u0013\b\u0000"+ + "\u01d81\u0001\u0000\u0000\u0000\u01d9\u01da\u0005/\u0000\u0000\u01da\u01db"+ + "\u0005*\u0000\u0000\u01db\u01e0\u0001\u0000\u0000\u0000\u01dc\u01df\u0003"+ + "2\u0014\u0000\u01dd\u01df\t\u0000\u0000\u0000\u01de\u01dc\u0001\u0000"+ + "\u0000\u0000\u01de\u01dd\u0001\u0000\u0000\u0000\u01df\u01e2\u0001\u0000"+ + "\u0000\u0000\u01e0\u01e1\u0001\u0000\u0000\u0000\u01e0\u01de\u0001\u0000"+ + "\u0000\u0000\u01e1\u01e3\u0001\u0000\u0000\u0000\u01e2\u01e0\u0001\u0000"+ + "\u0000\u0000\u01e3\u01e4\u0005*\u0000\u0000\u01e4\u01e5\u0005/\u0000\u0000"+ + "\u01e5\u01e6\u0001\u0000\u0000\u0000\u01e6\u01e7\u0006\u0014\b\u0000\u01e7"+ + "3\u0001\u0000\u0000\u0000\u01e8\u01ea\u0007\u0002\u0000\u0000\u01e9\u01e8"+ + "\u0001\u0000\u0000\u0000\u01ea\u01eb\u0001\u0000\u0000\u0000\u01eb\u01e9"+ + "\u0001\u0000\u0000\u0000\u01eb\u01ec\u0001\u0000\u0000\u0000\u01ec\u01ed"+ + "\u0001\u0000\u0000\u0000\u01ed\u01ee\u0006\u0015\b\u0000\u01ee5\u0001"+ + "\u0000\u0000\u0000\u01ef\u01f0\u0003\u009eJ\u0000\u01f0\u01f1\u0001\u0000"+ + "\u0000\u0000\u01f1\u01f2\u0006\u0016\t\u0000\u01f2\u01f3\u0006\u0016\n"+ + "\u0000\u01f37\u0001\u0000\u0000\u0000\u01f4\u01f5\u0003@\u001b\u0000\u01f5"+ + "\u01f6\u0001\u0000\u0000\u0000\u01f6\u01f7\u0006\u0017\u000b\u0000\u01f7"+ + "\u01f8\u0006\u0017\f\u0000\u01f89\u0001\u0000\u0000\u0000\u01f9\u01fa"+ + "\u00034\u0015\u0000\u01fa\u01fb\u0001\u0000\u0000\u0000\u01fb\u01fc\u0006"+ + "\u0018\b\u0000\u01fc;\u0001\u0000\u0000\u0000\u01fd\u01fe\u00030\u0013"+ + "\u0000\u01fe\u01ff\u0001\u0000\u0000\u0000\u01ff\u0200\u0006\u0019\b\u0000"+ + "\u0200=\u0001\u0000\u0000\u0000\u0201\u0202\u00032\u0014\u0000\u0202\u0203"+ + "\u0001\u0000\u0000\u0000\u0203\u0204\u0006\u001a\b\u0000\u0204?\u0001"+ + "\u0000\u0000\u0000\u0205\u0206\u0005|\u0000\u0000\u0206\u0207\u0001\u0000"+ + "\u0000\u0000\u0207\u0208\u0006\u001b\f\u0000\u0208A\u0001\u0000\u0000"+ + "\u0000\u0209\u020a\u0007\u0003\u0000\u0000\u020aC\u0001\u0000\u0000\u0000"+ + "\u020b\u020c\u0007\u0004\u0000\u0000\u020cE\u0001\u0000\u0000\u0000\u020d"+ + "\u020e\u0005\\\u0000\u0000\u020e\u020f\u0007\u0005\u0000\u0000\u020fG"+ + "\u0001\u0000\u0000\u0000\u0210\u0211\b\u0006\u0000\u0000\u0211I\u0001"+ + "\u0000\u0000\u0000\u0212\u0214\u0007\u0007\u0000\u0000\u0213\u0215\u0007"+ + "\b\u0000\u0000\u0214\u0213\u0001\u0000\u0000\u0000\u0214\u0215\u0001\u0000"+ + "\u0000\u0000\u0215\u0217\u0001\u0000\u0000\u0000\u0216\u0218\u0003B\u001c"+ + "\u0000\u0217\u0216\u0001\u0000\u0000\u0000\u0218\u0219\u0001\u0000\u0000"+ + "\u0000\u0219\u0217\u0001\u0000\u0000\u0000\u0219\u021a\u0001\u0000\u0000"+ + "\u0000\u021aK\u0001\u0000\u0000\u0000\u021b\u021c\u0005@\u0000\u0000\u021c"+ + "M\u0001\u0000\u0000\u0000\u021d\u021e\u0005`\u0000\u0000\u021eO\u0001"+ + "\u0000\u0000\u0000\u021f\u0223\b\t\u0000\u0000\u0220\u0221\u0005`\u0000"+ + "\u0000\u0221\u0223\u0005`\u0000\u0000\u0222\u021f\u0001\u0000\u0000\u0000"+ + "\u0222\u0220\u0001\u0000\u0000\u0000\u0223Q\u0001\u0000\u0000\u0000\u0224"+ + "\u0225\u0005_\u0000\u0000\u0225S\u0001\u0000\u0000\u0000\u0226\u022a\u0003"+ + "D\u001d\u0000\u0227\u022a\u0003B\u001c\u0000\u0228\u022a\u0003R$\u0000"+ + "\u0229\u0226\u0001\u0000\u0000\u0000\u0229\u0227\u0001\u0000\u0000\u0000"+ + "\u0229\u0228\u0001\u0000\u0000\u0000\u022aU\u0001\u0000\u0000\u0000\u022b"+ + "\u0230\u0005\"\u0000\u0000\u022c\u022f\u0003F\u001e\u0000\u022d\u022f"+ + "\u0003H\u001f\u0000\u022e\u022c\u0001\u0000\u0000\u0000\u022e\u022d\u0001"+ + "\u0000\u0000\u0000\u022f\u0232\u0001\u0000\u0000\u0000\u0230\u022e\u0001"+ + "\u0000\u0000\u0000\u0230\u0231\u0001\u0000\u0000\u0000\u0231\u0233\u0001"+ + "\u0000\u0000\u0000\u0232\u0230\u0001\u0000\u0000\u0000\u0233\u0249\u0005"+ + "\"\u0000\u0000\u0234\u0235\u0005\"\u0000\u0000\u0235\u0236\u0005\"\u0000"+ + "\u0000\u0236\u0237\u0005\"\u0000\u0000\u0237\u023b\u0001\u0000\u0000\u0000"+ + "\u0238\u023a\b\u0001\u0000\u0000\u0239\u0238\u0001\u0000\u0000\u0000\u023a"+ + "\u023d\u0001\u0000\u0000\u0000\u023b\u023c\u0001\u0000\u0000\u0000\u023b"+ + "\u0239\u0001\u0000\u0000\u0000\u023c\u023e\u0001\u0000\u0000\u0000\u023d"+ + "\u023b\u0001\u0000\u0000\u0000\u023e\u023f\u0005\"\u0000\u0000\u023f\u0240"+ + "\u0005\"\u0000\u0000\u0240\u0241\u0005\"\u0000\u0000\u0241\u0243\u0001"+ + "\u0000\u0000\u0000\u0242\u0244\u0005\"\u0000\u0000\u0243\u0242\u0001\u0000"+ + "\u0000\u0000\u0243\u0244\u0001\u0000\u0000\u0000\u0244\u0246\u0001\u0000"+ + "\u0000\u0000\u0245\u0247\u0005\"\u0000\u0000\u0246\u0245\u0001\u0000\u0000"+ + "\u0000\u0246\u0247\u0001\u0000\u0000\u0000\u0247\u0249\u0001\u0000\u0000"+ + "\u0000\u0248\u022b\u0001\u0000\u0000\u0000\u0248\u0234\u0001\u0000\u0000"+ + "\u0000\u0249W\u0001\u0000\u0000\u0000\u024a\u024c\u0003B\u001c\u0000\u024b"+ + "\u024a\u0001\u0000\u0000\u0000\u024c\u024d\u0001\u0000\u0000\u0000\u024d"+ + "\u024b\u0001\u0000\u0000\u0000\u024d\u024e\u0001\u0000\u0000\u0000\u024e"+ + "Y\u0001\u0000\u0000\u0000\u024f\u0251\u0003B\u001c\u0000\u0250\u024f\u0001"+ + "\u0000\u0000\u0000\u0251\u0252\u0001\u0000\u0000\u0000\u0252\u0250\u0001"+ + "\u0000\u0000\u0000\u0252\u0253\u0001\u0000\u0000\u0000\u0253\u0254\u0001"+ + "\u0000\u0000\u0000\u0254\u0258\u0003h/\u0000\u0255\u0257\u0003B\u001c"+ + "\u0000\u0256\u0255\u0001\u0000\u0000\u0000\u0257\u025a\u0001\u0000\u0000"+ + "\u0000\u0258\u0256\u0001\u0000\u0000\u0000\u0258\u0259\u0001\u0000\u0000"+ + "\u0000\u0259\u027a\u0001\u0000\u0000\u0000\u025a\u0258\u0001\u0000\u0000"+ + "\u0000\u025b\u025d\u0003h/\u0000\u025c\u025e\u0003B\u001c\u0000\u025d"+ + "\u025c\u0001\u0000\u0000\u0000\u025e\u025f\u0001\u0000\u0000\u0000\u025f"+ + "\u025d\u0001\u0000\u0000\u0000\u025f\u0260\u0001\u0000\u0000\u0000\u0260"+ + "\u027a\u0001\u0000\u0000\u0000\u0261\u0263\u0003B\u001c\u0000\u0262\u0261"+ + "\u0001\u0000\u0000\u0000\u0263\u0264\u0001\u0000\u0000\u0000\u0264\u0262"+ + "\u0001\u0000\u0000\u0000\u0264\u0265\u0001\u0000\u0000\u0000\u0265\u026d"+ + "\u0001\u0000\u0000\u0000\u0266\u026a\u0003h/\u0000\u0267\u0269\u0003B"+ + "\u001c\u0000\u0268\u0267\u0001\u0000\u0000\u0000\u0269\u026c\u0001\u0000"+ + "\u0000\u0000\u026a\u0268\u0001\u0000\u0000\u0000\u026a\u026b\u0001\u0000"+ + "\u0000\u0000\u026b\u026e\u0001\u0000\u0000\u0000\u026c\u026a\u0001\u0000"+ + "\u0000\u0000\u026d\u0266\u0001\u0000\u0000\u0000\u026d\u026e\u0001\u0000"+ + "\u0000\u0000\u026e\u026f\u0001\u0000\u0000\u0000\u026f\u0270\u0003J \u0000"+ + "\u0270\u027a\u0001\u0000\u0000\u0000\u0271\u0273\u0003h/\u0000\u0272\u0274"+ + "\u0003B\u001c\u0000\u0273\u0272\u0001\u0000\u0000\u0000\u0274\u0275\u0001"+ + "\u0000\u0000\u0000\u0275\u0273\u0001\u0000\u0000\u0000\u0275\u0276\u0001"+ + "\u0000\u0000\u0000\u0276\u0277\u0001\u0000\u0000\u0000\u0277\u0278\u0003"+ + "J \u0000\u0278\u027a\u0001\u0000\u0000\u0000\u0279\u0250\u0001\u0000\u0000"+ + "\u0000\u0279\u025b\u0001\u0000\u0000\u0000\u0279\u0262\u0001\u0000\u0000"+ + "\u0000\u0279\u0271\u0001\u0000\u0000\u0000\u027a[\u0001\u0000\u0000\u0000"+ + "\u027b\u027c\u0005b\u0000\u0000\u027c\u027d\u0005y\u0000\u0000\u027d]"+ + "\u0001\u0000\u0000\u0000\u027e\u027f\u0005a\u0000\u0000\u027f\u0280\u0005"+ + "n\u0000\u0000\u0280\u0281\u0005d\u0000\u0000\u0281_\u0001\u0000\u0000"+ + "\u0000\u0282\u0283\u0005a\u0000\u0000\u0283\u0284\u0005s\u0000\u0000\u0284"+ + "\u0285\u0005c\u0000\u0000\u0285a\u0001\u0000\u0000\u0000\u0286\u0287\u0005"+ + "=\u0000\u0000\u0287c\u0001\u0000\u0000\u0000\u0288\u0289\u0005,\u0000"+ + "\u0000\u0289e\u0001\u0000\u0000\u0000\u028a\u028b\u0005d\u0000\u0000\u028b"+ + "\u028c\u0005e\u0000\u0000\u028c\u028d\u0005s\u0000\u0000\u028d\u028e\u0005"+ + "c\u0000\u0000\u028eg\u0001\u0000\u0000\u0000\u028f\u0290\u0005.\u0000"+ + "\u0000\u0290i\u0001\u0000\u0000\u0000\u0291\u0292\u0005f\u0000\u0000\u0292"+ + "\u0293\u0005a\u0000\u0000\u0293\u0294\u0005l\u0000\u0000\u0294\u0295\u0005"+ + "s\u0000\u0000\u0295\u0296\u0005e\u0000\u0000\u0296k\u0001\u0000\u0000"+ + "\u0000\u0297\u0298\u0005f\u0000\u0000\u0298\u0299\u0005i\u0000\u0000\u0299"+ + "\u029a\u0005r\u0000\u0000\u029a\u029b\u0005s\u0000\u0000\u029b\u029c\u0005"+ + "t\u0000\u0000\u029cm\u0001\u0000\u0000\u0000\u029d\u029e\u0005l\u0000"+ + "\u0000\u029e\u029f\u0005a\u0000\u0000\u029f\u02a0\u0005s\u0000\u0000\u02a0"+ + "\u02a1\u0005t\u0000\u0000\u02a1o\u0001\u0000\u0000\u0000\u02a2\u02a3\u0005"+ + "(\u0000\u0000\u02a3q\u0001\u0000\u0000\u0000\u02a4\u02a5\u0005i\u0000"+ + "\u0000\u02a5\u02a6\u0005n\u0000\u0000\u02a6s\u0001\u0000\u0000\u0000\u02a7"+ + "\u02a8\u0005i\u0000\u0000\u02a8\u02a9\u0005s\u0000\u0000\u02a9u\u0001"+ + "\u0000\u0000\u0000\u02aa\u02ab\u0005l\u0000\u0000\u02ab\u02ac\u0005i\u0000"+ + "\u0000\u02ac\u02ad\u0005k\u0000\u0000\u02ad\u02ae\u0005e\u0000\u0000\u02ae"+ + "w\u0001\u0000\u0000\u0000\u02af\u02b0\u0005n\u0000\u0000\u02b0\u02b1\u0005"+ + "o\u0000\u0000\u02b1\u02b2\u0005t\u0000\u0000\u02b2y\u0001\u0000\u0000"+ + "\u0000\u02b3\u02b4\u0005n\u0000\u0000\u02b4\u02b5\u0005u\u0000\u0000\u02b5"+ + "\u02b6\u0005l\u0000\u0000\u02b6\u02b7\u0005l\u0000\u0000\u02b7{\u0001"+ + "\u0000\u0000\u0000\u02b8\u02b9\u0005n\u0000\u0000\u02b9\u02ba\u0005u\u0000"+ + "\u0000\u02ba\u02bb\u0005l\u0000\u0000\u02bb\u02bc\u0005l\u0000\u0000\u02bc"+ + "\u02bd\u0005s\u0000\u0000\u02bd}\u0001\u0000\u0000\u0000\u02be\u02bf\u0005"+ + "o\u0000\u0000\u02bf\u02c0\u0005r\u0000\u0000\u02c0\u007f\u0001\u0000\u0000"+ + "\u0000\u02c1\u02c2\u0005?\u0000\u0000\u02c2\u0081\u0001\u0000\u0000\u0000"+ + "\u02c3\u02c4\u0005r\u0000\u0000\u02c4\u02c5\u0005l\u0000\u0000\u02c5\u02c6"+ + "\u0005i\u0000\u0000\u02c6\u02c7\u0005k\u0000\u0000\u02c7\u02c8\u0005e"+ + "\u0000\u0000\u02c8\u0083\u0001\u0000\u0000\u0000\u02c9\u02ca\u0005)\u0000"+ + "\u0000\u02ca\u0085\u0001\u0000\u0000\u0000\u02cb\u02cc\u0005t\u0000\u0000"+ + "\u02cc\u02cd\u0005r\u0000\u0000\u02cd\u02ce\u0005u\u0000\u0000\u02ce\u02cf"+ + "\u0005e\u0000\u0000\u02cf\u0087\u0001\u0000\u0000\u0000\u02d0\u02d1\u0005"+ + "=\u0000\u0000\u02d1\u02d2\u0005=\u0000\u0000\u02d2\u0089\u0001\u0000\u0000"+ + "\u0000\u02d3\u02d4\u0005!\u0000\u0000\u02d4\u02d5\u0005=\u0000\u0000\u02d5"+ + "\u008b\u0001\u0000\u0000\u0000\u02d6\u02d7\u0005<\u0000\u0000\u02d7\u008d"+ + "\u0001\u0000\u0000\u0000\u02d8\u02d9\u0005<\u0000\u0000\u02d9\u02da\u0005"+ + "=\u0000\u0000\u02da\u008f\u0001\u0000\u0000\u0000\u02db\u02dc\u0005>\u0000"+ + "\u0000\u02dc\u0091\u0001\u0000\u0000\u0000\u02dd\u02de\u0005>\u0000\u0000"+ + "\u02de\u02df\u0005=\u0000\u0000\u02df\u0093\u0001\u0000\u0000\u0000\u02e0"+ + "\u02e1\u0005+\u0000\u0000\u02e1\u0095\u0001\u0000\u0000\u0000\u02e2\u02e3"+ + "\u0005-\u0000\u0000\u02e3\u0097\u0001\u0000\u0000\u0000\u02e4\u02e5\u0005"+ + "*\u0000\u0000\u02e5\u0099\u0001\u0000\u0000\u0000\u02e6\u02e7\u0005/\u0000"+ + "\u0000\u02e7\u009b\u0001\u0000\u0000\u0000\u02e8\u02e9\u0005%\u0000\u0000"+ + "\u02e9\u009d\u0001\u0000\u0000\u0000\u02ea\u02eb\u0005[\u0000\u0000\u02eb"+ + "\u02ec\u0001\u0000\u0000\u0000\u02ec\u02ed\u0006J\u0000\u0000\u02ed\u02ee"+ + "\u0006J\u0000\u0000\u02ee\u009f\u0001\u0000\u0000\u0000\u02ef\u02f0\u0005"+ + "]\u0000\u0000\u02f0\u02f1\u0001\u0000\u0000\u0000\u02f1\u02f2\u0006K\f"+ + "\u0000\u02f2\u02f3\u0006K\f\u0000\u02f3\u00a1\u0001\u0000\u0000\u0000"+ + "\u02f4\u02f8\u0003D\u001d\u0000\u02f5\u02f7\u0003T%\u0000\u02f6\u02f5"+ + "\u0001\u0000\u0000\u0000\u02f7\u02fa\u0001\u0000\u0000\u0000\u02f8\u02f6"+ + "\u0001\u0000\u0000\u0000\u02f8\u02f9\u0001\u0000\u0000\u0000\u02f9\u0305"+ + "\u0001\u0000\u0000\u0000\u02fa\u02f8\u0001\u0000\u0000\u0000\u02fb\u02fe"+ + "\u0003R$\u0000\u02fc\u02fe\u0003L!\u0000\u02fd\u02fb\u0001\u0000\u0000"+ + "\u0000\u02fd\u02fc\u0001\u0000\u0000\u0000\u02fe\u0300\u0001\u0000\u0000"+ + "\u0000\u02ff\u0301\u0003T%\u0000\u0300\u02ff\u0001\u0000\u0000\u0000\u0301"+ + "\u0302\u0001\u0000\u0000\u0000\u0302\u0300\u0001\u0000\u0000\u0000\u0302"+ + "\u0303\u0001\u0000\u0000\u0000\u0303\u0305\u0001\u0000\u0000\u0000\u0304"+ + "\u02f4\u0001\u0000\u0000\u0000\u0304\u02fd\u0001\u0000\u0000\u0000\u0305"+ + "\u00a3\u0001\u0000\u0000\u0000\u0306\u0308\u0003N\"\u0000\u0307\u0309"+ + "\u0003P#\u0000\u0308\u0307\u0001\u0000\u0000\u0000\u0309\u030a\u0001\u0000"+ + "\u0000\u0000\u030a\u0308\u0001\u0000\u0000\u0000\u030a\u030b\u0001\u0000"+ + "\u0000\u0000\u030b\u030c\u0001\u0000\u0000\u0000\u030c\u030d\u0003N\""+ + "\u0000\u030d\u00a5\u0001\u0000\u0000\u0000\u030e\u030f\u00030\u0013\u0000"+ + "\u030f\u0310\u0001\u0000\u0000\u0000\u0310\u0311\u0006N\b\u0000\u0311"+ + "\u00a7\u0001\u0000\u0000\u0000\u0312\u0313\u00032\u0014\u0000\u0313\u0314"+ + "\u0001\u0000\u0000\u0000\u0314\u0315\u0006O\b\u0000\u0315\u00a9\u0001"+ + "\u0000\u0000\u0000\u0316\u0317\u00034\u0015\u0000\u0317\u0318\u0001\u0000"+ + "\u0000\u0000\u0318\u0319\u0006P\b\u0000\u0319\u00ab\u0001\u0000\u0000"+ + "\u0000\u031a\u031b\u0003@\u001b\u0000\u031b\u031c\u0001\u0000\u0000\u0000"+ + "\u031c\u031d\u0006Q\u000b\u0000\u031d\u031e\u0006Q\f\u0000\u031e\u00ad"+ + "\u0001\u0000\u0000\u0000\u031f\u0320\u0003\u009eJ\u0000\u0320\u0321\u0001"+ + "\u0000\u0000\u0000\u0321\u0322\u0006R\t\u0000\u0322\u0323\u0006R\u0004"+ + "\u0000\u0323\u0324\u0006R\u0004\u0000\u0324\u00af\u0001\u0000\u0000\u0000"+ + "\u0325\u0326\u0003\u00a0K\u0000\u0326\u0327\u0001\u0000\u0000\u0000\u0327"+ + "\u0328\u0006S\r\u0000\u0328\u0329\u0006S\f\u0000\u0329\u032a\u0006S\f"+ + "\u0000\u032a\u00b1\u0001\u0000\u0000\u0000\u032b\u032c\u0003d-\u0000\u032c"+ + "\u032d\u0001\u0000\u0000\u0000\u032d\u032e\u0006T\u000e\u0000\u032e\u00b3"+ + "\u0001\u0000\u0000\u0000\u032f\u0330\u0003b,\u0000\u0330\u0331\u0001\u0000"+ + "\u0000\u0000\u0331\u0332\u0006U\u000f\u0000\u0332\u00b5\u0001\u0000\u0000"+ + "\u0000\u0333\u0334\u0005m\u0000\u0000\u0334\u0335\u0005e\u0000\u0000\u0335"+ + "\u0336\u0005t\u0000\u0000\u0336\u0337\u0005a\u0000\u0000\u0337\u0338\u0005"+ + "d\u0000\u0000\u0338\u0339\u0005a\u0000\u0000\u0339\u033a\u0005t\u0000"+ + "\u0000\u033a\u033b\u0005a\u0000\u0000\u033b\u00b7\u0001\u0000\u0000\u0000"+ + "\u033c\u0340\b\n\u0000\u0000\u033d\u033e\u0005/\u0000\u0000\u033e\u0340"+ + "\b\u000b\u0000\u0000\u033f\u033c\u0001\u0000\u0000\u0000\u033f\u033d\u0001"+ + "\u0000\u0000\u0000\u0340\u00b9\u0001\u0000\u0000\u0000\u0341\u0343\u0003"+ + "\u00b8W\u0000\u0342\u0341\u0001\u0000\u0000\u0000\u0343\u0344\u0001\u0000"+ + "\u0000\u0000\u0344\u0342\u0001\u0000\u0000\u0000\u0344\u0345\u0001\u0000"+ + "\u0000\u0000\u0345\u00bb\u0001\u0000\u0000\u0000\u0346\u0347\u0003\u00a4"+ + "M\u0000\u0347\u0348\u0001\u0000\u0000\u0000\u0348\u0349\u0006Y\u0010\u0000"+ + "\u0349\u00bd\u0001\u0000\u0000\u0000\u034a\u034b\u00030\u0013\u0000\u034b"+ + "\u034c\u0001\u0000\u0000\u0000\u034c\u034d\u0006Z\b\u0000\u034d\u00bf"+ + "\u0001\u0000\u0000\u0000\u034e\u034f\u00032\u0014\u0000\u034f\u0350\u0001"+ + "\u0000\u0000\u0000\u0350\u0351\u0006[\b\u0000\u0351\u00c1\u0001\u0000"+ + "\u0000\u0000\u0352\u0353\u00034\u0015\u0000\u0353\u0354\u0001\u0000\u0000"+ + "\u0000\u0354\u0355\u0006\\\b\u0000\u0355\u00c3\u0001\u0000\u0000\u0000"+ + "\u0356\u0357\u0003@\u001b\u0000\u0357\u0358\u0001\u0000\u0000\u0000\u0358"+ + "\u0359\u0006]\u000b\u0000\u0359\u035a\u0006]\f\u0000\u035a\u00c5\u0001"+ + "\u0000\u0000\u0000\u035b\u035c\u0003h/\u0000\u035c\u035d\u0001\u0000\u0000"+ + "\u0000\u035d\u035e\u0006^\u0011\u0000\u035e\u00c7\u0001\u0000\u0000\u0000"+ + "\u035f\u0360\u0003d-\u0000\u0360\u0361\u0001\u0000\u0000\u0000\u0361\u0362"+ + "\u0006_\u000e\u0000\u0362\u00c9\u0001\u0000\u0000\u0000\u0363\u0368\u0003"+ + "D\u001d\u0000\u0364\u0368\u0003B\u001c\u0000\u0365\u0368\u0003R$\u0000"+ + "\u0366\u0368\u0003\u0098G\u0000\u0367\u0363\u0001\u0000\u0000\u0000\u0367"+ + "\u0364\u0001\u0000\u0000\u0000\u0367\u0365\u0001\u0000\u0000\u0000\u0367"+ + "\u0366\u0001\u0000\u0000\u0000\u0368\u00cb\u0001\u0000\u0000\u0000\u0369"+ + "\u036c\u0003D\u001d\u0000\u036a\u036c\u0003\u0098G\u0000\u036b\u0369\u0001"+ + "\u0000\u0000\u0000\u036b\u036a\u0001\u0000\u0000\u0000\u036c\u0370\u0001"+ + "\u0000\u0000\u0000\u036d\u036f\u0003\u00ca`\u0000\u036e\u036d\u0001\u0000"+ + "\u0000\u0000\u036f\u0372\u0001\u0000\u0000\u0000\u0370\u036e\u0001\u0000"+ + "\u0000\u0000\u0370\u0371\u0001\u0000\u0000\u0000\u0371\u037d\u0001\u0000"+ + "\u0000\u0000\u0372\u0370\u0001\u0000\u0000\u0000\u0373\u0376\u0003R$\u0000"+ + "\u0374\u0376\u0003L!\u0000\u0375\u0373\u0001\u0000\u0000\u0000\u0375\u0374"+ + "\u0001\u0000\u0000\u0000\u0376\u0378\u0001\u0000\u0000\u0000\u0377\u0379"+ + "\u0003\u00ca`\u0000\u0378\u0377\u0001\u0000\u0000\u0000\u0379\u037a\u0001"+ + "\u0000\u0000\u0000\u037a\u0378\u0001\u0000\u0000\u0000\u037a\u037b\u0001"+ + "\u0000\u0000\u0000\u037b\u037d\u0001\u0000\u0000\u0000\u037c\u036b\u0001"+ + "\u0000\u0000\u0000\u037c\u0375\u0001\u0000\u0000\u0000\u037d\u00cd\u0001"+ + "\u0000\u0000\u0000\u037e\u037f\u0003\u00a4M\u0000\u037f\u0380\u0001\u0000"+ + "\u0000\u0000\u0380\u0381\u0006b\u0010\u0000\u0381\u00cf\u0001\u0000\u0000"+ + "\u0000\u0382\u0383\u00030\u0013\u0000\u0383\u0384\u0001\u0000\u0000\u0000"+ + "\u0384\u0385\u0006c\b\u0000\u0385\u00d1\u0001\u0000\u0000\u0000\u0386"+ + "\u0387\u00032\u0014\u0000\u0387\u0388\u0001\u0000\u0000\u0000\u0388\u0389"+ + "\u0006d\b\u0000\u0389\u00d3\u0001\u0000\u0000\u0000\u038a\u038b\u0003"+ + "4\u0015\u0000\u038b\u038c\u0001\u0000\u0000\u0000\u038c\u038d\u0006e\b"+ + "\u0000\u038d\u00d5\u0001\u0000\u0000\u0000\u038e\u038f\u0003@\u001b\u0000"+ + "\u038f\u0390\u0001\u0000\u0000\u0000\u0390\u0391\u0006f\u000b\u0000\u0391"+ + "\u0392\u0006f\f\u0000\u0392\u00d7\u0001\u0000\u0000\u0000\u0393\u0394"+ + "\u0003b,\u0000\u0394\u0395\u0001\u0000\u0000\u0000\u0395\u0396\u0006g"+ + "\u000f\u0000\u0396\u00d9\u0001\u0000\u0000\u0000\u0397\u0398\u0003d-\u0000"+ + "\u0398\u0399\u0001\u0000\u0000\u0000\u0399\u039a\u0006h\u000e\u0000\u039a"+ + "\u00db\u0001\u0000\u0000\u0000\u039b\u039c\u0003h/\u0000\u039c\u039d\u0001"+ + "\u0000\u0000\u0000\u039d\u039e\u0006i\u0011\u0000\u039e\u00dd\u0001\u0000"+ + "\u0000\u0000\u039f\u03a0\u0005a\u0000\u0000\u03a0\u03a1\u0005s\u0000\u0000"+ + "\u03a1\u00df\u0001\u0000\u0000\u0000\u03a2\u03a3\u0003\u00a4M\u0000\u03a3"+ + "\u03a4\u0001\u0000\u0000\u0000\u03a4\u03a5\u0006k\u0010\u0000\u03a5\u00e1"+ + "\u0001\u0000\u0000\u0000\u03a6\u03a7\u0003\u00cca\u0000\u03a7\u03a8\u0001"+ + "\u0000\u0000\u0000\u03a8\u03a9\u0006l\u0012\u0000\u03a9\u00e3\u0001\u0000"+ + "\u0000\u0000\u03aa\u03ab\u00030\u0013\u0000\u03ab\u03ac\u0001\u0000\u0000"+ + "\u0000\u03ac\u03ad\u0006m\b\u0000\u03ad\u00e5\u0001\u0000\u0000\u0000"+ + "\u03ae\u03af\u00032\u0014\u0000\u03af\u03b0\u0001\u0000\u0000\u0000\u03b0"+ + "\u03b1\u0006n\b\u0000\u03b1\u00e7\u0001\u0000\u0000\u0000\u03b2\u03b3"+ + "\u00034\u0015\u0000\u03b3\u03b4\u0001\u0000\u0000\u0000\u03b4\u03b5\u0006"+ + "o\b\u0000\u03b5\u00e9\u0001\u0000\u0000\u0000\u03b6\u03b7\u0003@\u001b"+ + "\u0000\u03b7\u03b8\u0001\u0000\u0000\u0000\u03b8\u03b9\u0006p\u000b\u0000"+ + "\u03b9\u03ba\u0006p\f\u0000\u03ba\u00eb\u0001\u0000\u0000\u0000\u03bb"+ + "\u03bc\u0005o\u0000\u0000\u03bc\u03bd\u0005n\u0000\u0000\u03bd\u03be\u0001"+ + "\u0000\u0000\u0000\u03be\u03bf\u0006q\u0013\u0000\u03bf\u00ed\u0001\u0000"+ + "\u0000\u0000\u03c0\u03c1\u0005w\u0000\u0000\u03c1\u03c2\u0005i\u0000\u0000"+ + "\u03c2\u03c3\u0005t\u0000\u0000\u03c3\u03c4\u0005h\u0000\u0000\u03c4\u03c5"+ + "\u0001\u0000\u0000\u0000\u03c5\u03c6\u0006r\u0013\u0000\u03c6\u00ef\u0001"+ + "\u0000\u0000\u0000\u03c7\u03c8\u0003\u00baX\u0000\u03c8\u03c9\u0001\u0000"+ + "\u0000\u0000\u03c9\u03ca\u0006s\u0014\u0000\u03ca\u00f1\u0001\u0000\u0000"+ + "\u0000\u03cb\u03cc\u0003\u00a4M\u0000\u03cc\u03cd\u0001\u0000\u0000\u0000"+ + "\u03cd\u03ce\u0006t\u0010\u0000\u03ce\u00f3\u0001\u0000\u0000\u0000\u03cf"+ + "\u03d0\u00030\u0013\u0000\u03d0\u03d1\u0001\u0000\u0000\u0000\u03d1\u03d2"+ + "\u0006u\b\u0000\u03d2\u00f5\u0001\u0000\u0000\u0000\u03d3\u03d4\u0003"+ + "2\u0014\u0000\u03d4\u03d5\u0001\u0000\u0000\u0000\u03d5\u03d6\u0006v\b"+ + "\u0000\u03d6\u00f7\u0001\u0000\u0000\u0000\u03d7\u03d8\u00034\u0015\u0000"+ + "\u03d8\u03d9\u0001\u0000\u0000\u0000\u03d9\u03da\u0006w\b\u0000\u03da"+ + "\u00f9\u0001\u0000\u0000\u0000\u03db\u03dc\u0003@\u001b\u0000\u03dc\u03dd"+ + "\u0001\u0000\u0000\u0000\u03dd\u03de\u0006x\u000b\u0000\u03de\u03df\u0006"+ + "x\f\u0000\u03df\u03e0\u0006x\f\u0000\u03e0\u00fb\u0001\u0000\u0000\u0000"+ + "\u03e1\u03e2\u0003b,\u0000\u03e2\u03e3\u0001\u0000\u0000\u0000\u03e3\u03e4"+ + "\u0006y\u000f\u0000\u03e4\u00fd\u0001\u0000\u0000\u0000\u03e5\u03e6\u0003"+ + "d-\u0000\u03e6\u03e7\u0001\u0000\u0000\u0000\u03e7\u03e8\u0006z\u000e"+ + "\u0000\u03e8\u00ff\u0001\u0000\u0000\u0000\u03e9\u03ea\u0003h/\u0000\u03ea"+ + "\u03eb\u0001\u0000\u0000\u0000\u03eb\u03ec\u0006{\u0011\u0000\u03ec\u0101"+ + "\u0001\u0000\u0000\u0000\u03ed\u03ee\u0003\u00eer\u0000\u03ee\u03ef\u0001"+ + "\u0000\u0000\u0000\u03ef\u03f0\u0006|\u0015\u0000\u03f0\u0103\u0001\u0000"+ + "\u0000\u0000\u03f1\u03f2\u0003\u00cca\u0000\u03f2\u03f3\u0001\u0000\u0000"+ + "\u0000\u03f3\u03f4\u0006}\u0012\u0000\u03f4\u0105\u0001\u0000\u0000\u0000"+ + "\u03f5\u03f6\u0003\u00a4M\u0000\u03f6\u03f7\u0001\u0000\u0000\u0000\u03f7"+ + "\u03f8\u0006~\u0010\u0000\u03f8\u0107\u0001\u0000\u0000\u0000\u03f9\u03fa"+ + "\u00030\u0013\u0000\u03fa\u03fb\u0001\u0000\u0000\u0000\u03fb\u03fc\u0006"+ + "\u007f\b\u0000\u03fc\u0109\u0001\u0000\u0000\u0000\u03fd\u03fe\u00032"+ + "\u0014\u0000\u03fe\u03ff\u0001\u0000\u0000\u0000\u03ff\u0400\u0006\u0080"+ + "\b\u0000\u0400\u010b\u0001\u0000\u0000\u0000\u0401\u0402\u00034\u0015"+ + "\u0000\u0402\u0403\u0001\u0000\u0000\u0000\u0403\u0404\u0006\u0081\b\u0000"+ + "\u0404\u010d\u0001\u0000\u0000\u0000\u0405\u0406\u0003@\u001b\u0000\u0406"+ + "\u0407\u0001\u0000\u0000\u0000\u0407\u0408\u0006\u0082\u000b\u0000\u0408"+ + "\u0409\u0006\u0082\f\u0000\u0409\u010f\u0001\u0000\u0000\u0000\u040a\u040b"+ + "\u0003h/\u0000\u040b\u040c\u0001\u0000\u0000\u0000\u040c\u040d\u0006\u0083"+ + "\u0011\u0000\u040d\u0111\u0001\u0000\u0000\u0000\u040e\u040f\u0003\u00a4"+ + "M\u0000\u040f\u0410\u0001\u0000\u0000\u0000\u0410\u0411\u0006\u0084\u0010"+ + "\u0000\u0411\u0113\u0001\u0000\u0000\u0000\u0412\u0413\u0003\u00a2L\u0000"+ + "\u0413\u0414\u0001\u0000\u0000\u0000\u0414\u0415\u0006\u0085\u0016\u0000"+ + "\u0415\u0115\u0001\u0000\u0000\u0000\u0416\u0417\u00030\u0013\u0000\u0417"+ + "\u0418\u0001\u0000\u0000\u0000\u0418\u0419\u0006\u0086\b\u0000\u0419\u0117"+ + "\u0001\u0000\u0000\u0000\u041a\u041b\u00032\u0014\u0000\u041b\u041c\u0001"+ + "\u0000\u0000\u0000\u041c\u041d\u0006\u0087\b\u0000\u041d\u0119\u0001\u0000"+ + "\u0000\u0000\u041e\u041f\u00034\u0015\u0000\u041f\u0420\u0001\u0000\u0000"+ + "\u0000\u0420\u0421\u0006\u0088\b\u0000\u0421\u011b\u0001\u0000\u0000\u0000"+ + "\u0422\u0423\u0003@\u001b\u0000\u0423\u0424\u0001\u0000\u0000\u0000\u0424"+ + "\u0425\u0006\u0089\u000b\u0000\u0425\u0426\u0006\u0089\f\u0000\u0426\u011d"+ + "\u0001\u0000\u0000\u0000\u0427\u0428\u0005i\u0000\u0000\u0428\u0429\u0005"+ + "n\u0000\u0000\u0429\u042a\u0005f\u0000\u0000\u042a\u042b\u0005o\u0000"+ + "\u0000\u042b\u011f\u0001\u0000\u0000\u0000\u042c\u042d\u0005f\u0000\u0000"+ + "\u042d\u042e\u0005u\u0000\u0000\u042e\u042f\u0005n\u0000\u0000\u042f\u0430"+ + "\u0005c\u0000\u0000\u0430\u0431\u0005t\u0000\u0000\u0431\u0432\u0005i"+ + "\u0000\u0000\u0432\u0433\u0005o\u0000\u0000\u0433\u0434\u0005n\u0000\u0000"+ + "\u0434\u0435\u0005s\u0000\u0000\u0435\u0121\u0001\u0000\u0000\u0000\u0436"+ + "\u0437\u00030\u0013\u0000\u0437\u0438\u0001\u0000\u0000\u0000\u0438\u0439"+ + "\u0006\u008c\b\u0000\u0439\u0123\u0001\u0000\u0000\u0000\u043a\u043b\u0003"+ + "2\u0014\u0000\u043b\u043c\u0001\u0000\u0000\u0000\u043c\u043d\u0006\u008d"+ + "\b\u0000\u043d\u0125\u0001\u0000\u0000\u0000\u043e\u043f\u00034\u0015"+ + "\u0000\u043f\u0440\u0001\u0000\u0000\u0000\u0440\u0441\u0006\u008e\b\u0000"+ + "\u0441\u0127\u0001\u0000\u0000\u00001\u0000\u0001\u0002\u0003\u0004\u0005"+ + "\u0006\u0007\b\t\u01c4\u01ce\u01d2\u01d5\u01de\u01e0\u01eb\u0214\u0219"+ + "\u0222\u0229\u022e\u0230\u023b\u0243\u0246\u0248\u024d\u0252\u0258\u025f"+ + "\u0264\u026a\u026d\u0275\u0279\u02f8\u02fd\u0302\u0304\u030a\u033f\u0344"+ + "\u0367\u036b\u0370\u0375\u037a\u037c\u0017\u0005\u0002\u0000\u0005\u0004"+ + "\u0000\u0005\u0006\u0000\u0005\u0001\u0000\u0005\u0003\u0000\u0005\b\u0000"+ + "\u0005\u0005\u0000\u0005\t\u0000\u0000\u0001\u0000\u0007?\u0000\u0005"+ + "\u0000\u0000\u0007\u001a\u0000\u0004\u0000\u0000\u0007@\u0000\u0007\""+ + "\u0000\u0007!\u0000\u0007B\u0000\u0007$\u0000\u0007K\u0000\u0005\u0007"+ + "\u0000\u0007G\u0000\u0007T\u0000\u0007A\u0000"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp index 658e09ca4b190..3acc73b1b592c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp @@ -25,15 +25,15 @@ null null null null -null +'|' null null null 'by' 'and' 'asc' -null -null +'=' +',' 'desc' '.' 'false' @@ -51,8 +51,6 @@ null 'rlike' ')' 'true' -'info' -'functions' '==' '!=' '<' @@ -71,8 +69,19 @@ null null null null -'as' 'metadata' +null +null +null +null +null +null +null +null +'as' +null +null +null 'on' 'with' null @@ -81,6 +90,14 @@ null null null null +null +null +null +'info' +'functions' +null +null +null token symbolic names: null @@ -135,8 +152,6 @@ PARAM RLIKE RP TRUE -INFO -FUNCTIONS EQ NEQ LT @@ -155,16 +170,35 @@ QUOTED_IDENTIFIER EXPR_LINE_COMMENT EXPR_MULTILINE_COMMENT EXPR_WS -AS METADATA +FROM_UNQUOTED_IDENTIFIER +FROM_LINE_COMMENT +FROM_MULTILINE_COMMENT +FROM_WS +PROJECT_UNQUOTED_IDENTIFIER +PROJECT_LINE_COMMENT +PROJECT_MULTILINE_COMMENT +PROJECT_WS +AS +RENAME_LINE_COMMENT +RENAME_MULTILINE_COMMENT +RENAME_WS ON WITH -SRC_UNQUOTED_IDENTIFIER -SRC_QUOTED_IDENTIFIER -SRC_LINE_COMMENT -SRC_MULTILINE_COMMENT -SRC_WS -EXPLAIN_PIPE +ENRICH_LINE_COMMENT +ENRICH_MULTILINE_COMMENT +ENRICH_WS +ENRICH_FIELD_LINE_COMMENT +ENRICH_FIELD_MULTILINE_COMMENT +ENRICH_FIELD_WS +MVEXPAND_LINE_COMMENT +MVEXPAND_MULTILINE_COMMENT +MVEXPAND_WS +INFO +FUNCTIONS +SHOW_LINE_COMMENT +SHOW_MULTILINE_COMMENT +SHOW_WS rule names: singleStatement @@ -187,9 +221,11 @@ evalCommand statsCommand inlinestatsCommand grouping -sourceIdentifier +fromIdentifier qualifiedName +qualifiedNamePattern identifier +identifierPattern constant limitCommand sortCommand @@ -217,4 +253,4 @@ enrichWithClause atn: -[4, 1, 81, 505, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 104, 8, 1, 10, 1, 12, 1, 107, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 113, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 128, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 140, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 147, 8, 5, 10, 5, 12, 5, 150, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 157, 8, 5, 1, 5, 1, 5, 3, 5, 161, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 169, 8, 5, 10, 5, 12, 5, 172, 9, 5, 1, 6, 1, 6, 3, 6, 176, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 183, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 188, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 3, 7, 195, 8, 7, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 201, 8, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 5, 8, 209, 8, 8, 10, 8, 12, 8, 212, 9, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 221, 8, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 5, 10, 229, 8, 10, 10, 10, 12, 10, 232, 9, 10, 3, 10, 234, 8, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 5, 12, 244, 8, 12, 10, 12, 12, 12, 247, 9, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 3, 13, 254, 8, 13, 1, 14, 1, 14, 1, 14, 1, 14, 5, 14, 260, 8, 14, 10, 14, 12, 14, 263, 9, 14, 1, 14, 3, 14, 266, 8, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 5, 15, 273, 8, 15, 10, 15, 12, 15, 276, 9, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 3, 17, 285, 8, 17, 1, 17, 1, 17, 3, 17, 289, 8, 17, 1, 18, 1, 18, 1, 18, 1, 18, 3, 18, 295, 8, 18, 1, 19, 1, 19, 1, 19, 5, 19, 300, 8, 19, 10, 19, 12, 19, 303, 9, 19, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 5, 21, 310, 8, 21, 10, 21, 12, 21, 313, 9, 21, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 5, 23, 330, 8, 23, 10, 23, 12, 23, 333, 9, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 5, 23, 341, 8, 23, 10, 23, 12, 23, 344, 9, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 5, 23, 352, 8, 23, 10, 23, 12, 23, 355, 9, 23, 1, 23, 1, 23, 3, 23, 359, 8, 23, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 5, 25, 368, 8, 25, 10, 25, 12, 25, 371, 9, 25, 1, 26, 1, 26, 3, 26, 375, 8, 26, 1, 26, 1, 26, 3, 26, 379, 8, 26, 1, 27, 1, 27, 1, 27, 1, 27, 5, 27, 385, 8, 27, 10, 27, 12, 27, 388, 9, 27, 1, 27, 1, 27, 1, 27, 1, 27, 5, 27, 394, 8, 27, 10, 27, 12, 27, 397, 9, 27, 3, 27, 399, 8, 27, 1, 28, 1, 28, 1, 28, 1, 28, 5, 28, 405, 8, 28, 10, 28, 12, 28, 408, 9, 28, 1, 29, 1, 29, 1, 29, 1, 29, 5, 29, 414, 8, 29, 10, 29, 12, 29, 417, 9, 29, 1, 30, 1, 30, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 1, 31, 3, 31, 427, 8, 31, 1, 32, 1, 32, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 1, 34, 1, 34, 1, 34, 5, 34, 439, 8, 34, 10, 34, 12, 34, 442, 9, 34, 1, 35, 1, 35, 1, 35, 1, 35, 1, 36, 1, 36, 1, 37, 1, 37, 3, 37, 452, 8, 37, 1, 38, 3, 38, 455, 8, 38, 1, 38, 1, 38, 1, 39, 3, 39, 460, 8, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 479, 8, 44, 1, 45, 1, 45, 1, 45, 1, 45, 3, 45, 485, 8, 45, 1, 45, 1, 45, 1, 45, 1, 45, 5, 45, 491, 8, 45, 10, 45, 12, 45, 494, 9, 45, 3, 45, 496, 8, 45, 1, 46, 1, 46, 1, 46, 3, 46, 501, 8, 46, 1, 46, 1, 46, 1, 46, 0, 3, 2, 10, 16, 47, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 0, 8, 1, 0, 60, 61, 1, 0, 62, 64, 1, 0, 76, 77, 1, 0, 67, 68, 2, 0, 32, 32, 35, 35, 1, 0, 38, 39, 2, 0, 37, 37, 51, 51, 1, 0, 54, 59, 535, 0, 94, 1, 0, 0, 0, 2, 97, 1, 0, 0, 0, 4, 112, 1, 0, 0, 0, 6, 127, 1, 0, 0, 0, 8, 129, 1, 0, 0, 0, 10, 160, 1, 0, 0, 0, 12, 187, 1, 0, 0, 0, 14, 194, 1, 0, 0, 0, 16, 200, 1, 0, 0, 0, 18, 220, 1, 0, 0, 0, 20, 222, 1, 0, 0, 0, 22, 237, 1, 0, 0, 0, 24, 240, 1, 0, 0, 0, 26, 253, 1, 0, 0, 0, 28, 255, 1, 0, 0, 0, 30, 267, 1, 0, 0, 0, 32, 279, 1, 0, 0, 0, 34, 282, 1, 0, 0, 0, 36, 290, 1, 0, 0, 0, 38, 296, 1, 0, 0, 0, 40, 304, 1, 0, 0, 0, 42, 306, 1, 0, 0, 0, 44, 314, 1, 0, 0, 0, 46, 358, 1, 0, 0, 0, 48, 360, 1, 0, 0, 0, 50, 363, 1, 0, 0, 0, 52, 372, 1, 0, 0, 0, 54, 398, 1, 0, 0, 0, 56, 400, 1, 0, 0, 0, 58, 409, 1, 0, 0, 0, 60, 418, 1, 0, 0, 0, 62, 422, 1, 0, 0, 0, 64, 428, 1, 0, 0, 0, 66, 432, 1, 0, 0, 0, 68, 435, 1, 0, 0, 0, 70, 443, 1, 0, 0, 0, 72, 447, 1, 0, 0, 0, 74, 451, 1, 0, 0, 0, 76, 454, 1, 0, 0, 0, 78, 459, 1, 0, 0, 0, 80, 463, 1, 0, 0, 0, 82, 465, 1, 0, 0, 0, 84, 467, 1, 0, 0, 0, 86, 470, 1, 0, 0, 0, 88, 478, 1, 0, 0, 0, 90, 480, 1, 0, 0, 0, 92, 500, 1, 0, 0, 0, 94, 95, 3, 2, 1, 0, 95, 96, 5, 0, 0, 1, 96, 1, 1, 0, 0, 0, 97, 98, 6, 1, -1, 0, 98, 99, 3, 4, 2, 0, 99, 105, 1, 0, 0, 0, 100, 101, 10, 1, 0, 0, 101, 102, 5, 26, 0, 0, 102, 104, 3, 6, 3, 0, 103, 100, 1, 0, 0, 0, 104, 107, 1, 0, 0, 0, 105, 103, 1, 0, 0, 0, 105, 106, 1, 0, 0, 0, 106, 3, 1, 0, 0, 0, 107, 105, 1, 0, 0, 0, 108, 113, 3, 84, 42, 0, 109, 113, 3, 28, 14, 0, 110, 113, 3, 22, 11, 0, 111, 113, 3, 88, 44, 0, 112, 108, 1, 0, 0, 0, 112, 109, 1, 0, 0, 0, 112, 110, 1, 0, 0, 0, 112, 111, 1, 0, 0, 0, 113, 5, 1, 0, 0, 0, 114, 128, 3, 32, 16, 0, 115, 128, 3, 36, 18, 0, 116, 128, 3, 48, 24, 0, 117, 128, 3, 54, 27, 0, 118, 128, 3, 50, 25, 0, 119, 128, 3, 34, 17, 0, 120, 128, 3, 8, 4, 0, 121, 128, 3, 56, 28, 0, 122, 128, 3, 58, 29, 0, 123, 128, 3, 62, 31, 0, 124, 128, 3, 64, 32, 0, 125, 128, 3, 90, 45, 0, 126, 128, 3, 66, 33, 0, 127, 114, 1, 0, 0, 0, 127, 115, 1, 0, 0, 0, 127, 116, 1, 0, 0, 0, 127, 117, 1, 0, 0, 0, 127, 118, 1, 0, 0, 0, 127, 119, 1, 0, 0, 0, 127, 120, 1, 0, 0, 0, 127, 121, 1, 0, 0, 0, 127, 122, 1, 0, 0, 0, 127, 123, 1, 0, 0, 0, 127, 124, 1, 0, 0, 0, 127, 125, 1, 0, 0, 0, 127, 126, 1, 0, 0, 0, 128, 7, 1, 0, 0, 0, 129, 130, 5, 18, 0, 0, 130, 131, 3, 10, 5, 0, 131, 9, 1, 0, 0, 0, 132, 133, 6, 5, -1, 0, 133, 134, 5, 44, 0, 0, 134, 161, 3, 10, 5, 7, 135, 161, 3, 14, 7, 0, 136, 161, 3, 12, 6, 0, 137, 139, 3, 14, 7, 0, 138, 140, 5, 44, 0, 0, 139, 138, 1, 0, 0, 0, 139, 140, 1, 0, 0, 0, 140, 141, 1, 0, 0, 0, 141, 142, 5, 41, 0, 0, 142, 143, 5, 40, 0, 0, 143, 148, 3, 14, 7, 0, 144, 145, 5, 34, 0, 0, 145, 147, 3, 14, 7, 0, 146, 144, 1, 0, 0, 0, 147, 150, 1, 0, 0, 0, 148, 146, 1, 0, 0, 0, 148, 149, 1, 0, 0, 0, 149, 151, 1, 0, 0, 0, 150, 148, 1, 0, 0, 0, 151, 152, 5, 50, 0, 0, 152, 161, 1, 0, 0, 0, 153, 154, 3, 14, 7, 0, 154, 156, 5, 42, 0, 0, 155, 157, 5, 44, 0, 0, 156, 155, 1, 0, 0, 0, 156, 157, 1, 0, 0, 0, 157, 158, 1, 0, 0, 0, 158, 159, 5, 45, 0, 0, 159, 161, 1, 0, 0, 0, 160, 132, 1, 0, 0, 0, 160, 135, 1, 0, 0, 0, 160, 136, 1, 0, 0, 0, 160, 137, 1, 0, 0, 0, 160, 153, 1, 0, 0, 0, 161, 170, 1, 0, 0, 0, 162, 163, 10, 4, 0, 0, 163, 164, 5, 31, 0, 0, 164, 169, 3, 10, 5, 5, 165, 166, 10, 3, 0, 0, 166, 167, 5, 47, 0, 0, 167, 169, 3, 10, 5, 4, 168, 162, 1, 0, 0, 0, 168, 165, 1, 0, 0, 0, 169, 172, 1, 0, 0, 0, 170, 168, 1, 0, 0, 0, 170, 171, 1, 0, 0, 0, 171, 11, 1, 0, 0, 0, 172, 170, 1, 0, 0, 0, 173, 175, 3, 14, 7, 0, 174, 176, 5, 44, 0, 0, 175, 174, 1, 0, 0, 0, 175, 176, 1, 0, 0, 0, 176, 177, 1, 0, 0, 0, 177, 178, 5, 43, 0, 0, 178, 179, 3, 80, 40, 0, 179, 188, 1, 0, 0, 0, 180, 182, 3, 14, 7, 0, 181, 183, 5, 44, 0, 0, 182, 181, 1, 0, 0, 0, 182, 183, 1, 0, 0, 0, 183, 184, 1, 0, 0, 0, 184, 185, 5, 49, 0, 0, 185, 186, 3, 80, 40, 0, 186, 188, 1, 0, 0, 0, 187, 173, 1, 0, 0, 0, 187, 180, 1, 0, 0, 0, 188, 13, 1, 0, 0, 0, 189, 195, 3, 16, 8, 0, 190, 191, 3, 16, 8, 0, 191, 192, 3, 82, 41, 0, 192, 193, 3, 16, 8, 0, 193, 195, 1, 0, 0, 0, 194, 189, 1, 0, 0, 0, 194, 190, 1, 0, 0, 0, 195, 15, 1, 0, 0, 0, 196, 197, 6, 8, -1, 0, 197, 201, 3, 18, 9, 0, 198, 199, 7, 0, 0, 0, 199, 201, 3, 16, 8, 3, 200, 196, 1, 0, 0, 0, 200, 198, 1, 0, 0, 0, 201, 210, 1, 0, 0, 0, 202, 203, 10, 2, 0, 0, 203, 204, 7, 1, 0, 0, 204, 209, 3, 16, 8, 3, 205, 206, 10, 1, 0, 0, 206, 207, 7, 0, 0, 0, 207, 209, 3, 16, 8, 2, 208, 202, 1, 0, 0, 0, 208, 205, 1, 0, 0, 0, 209, 212, 1, 0, 0, 0, 210, 208, 1, 0, 0, 0, 210, 211, 1, 0, 0, 0, 211, 17, 1, 0, 0, 0, 212, 210, 1, 0, 0, 0, 213, 221, 3, 46, 23, 0, 214, 221, 3, 42, 21, 0, 215, 221, 3, 20, 10, 0, 216, 217, 5, 40, 0, 0, 217, 218, 3, 10, 5, 0, 218, 219, 5, 50, 0, 0, 219, 221, 1, 0, 0, 0, 220, 213, 1, 0, 0, 0, 220, 214, 1, 0, 0, 0, 220, 215, 1, 0, 0, 0, 220, 216, 1, 0, 0, 0, 221, 19, 1, 0, 0, 0, 222, 223, 3, 44, 22, 0, 223, 233, 5, 40, 0, 0, 224, 234, 5, 62, 0, 0, 225, 230, 3, 10, 5, 0, 226, 227, 5, 34, 0, 0, 227, 229, 3, 10, 5, 0, 228, 226, 1, 0, 0, 0, 229, 232, 1, 0, 0, 0, 230, 228, 1, 0, 0, 0, 230, 231, 1, 0, 0, 0, 231, 234, 1, 0, 0, 0, 232, 230, 1, 0, 0, 0, 233, 224, 1, 0, 0, 0, 233, 225, 1, 0, 0, 0, 233, 234, 1, 0, 0, 0, 234, 235, 1, 0, 0, 0, 235, 236, 5, 50, 0, 0, 236, 21, 1, 0, 0, 0, 237, 238, 5, 14, 0, 0, 238, 239, 3, 24, 12, 0, 239, 23, 1, 0, 0, 0, 240, 245, 3, 26, 13, 0, 241, 242, 5, 34, 0, 0, 242, 244, 3, 26, 13, 0, 243, 241, 1, 0, 0, 0, 244, 247, 1, 0, 0, 0, 245, 243, 1, 0, 0, 0, 245, 246, 1, 0, 0, 0, 246, 25, 1, 0, 0, 0, 247, 245, 1, 0, 0, 0, 248, 254, 3, 10, 5, 0, 249, 250, 3, 42, 21, 0, 250, 251, 5, 33, 0, 0, 251, 252, 3, 10, 5, 0, 252, 254, 1, 0, 0, 0, 253, 248, 1, 0, 0, 0, 253, 249, 1, 0, 0, 0, 254, 27, 1, 0, 0, 0, 255, 256, 5, 6, 0, 0, 256, 261, 3, 40, 20, 0, 257, 258, 5, 34, 0, 0, 258, 260, 3, 40, 20, 0, 259, 257, 1, 0, 0, 0, 260, 263, 1, 0, 0, 0, 261, 259, 1, 0, 0, 0, 261, 262, 1, 0, 0, 0, 262, 265, 1, 0, 0, 0, 263, 261, 1, 0, 0, 0, 264, 266, 3, 30, 15, 0, 265, 264, 1, 0, 0, 0, 265, 266, 1, 0, 0, 0, 266, 29, 1, 0, 0, 0, 267, 268, 5, 65, 0, 0, 268, 269, 5, 73, 0, 0, 269, 274, 3, 40, 20, 0, 270, 271, 5, 34, 0, 0, 271, 273, 3, 40, 20, 0, 272, 270, 1, 0, 0, 0, 273, 276, 1, 0, 0, 0, 274, 272, 1, 0, 0, 0, 274, 275, 1, 0, 0, 0, 275, 277, 1, 0, 0, 0, 276, 274, 1, 0, 0, 0, 277, 278, 5, 66, 0, 0, 278, 31, 1, 0, 0, 0, 279, 280, 5, 4, 0, 0, 280, 281, 3, 24, 12, 0, 281, 33, 1, 0, 0, 0, 282, 284, 5, 17, 0, 0, 283, 285, 3, 24, 12, 0, 284, 283, 1, 0, 0, 0, 284, 285, 1, 0, 0, 0, 285, 288, 1, 0, 0, 0, 286, 287, 5, 30, 0, 0, 287, 289, 3, 38, 19, 0, 288, 286, 1, 0, 0, 0, 288, 289, 1, 0, 0, 0, 289, 35, 1, 0, 0, 0, 290, 291, 5, 8, 0, 0, 291, 294, 3, 24, 12, 0, 292, 293, 5, 30, 0, 0, 293, 295, 3, 38, 19, 0, 294, 292, 1, 0, 0, 0, 294, 295, 1, 0, 0, 0, 295, 37, 1, 0, 0, 0, 296, 301, 3, 42, 21, 0, 297, 298, 5, 34, 0, 0, 298, 300, 3, 42, 21, 0, 299, 297, 1, 0, 0, 0, 300, 303, 1, 0, 0, 0, 301, 299, 1, 0, 0, 0, 301, 302, 1, 0, 0, 0, 302, 39, 1, 0, 0, 0, 303, 301, 1, 0, 0, 0, 304, 305, 7, 2, 0, 0, 305, 41, 1, 0, 0, 0, 306, 311, 3, 44, 22, 0, 307, 308, 5, 36, 0, 0, 308, 310, 3, 44, 22, 0, 309, 307, 1, 0, 0, 0, 310, 313, 1, 0, 0, 0, 311, 309, 1, 0, 0, 0, 311, 312, 1, 0, 0, 0, 312, 43, 1, 0, 0, 0, 313, 311, 1, 0, 0, 0, 314, 315, 7, 3, 0, 0, 315, 45, 1, 0, 0, 0, 316, 359, 5, 45, 0, 0, 317, 318, 3, 78, 39, 0, 318, 319, 5, 67, 0, 0, 319, 359, 1, 0, 0, 0, 320, 359, 3, 76, 38, 0, 321, 359, 3, 78, 39, 0, 322, 359, 3, 72, 36, 0, 323, 359, 5, 48, 0, 0, 324, 359, 3, 80, 40, 0, 325, 326, 5, 65, 0, 0, 326, 331, 3, 74, 37, 0, 327, 328, 5, 34, 0, 0, 328, 330, 3, 74, 37, 0, 329, 327, 1, 0, 0, 0, 330, 333, 1, 0, 0, 0, 331, 329, 1, 0, 0, 0, 331, 332, 1, 0, 0, 0, 332, 334, 1, 0, 0, 0, 333, 331, 1, 0, 0, 0, 334, 335, 5, 66, 0, 0, 335, 359, 1, 0, 0, 0, 336, 337, 5, 65, 0, 0, 337, 342, 3, 72, 36, 0, 338, 339, 5, 34, 0, 0, 339, 341, 3, 72, 36, 0, 340, 338, 1, 0, 0, 0, 341, 344, 1, 0, 0, 0, 342, 340, 1, 0, 0, 0, 342, 343, 1, 0, 0, 0, 343, 345, 1, 0, 0, 0, 344, 342, 1, 0, 0, 0, 345, 346, 5, 66, 0, 0, 346, 359, 1, 0, 0, 0, 347, 348, 5, 65, 0, 0, 348, 353, 3, 80, 40, 0, 349, 350, 5, 34, 0, 0, 350, 352, 3, 80, 40, 0, 351, 349, 1, 0, 0, 0, 352, 355, 1, 0, 0, 0, 353, 351, 1, 0, 0, 0, 353, 354, 1, 0, 0, 0, 354, 356, 1, 0, 0, 0, 355, 353, 1, 0, 0, 0, 356, 357, 5, 66, 0, 0, 357, 359, 1, 0, 0, 0, 358, 316, 1, 0, 0, 0, 358, 317, 1, 0, 0, 0, 358, 320, 1, 0, 0, 0, 358, 321, 1, 0, 0, 0, 358, 322, 1, 0, 0, 0, 358, 323, 1, 0, 0, 0, 358, 324, 1, 0, 0, 0, 358, 325, 1, 0, 0, 0, 358, 336, 1, 0, 0, 0, 358, 347, 1, 0, 0, 0, 359, 47, 1, 0, 0, 0, 360, 361, 5, 10, 0, 0, 361, 362, 5, 28, 0, 0, 362, 49, 1, 0, 0, 0, 363, 364, 5, 16, 0, 0, 364, 369, 3, 52, 26, 0, 365, 366, 5, 34, 0, 0, 366, 368, 3, 52, 26, 0, 367, 365, 1, 0, 0, 0, 368, 371, 1, 0, 0, 0, 369, 367, 1, 0, 0, 0, 369, 370, 1, 0, 0, 0, 370, 51, 1, 0, 0, 0, 371, 369, 1, 0, 0, 0, 372, 374, 3, 10, 5, 0, 373, 375, 7, 4, 0, 0, 374, 373, 1, 0, 0, 0, 374, 375, 1, 0, 0, 0, 375, 378, 1, 0, 0, 0, 376, 377, 5, 46, 0, 0, 377, 379, 7, 5, 0, 0, 378, 376, 1, 0, 0, 0, 378, 379, 1, 0, 0, 0, 379, 53, 1, 0, 0, 0, 380, 381, 5, 9, 0, 0, 381, 386, 3, 40, 20, 0, 382, 383, 5, 34, 0, 0, 383, 385, 3, 40, 20, 0, 384, 382, 1, 0, 0, 0, 385, 388, 1, 0, 0, 0, 386, 384, 1, 0, 0, 0, 386, 387, 1, 0, 0, 0, 387, 399, 1, 0, 0, 0, 388, 386, 1, 0, 0, 0, 389, 390, 5, 12, 0, 0, 390, 395, 3, 40, 20, 0, 391, 392, 5, 34, 0, 0, 392, 394, 3, 40, 20, 0, 393, 391, 1, 0, 0, 0, 394, 397, 1, 0, 0, 0, 395, 393, 1, 0, 0, 0, 395, 396, 1, 0, 0, 0, 396, 399, 1, 0, 0, 0, 397, 395, 1, 0, 0, 0, 398, 380, 1, 0, 0, 0, 398, 389, 1, 0, 0, 0, 399, 55, 1, 0, 0, 0, 400, 401, 5, 2, 0, 0, 401, 406, 3, 40, 20, 0, 402, 403, 5, 34, 0, 0, 403, 405, 3, 40, 20, 0, 404, 402, 1, 0, 0, 0, 405, 408, 1, 0, 0, 0, 406, 404, 1, 0, 0, 0, 406, 407, 1, 0, 0, 0, 407, 57, 1, 0, 0, 0, 408, 406, 1, 0, 0, 0, 409, 410, 5, 13, 0, 0, 410, 415, 3, 60, 30, 0, 411, 412, 5, 34, 0, 0, 412, 414, 3, 60, 30, 0, 413, 411, 1, 0, 0, 0, 414, 417, 1, 0, 0, 0, 415, 413, 1, 0, 0, 0, 415, 416, 1, 0, 0, 0, 416, 59, 1, 0, 0, 0, 417, 415, 1, 0, 0, 0, 418, 419, 3, 40, 20, 0, 419, 420, 5, 72, 0, 0, 420, 421, 3, 40, 20, 0, 421, 61, 1, 0, 0, 0, 422, 423, 5, 1, 0, 0, 423, 424, 3, 18, 9, 0, 424, 426, 3, 80, 40, 0, 425, 427, 3, 68, 34, 0, 426, 425, 1, 0, 0, 0, 426, 427, 1, 0, 0, 0, 427, 63, 1, 0, 0, 0, 428, 429, 5, 7, 0, 0, 429, 430, 3, 18, 9, 0, 430, 431, 3, 80, 40, 0, 431, 65, 1, 0, 0, 0, 432, 433, 5, 11, 0, 0, 433, 434, 3, 40, 20, 0, 434, 67, 1, 0, 0, 0, 435, 440, 3, 70, 35, 0, 436, 437, 5, 34, 0, 0, 437, 439, 3, 70, 35, 0, 438, 436, 1, 0, 0, 0, 439, 442, 1, 0, 0, 0, 440, 438, 1, 0, 0, 0, 440, 441, 1, 0, 0, 0, 441, 69, 1, 0, 0, 0, 442, 440, 1, 0, 0, 0, 443, 444, 3, 44, 22, 0, 444, 445, 5, 33, 0, 0, 445, 446, 3, 46, 23, 0, 446, 71, 1, 0, 0, 0, 447, 448, 7, 6, 0, 0, 448, 73, 1, 0, 0, 0, 449, 452, 3, 76, 38, 0, 450, 452, 3, 78, 39, 0, 451, 449, 1, 0, 0, 0, 451, 450, 1, 0, 0, 0, 452, 75, 1, 0, 0, 0, 453, 455, 7, 0, 0, 0, 454, 453, 1, 0, 0, 0, 454, 455, 1, 0, 0, 0, 455, 456, 1, 0, 0, 0, 456, 457, 5, 29, 0, 0, 457, 77, 1, 0, 0, 0, 458, 460, 7, 0, 0, 0, 459, 458, 1, 0, 0, 0, 459, 460, 1, 0, 0, 0, 460, 461, 1, 0, 0, 0, 461, 462, 5, 28, 0, 0, 462, 79, 1, 0, 0, 0, 463, 464, 5, 27, 0, 0, 464, 81, 1, 0, 0, 0, 465, 466, 7, 7, 0, 0, 466, 83, 1, 0, 0, 0, 467, 468, 5, 5, 0, 0, 468, 469, 3, 86, 43, 0, 469, 85, 1, 0, 0, 0, 470, 471, 5, 65, 0, 0, 471, 472, 3, 2, 1, 0, 472, 473, 5, 66, 0, 0, 473, 87, 1, 0, 0, 0, 474, 475, 5, 15, 0, 0, 475, 479, 5, 52, 0, 0, 476, 477, 5, 15, 0, 0, 477, 479, 5, 53, 0, 0, 478, 474, 1, 0, 0, 0, 478, 476, 1, 0, 0, 0, 479, 89, 1, 0, 0, 0, 480, 481, 5, 3, 0, 0, 481, 484, 3, 40, 20, 0, 482, 483, 5, 74, 0, 0, 483, 485, 3, 40, 20, 0, 484, 482, 1, 0, 0, 0, 484, 485, 1, 0, 0, 0, 485, 495, 1, 0, 0, 0, 486, 487, 5, 75, 0, 0, 487, 492, 3, 92, 46, 0, 488, 489, 5, 34, 0, 0, 489, 491, 3, 92, 46, 0, 490, 488, 1, 0, 0, 0, 491, 494, 1, 0, 0, 0, 492, 490, 1, 0, 0, 0, 492, 493, 1, 0, 0, 0, 493, 496, 1, 0, 0, 0, 494, 492, 1, 0, 0, 0, 495, 486, 1, 0, 0, 0, 495, 496, 1, 0, 0, 0, 496, 91, 1, 0, 0, 0, 497, 498, 3, 40, 20, 0, 498, 499, 5, 33, 0, 0, 499, 501, 1, 0, 0, 0, 500, 497, 1, 0, 0, 0, 500, 501, 1, 0, 0, 0, 501, 502, 1, 0, 0, 0, 502, 503, 3, 40, 20, 0, 503, 93, 1, 0, 0, 0, 51, 105, 112, 127, 139, 148, 156, 160, 168, 170, 175, 182, 187, 194, 200, 208, 210, 220, 230, 233, 245, 253, 261, 265, 274, 284, 288, 294, 301, 311, 331, 342, 353, 358, 369, 374, 378, 386, 395, 398, 406, 415, 426, 440, 451, 454, 459, 478, 484, 492, 495, 500] \ No newline at end of file +[4, 1, 98, 519, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 108, 8, 1, 10, 1, 12, 1, 111, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 117, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 132, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 144, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 151, 8, 5, 10, 5, 12, 5, 154, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 161, 8, 5, 1, 5, 1, 5, 3, 5, 165, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 173, 8, 5, 10, 5, 12, 5, 176, 9, 5, 1, 6, 1, 6, 3, 6, 180, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 187, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 192, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 3, 7, 199, 8, 7, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 205, 8, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 5, 8, 213, 8, 8, 10, 8, 12, 8, 216, 9, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 225, 8, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 5, 10, 233, 8, 10, 10, 10, 12, 10, 236, 9, 10, 3, 10, 238, 8, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 5, 12, 248, 8, 12, 10, 12, 12, 12, 251, 9, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 3, 13, 258, 8, 13, 1, 14, 1, 14, 1, 14, 1, 14, 5, 14, 264, 8, 14, 10, 14, 12, 14, 267, 9, 14, 1, 14, 3, 14, 270, 8, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 5, 15, 277, 8, 15, 10, 15, 12, 15, 280, 9, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 3, 17, 289, 8, 17, 1, 17, 1, 17, 3, 17, 293, 8, 17, 1, 18, 1, 18, 1, 18, 1, 18, 3, 18, 299, 8, 18, 1, 19, 1, 19, 1, 19, 5, 19, 304, 8, 19, 10, 19, 12, 19, 307, 9, 19, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 5, 21, 314, 8, 21, 10, 21, 12, 21, 317, 9, 21, 1, 22, 1, 22, 1, 22, 5, 22, 322, 8, 22, 10, 22, 12, 22, 325, 9, 22, 1, 23, 1, 23, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 5, 25, 344, 8, 25, 10, 25, 12, 25, 347, 9, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 5, 25, 355, 8, 25, 10, 25, 12, 25, 358, 9, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 5, 25, 366, 8, 25, 10, 25, 12, 25, 369, 9, 25, 1, 25, 1, 25, 3, 25, 373, 8, 25, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 27, 5, 27, 382, 8, 27, 10, 27, 12, 27, 385, 9, 27, 1, 28, 1, 28, 3, 28, 389, 8, 28, 1, 28, 1, 28, 3, 28, 393, 8, 28, 1, 29, 1, 29, 1, 29, 1, 29, 5, 29, 399, 8, 29, 10, 29, 12, 29, 402, 9, 29, 1, 29, 1, 29, 1, 29, 1, 29, 5, 29, 408, 8, 29, 10, 29, 12, 29, 411, 9, 29, 3, 29, 413, 8, 29, 1, 30, 1, 30, 1, 30, 1, 30, 5, 30, 419, 8, 30, 10, 30, 12, 30, 422, 9, 30, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 428, 8, 31, 10, 31, 12, 31, 431, 9, 31, 1, 32, 1, 32, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 1, 33, 3, 33, 441, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 35, 1, 35, 1, 35, 1, 36, 1, 36, 1, 36, 5, 36, 453, 8, 36, 10, 36, 12, 36, 456, 9, 36, 1, 37, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 39, 1, 39, 3, 39, 466, 8, 39, 1, 40, 3, 40, 469, 8, 40, 1, 40, 1, 40, 1, 41, 3, 41, 474, 8, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 3, 46, 493, 8, 46, 1, 47, 1, 47, 1, 47, 1, 47, 3, 47, 499, 8, 47, 1, 47, 1, 47, 1, 47, 1, 47, 5, 47, 505, 8, 47, 10, 47, 12, 47, 508, 9, 47, 3, 47, 510, 8, 47, 1, 48, 1, 48, 1, 48, 3, 48, 515, 8, 48, 1, 48, 1, 48, 1, 48, 0, 3, 2, 10, 16, 49, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 0, 9, 1, 0, 58, 59, 1, 0, 60, 62, 2, 0, 66, 66, 71, 71, 1, 0, 65, 66, 2, 0, 66, 66, 75, 75, 2, 0, 32, 32, 35, 35, 1, 0, 38, 39, 2, 0, 37, 37, 51, 51, 1, 0, 52, 57, 548, 0, 98, 1, 0, 0, 0, 2, 101, 1, 0, 0, 0, 4, 116, 1, 0, 0, 0, 6, 131, 1, 0, 0, 0, 8, 133, 1, 0, 0, 0, 10, 164, 1, 0, 0, 0, 12, 191, 1, 0, 0, 0, 14, 198, 1, 0, 0, 0, 16, 204, 1, 0, 0, 0, 18, 224, 1, 0, 0, 0, 20, 226, 1, 0, 0, 0, 22, 241, 1, 0, 0, 0, 24, 244, 1, 0, 0, 0, 26, 257, 1, 0, 0, 0, 28, 259, 1, 0, 0, 0, 30, 271, 1, 0, 0, 0, 32, 283, 1, 0, 0, 0, 34, 286, 1, 0, 0, 0, 36, 294, 1, 0, 0, 0, 38, 300, 1, 0, 0, 0, 40, 308, 1, 0, 0, 0, 42, 310, 1, 0, 0, 0, 44, 318, 1, 0, 0, 0, 46, 326, 1, 0, 0, 0, 48, 328, 1, 0, 0, 0, 50, 372, 1, 0, 0, 0, 52, 374, 1, 0, 0, 0, 54, 377, 1, 0, 0, 0, 56, 386, 1, 0, 0, 0, 58, 412, 1, 0, 0, 0, 60, 414, 1, 0, 0, 0, 62, 423, 1, 0, 0, 0, 64, 432, 1, 0, 0, 0, 66, 436, 1, 0, 0, 0, 68, 442, 1, 0, 0, 0, 70, 446, 1, 0, 0, 0, 72, 449, 1, 0, 0, 0, 74, 457, 1, 0, 0, 0, 76, 461, 1, 0, 0, 0, 78, 465, 1, 0, 0, 0, 80, 468, 1, 0, 0, 0, 82, 473, 1, 0, 0, 0, 84, 477, 1, 0, 0, 0, 86, 479, 1, 0, 0, 0, 88, 481, 1, 0, 0, 0, 90, 484, 1, 0, 0, 0, 92, 492, 1, 0, 0, 0, 94, 494, 1, 0, 0, 0, 96, 514, 1, 0, 0, 0, 98, 99, 3, 2, 1, 0, 99, 100, 5, 0, 0, 1, 100, 1, 1, 0, 0, 0, 101, 102, 6, 1, -1, 0, 102, 103, 3, 4, 2, 0, 103, 109, 1, 0, 0, 0, 104, 105, 10, 1, 0, 0, 105, 106, 5, 26, 0, 0, 106, 108, 3, 6, 3, 0, 107, 104, 1, 0, 0, 0, 108, 111, 1, 0, 0, 0, 109, 107, 1, 0, 0, 0, 109, 110, 1, 0, 0, 0, 110, 3, 1, 0, 0, 0, 111, 109, 1, 0, 0, 0, 112, 117, 3, 88, 44, 0, 113, 117, 3, 28, 14, 0, 114, 117, 3, 22, 11, 0, 115, 117, 3, 92, 46, 0, 116, 112, 1, 0, 0, 0, 116, 113, 1, 0, 0, 0, 116, 114, 1, 0, 0, 0, 116, 115, 1, 0, 0, 0, 117, 5, 1, 0, 0, 0, 118, 132, 3, 32, 16, 0, 119, 132, 3, 36, 18, 0, 120, 132, 3, 52, 26, 0, 121, 132, 3, 58, 29, 0, 122, 132, 3, 54, 27, 0, 123, 132, 3, 34, 17, 0, 124, 132, 3, 8, 4, 0, 125, 132, 3, 60, 30, 0, 126, 132, 3, 62, 31, 0, 127, 132, 3, 66, 33, 0, 128, 132, 3, 68, 34, 0, 129, 132, 3, 94, 47, 0, 130, 132, 3, 70, 35, 0, 131, 118, 1, 0, 0, 0, 131, 119, 1, 0, 0, 0, 131, 120, 1, 0, 0, 0, 131, 121, 1, 0, 0, 0, 131, 122, 1, 0, 0, 0, 131, 123, 1, 0, 0, 0, 131, 124, 1, 0, 0, 0, 131, 125, 1, 0, 0, 0, 131, 126, 1, 0, 0, 0, 131, 127, 1, 0, 0, 0, 131, 128, 1, 0, 0, 0, 131, 129, 1, 0, 0, 0, 131, 130, 1, 0, 0, 0, 132, 7, 1, 0, 0, 0, 133, 134, 5, 18, 0, 0, 134, 135, 3, 10, 5, 0, 135, 9, 1, 0, 0, 0, 136, 137, 6, 5, -1, 0, 137, 138, 5, 44, 0, 0, 138, 165, 3, 10, 5, 7, 139, 165, 3, 14, 7, 0, 140, 165, 3, 12, 6, 0, 141, 143, 3, 14, 7, 0, 142, 144, 5, 44, 0, 0, 143, 142, 1, 0, 0, 0, 143, 144, 1, 0, 0, 0, 144, 145, 1, 0, 0, 0, 145, 146, 5, 41, 0, 0, 146, 147, 5, 40, 0, 0, 147, 152, 3, 14, 7, 0, 148, 149, 5, 34, 0, 0, 149, 151, 3, 14, 7, 0, 150, 148, 1, 0, 0, 0, 151, 154, 1, 0, 0, 0, 152, 150, 1, 0, 0, 0, 152, 153, 1, 0, 0, 0, 153, 155, 1, 0, 0, 0, 154, 152, 1, 0, 0, 0, 155, 156, 5, 50, 0, 0, 156, 165, 1, 0, 0, 0, 157, 158, 3, 14, 7, 0, 158, 160, 5, 42, 0, 0, 159, 161, 5, 44, 0, 0, 160, 159, 1, 0, 0, 0, 160, 161, 1, 0, 0, 0, 161, 162, 1, 0, 0, 0, 162, 163, 5, 45, 0, 0, 163, 165, 1, 0, 0, 0, 164, 136, 1, 0, 0, 0, 164, 139, 1, 0, 0, 0, 164, 140, 1, 0, 0, 0, 164, 141, 1, 0, 0, 0, 164, 157, 1, 0, 0, 0, 165, 174, 1, 0, 0, 0, 166, 167, 10, 4, 0, 0, 167, 168, 5, 31, 0, 0, 168, 173, 3, 10, 5, 5, 169, 170, 10, 3, 0, 0, 170, 171, 5, 47, 0, 0, 171, 173, 3, 10, 5, 4, 172, 166, 1, 0, 0, 0, 172, 169, 1, 0, 0, 0, 173, 176, 1, 0, 0, 0, 174, 172, 1, 0, 0, 0, 174, 175, 1, 0, 0, 0, 175, 11, 1, 0, 0, 0, 176, 174, 1, 0, 0, 0, 177, 179, 3, 14, 7, 0, 178, 180, 5, 44, 0, 0, 179, 178, 1, 0, 0, 0, 179, 180, 1, 0, 0, 0, 180, 181, 1, 0, 0, 0, 181, 182, 5, 43, 0, 0, 182, 183, 3, 84, 42, 0, 183, 192, 1, 0, 0, 0, 184, 186, 3, 14, 7, 0, 185, 187, 5, 44, 0, 0, 186, 185, 1, 0, 0, 0, 186, 187, 1, 0, 0, 0, 187, 188, 1, 0, 0, 0, 188, 189, 5, 49, 0, 0, 189, 190, 3, 84, 42, 0, 190, 192, 1, 0, 0, 0, 191, 177, 1, 0, 0, 0, 191, 184, 1, 0, 0, 0, 192, 13, 1, 0, 0, 0, 193, 199, 3, 16, 8, 0, 194, 195, 3, 16, 8, 0, 195, 196, 3, 86, 43, 0, 196, 197, 3, 16, 8, 0, 197, 199, 1, 0, 0, 0, 198, 193, 1, 0, 0, 0, 198, 194, 1, 0, 0, 0, 199, 15, 1, 0, 0, 0, 200, 201, 6, 8, -1, 0, 201, 205, 3, 18, 9, 0, 202, 203, 7, 0, 0, 0, 203, 205, 3, 16, 8, 3, 204, 200, 1, 0, 0, 0, 204, 202, 1, 0, 0, 0, 205, 214, 1, 0, 0, 0, 206, 207, 10, 2, 0, 0, 207, 208, 7, 1, 0, 0, 208, 213, 3, 16, 8, 3, 209, 210, 10, 1, 0, 0, 210, 211, 7, 0, 0, 0, 211, 213, 3, 16, 8, 2, 212, 206, 1, 0, 0, 0, 212, 209, 1, 0, 0, 0, 213, 216, 1, 0, 0, 0, 214, 212, 1, 0, 0, 0, 214, 215, 1, 0, 0, 0, 215, 17, 1, 0, 0, 0, 216, 214, 1, 0, 0, 0, 217, 225, 3, 50, 25, 0, 218, 225, 3, 42, 21, 0, 219, 225, 3, 20, 10, 0, 220, 221, 5, 40, 0, 0, 221, 222, 3, 10, 5, 0, 222, 223, 5, 50, 0, 0, 223, 225, 1, 0, 0, 0, 224, 217, 1, 0, 0, 0, 224, 218, 1, 0, 0, 0, 224, 219, 1, 0, 0, 0, 224, 220, 1, 0, 0, 0, 225, 19, 1, 0, 0, 0, 226, 227, 3, 46, 23, 0, 227, 237, 5, 40, 0, 0, 228, 238, 5, 60, 0, 0, 229, 234, 3, 10, 5, 0, 230, 231, 5, 34, 0, 0, 231, 233, 3, 10, 5, 0, 232, 230, 1, 0, 0, 0, 233, 236, 1, 0, 0, 0, 234, 232, 1, 0, 0, 0, 234, 235, 1, 0, 0, 0, 235, 238, 1, 0, 0, 0, 236, 234, 1, 0, 0, 0, 237, 228, 1, 0, 0, 0, 237, 229, 1, 0, 0, 0, 237, 238, 1, 0, 0, 0, 238, 239, 1, 0, 0, 0, 239, 240, 5, 50, 0, 0, 240, 21, 1, 0, 0, 0, 241, 242, 5, 14, 0, 0, 242, 243, 3, 24, 12, 0, 243, 23, 1, 0, 0, 0, 244, 249, 3, 26, 13, 0, 245, 246, 5, 34, 0, 0, 246, 248, 3, 26, 13, 0, 247, 245, 1, 0, 0, 0, 248, 251, 1, 0, 0, 0, 249, 247, 1, 0, 0, 0, 249, 250, 1, 0, 0, 0, 250, 25, 1, 0, 0, 0, 251, 249, 1, 0, 0, 0, 252, 258, 3, 10, 5, 0, 253, 254, 3, 42, 21, 0, 254, 255, 5, 33, 0, 0, 255, 256, 3, 10, 5, 0, 256, 258, 1, 0, 0, 0, 257, 252, 1, 0, 0, 0, 257, 253, 1, 0, 0, 0, 258, 27, 1, 0, 0, 0, 259, 260, 5, 6, 0, 0, 260, 265, 3, 40, 20, 0, 261, 262, 5, 34, 0, 0, 262, 264, 3, 40, 20, 0, 263, 261, 1, 0, 0, 0, 264, 267, 1, 0, 0, 0, 265, 263, 1, 0, 0, 0, 265, 266, 1, 0, 0, 0, 266, 269, 1, 0, 0, 0, 267, 265, 1, 0, 0, 0, 268, 270, 3, 30, 15, 0, 269, 268, 1, 0, 0, 0, 269, 270, 1, 0, 0, 0, 270, 29, 1, 0, 0, 0, 271, 272, 5, 63, 0, 0, 272, 273, 5, 70, 0, 0, 273, 278, 3, 40, 20, 0, 274, 275, 5, 34, 0, 0, 275, 277, 3, 40, 20, 0, 276, 274, 1, 0, 0, 0, 277, 280, 1, 0, 0, 0, 278, 276, 1, 0, 0, 0, 278, 279, 1, 0, 0, 0, 279, 281, 1, 0, 0, 0, 280, 278, 1, 0, 0, 0, 281, 282, 5, 64, 0, 0, 282, 31, 1, 0, 0, 0, 283, 284, 5, 4, 0, 0, 284, 285, 3, 24, 12, 0, 285, 33, 1, 0, 0, 0, 286, 288, 5, 17, 0, 0, 287, 289, 3, 24, 12, 0, 288, 287, 1, 0, 0, 0, 288, 289, 1, 0, 0, 0, 289, 292, 1, 0, 0, 0, 290, 291, 5, 30, 0, 0, 291, 293, 3, 38, 19, 0, 292, 290, 1, 0, 0, 0, 292, 293, 1, 0, 0, 0, 293, 35, 1, 0, 0, 0, 294, 295, 5, 8, 0, 0, 295, 298, 3, 24, 12, 0, 296, 297, 5, 30, 0, 0, 297, 299, 3, 38, 19, 0, 298, 296, 1, 0, 0, 0, 298, 299, 1, 0, 0, 0, 299, 37, 1, 0, 0, 0, 300, 305, 3, 42, 21, 0, 301, 302, 5, 34, 0, 0, 302, 304, 3, 42, 21, 0, 303, 301, 1, 0, 0, 0, 304, 307, 1, 0, 0, 0, 305, 303, 1, 0, 0, 0, 305, 306, 1, 0, 0, 0, 306, 39, 1, 0, 0, 0, 307, 305, 1, 0, 0, 0, 308, 309, 7, 2, 0, 0, 309, 41, 1, 0, 0, 0, 310, 315, 3, 46, 23, 0, 311, 312, 5, 36, 0, 0, 312, 314, 3, 46, 23, 0, 313, 311, 1, 0, 0, 0, 314, 317, 1, 0, 0, 0, 315, 313, 1, 0, 0, 0, 315, 316, 1, 0, 0, 0, 316, 43, 1, 0, 0, 0, 317, 315, 1, 0, 0, 0, 318, 323, 3, 48, 24, 0, 319, 320, 5, 36, 0, 0, 320, 322, 3, 48, 24, 0, 321, 319, 1, 0, 0, 0, 322, 325, 1, 0, 0, 0, 323, 321, 1, 0, 0, 0, 323, 324, 1, 0, 0, 0, 324, 45, 1, 0, 0, 0, 325, 323, 1, 0, 0, 0, 326, 327, 7, 3, 0, 0, 327, 47, 1, 0, 0, 0, 328, 329, 7, 4, 0, 0, 329, 49, 1, 0, 0, 0, 330, 373, 5, 45, 0, 0, 331, 332, 3, 82, 41, 0, 332, 333, 5, 65, 0, 0, 333, 373, 1, 0, 0, 0, 334, 373, 3, 80, 40, 0, 335, 373, 3, 82, 41, 0, 336, 373, 3, 76, 38, 0, 337, 373, 5, 48, 0, 0, 338, 373, 3, 84, 42, 0, 339, 340, 5, 63, 0, 0, 340, 345, 3, 78, 39, 0, 341, 342, 5, 34, 0, 0, 342, 344, 3, 78, 39, 0, 343, 341, 1, 0, 0, 0, 344, 347, 1, 0, 0, 0, 345, 343, 1, 0, 0, 0, 345, 346, 1, 0, 0, 0, 346, 348, 1, 0, 0, 0, 347, 345, 1, 0, 0, 0, 348, 349, 5, 64, 0, 0, 349, 373, 1, 0, 0, 0, 350, 351, 5, 63, 0, 0, 351, 356, 3, 76, 38, 0, 352, 353, 5, 34, 0, 0, 353, 355, 3, 76, 38, 0, 354, 352, 1, 0, 0, 0, 355, 358, 1, 0, 0, 0, 356, 354, 1, 0, 0, 0, 356, 357, 1, 0, 0, 0, 357, 359, 1, 0, 0, 0, 358, 356, 1, 0, 0, 0, 359, 360, 5, 64, 0, 0, 360, 373, 1, 0, 0, 0, 361, 362, 5, 63, 0, 0, 362, 367, 3, 84, 42, 0, 363, 364, 5, 34, 0, 0, 364, 366, 3, 84, 42, 0, 365, 363, 1, 0, 0, 0, 366, 369, 1, 0, 0, 0, 367, 365, 1, 0, 0, 0, 367, 368, 1, 0, 0, 0, 368, 370, 1, 0, 0, 0, 369, 367, 1, 0, 0, 0, 370, 371, 5, 64, 0, 0, 371, 373, 1, 0, 0, 0, 372, 330, 1, 0, 0, 0, 372, 331, 1, 0, 0, 0, 372, 334, 1, 0, 0, 0, 372, 335, 1, 0, 0, 0, 372, 336, 1, 0, 0, 0, 372, 337, 1, 0, 0, 0, 372, 338, 1, 0, 0, 0, 372, 339, 1, 0, 0, 0, 372, 350, 1, 0, 0, 0, 372, 361, 1, 0, 0, 0, 373, 51, 1, 0, 0, 0, 374, 375, 5, 10, 0, 0, 375, 376, 5, 28, 0, 0, 376, 53, 1, 0, 0, 0, 377, 378, 5, 16, 0, 0, 378, 383, 3, 56, 28, 0, 379, 380, 5, 34, 0, 0, 380, 382, 3, 56, 28, 0, 381, 379, 1, 0, 0, 0, 382, 385, 1, 0, 0, 0, 383, 381, 1, 0, 0, 0, 383, 384, 1, 0, 0, 0, 384, 55, 1, 0, 0, 0, 385, 383, 1, 0, 0, 0, 386, 388, 3, 10, 5, 0, 387, 389, 7, 5, 0, 0, 388, 387, 1, 0, 0, 0, 388, 389, 1, 0, 0, 0, 389, 392, 1, 0, 0, 0, 390, 391, 5, 46, 0, 0, 391, 393, 7, 6, 0, 0, 392, 390, 1, 0, 0, 0, 392, 393, 1, 0, 0, 0, 393, 57, 1, 0, 0, 0, 394, 395, 5, 9, 0, 0, 395, 400, 3, 44, 22, 0, 396, 397, 5, 34, 0, 0, 397, 399, 3, 44, 22, 0, 398, 396, 1, 0, 0, 0, 399, 402, 1, 0, 0, 0, 400, 398, 1, 0, 0, 0, 400, 401, 1, 0, 0, 0, 401, 413, 1, 0, 0, 0, 402, 400, 1, 0, 0, 0, 403, 404, 5, 12, 0, 0, 404, 409, 3, 44, 22, 0, 405, 406, 5, 34, 0, 0, 406, 408, 3, 44, 22, 0, 407, 405, 1, 0, 0, 0, 408, 411, 1, 0, 0, 0, 409, 407, 1, 0, 0, 0, 409, 410, 1, 0, 0, 0, 410, 413, 1, 0, 0, 0, 411, 409, 1, 0, 0, 0, 412, 394, 1, 0, 0, 0, 412, 403, 1, 0, 0, 0, 413, 59, 1, 0, 0, 0, 414, 415, 5, 2, 0, 0, 415, 420, 3, 44, 22, 0, 416, 417, 5, 34, 0, 0, 417, 419, 3, 44, 22, 0, 418, 416, 1, 0, 0, 0, 419, 422, 1, 0, 0, 0, 420, 418, 1, 0, 0, 0, 420, 421, 1, 0, 0, 0, 421, 61, 1, 0, 0, 0, 422, 420, 1, 0, 0, 0, 423, 424, 5, 13, 0, 0, 424, 429, 3, 64, 32, 0, 425, 426, 5, 34, 0, 0, 426, 428, 3, 64, 32, 0, 427, 425, 1, 0, 0, 0, 428, 431, 1, 0, 0, 0, 429, 427, 1, 0, 0, 0, 429, 430, 1, 0, 0, 0, 430, 63, 1, 0, 0, 0, 431, 429, 1, 0, 0, 0, 432, 433, 3, 44, 22, 0, 433, 434, 5, 79, 0, 0, 434, 435, 3, 44, 22, 0, 435, 65, 1, 0, 0, 0, 436, 437, 5, 1, 0, 0, 437, 438, 3, 18, 9, 0, 438, 440, 3, 84, 42, 0, 439, 441, 3, 72, 36, 0, 440, 439, 1, 0, 0, 0, 440, 441, 1, 0, 0, 0, 441, 67, 1, 0, 0, 0, 442, 443, 5, 7, 0, 0, 443, 444, 3, 18, 9, 0, 444, 445, 3, 84, 42, 0, 445, 69, 1, 0, 0, 0, 446, 447, 5, 11, 0, 0, 447, 448, 3, 42, 21, 0, 448, 71, 1, 0, 0, 0, 449, 454, 3, 74, 37, 0, 450, 451, 5, 34, 0, 0, 451, 453, 3, 74, 37, 0, 452, 450, 1, 0, 0, 0, 453, 456, 1, 0, 0, 0, 454, 452, 1, 0, 0, 0, 454, 455, 1, 0, 0, 0, 455, 73, 1, 0, 0, 0, 456, 454, 1, 0, 0, 0, 457, 458, 3, 46, 23, 0, 458, 459, 5, 33, 0, 0, 459, 460, 3, 50, 25, 0, 460, 75, 1, 0, 0, 0, 461, 462, 7, 7, 0, 0, 462, 77, 1, 0, 0, 0, 463, 466, 3, 80, 40, 0, 464, 466, 3, 82, 41, 0, 465, 463, 1, 0, 0, 0, 465, 464, 1, 0, 0, 0, 466, 79, 1, 0, 0, 0, 467, 469, 7, 0, 0, 0, 468, 467, 1, 0, 0, 0, 468, 469, 1, 0, 0, 0, 469, 470, 1, 0, 0, 0, 470, 471, 5, 29, 0, 0, 471, 81, 1, 0, 0, 0, 472, 474, 7, 0, 0, 0, 473, 472, 1, 0, 0, 0, 473, 474, 1, 0, 0, 0, 474, 475, 1, 0, 0, 0, 475, 476, 5, 28, 0, 0, 476, 83, 1, 0, 0, 0, 477, 478, 5, 27, 0, 0, 478, 85, 1, 0, 0, 0, 479, 480, 7, 8, 0, 0, 480, 87, 1, 0, 0, 0, 481, 482, 5, 5, 0, 0, 482, 483, 3, 90, 45, 0, 483, 89, 1, 0, 0, 0, 484, 485, 5, 63, 0, 0, 485, 486, 3, 2, 1, 0, 486, 487, 5, 64, 0, 0, 487, 91, 1, 0, 0, 0, 488, 489, 5, 15, 0, 0, 489, 493, 5, 94, 0, 0, 490, 491, 5, 15, 0, 0, 491, 493, 5, 95, 0, 0, 492, 488, 1, 0, 0, 0, 492, 490, 1, 0, 0, 0, 493, 93, 1, 0, 0, 0, 494, 495, 5, 3, 0, 0, 495, 498, 3, 40, 20, 0, 496, 497, 5, 83, 0, 0, 497, 499, 3, 44, 22, 0, 498, 496, 1, 0, 0, 0, 498, 499, 1, 0, 0, 0, 499, 509, 1, 0, 0, 0, 500, 501, 5, 84, 0, 0, 501, 506, 3, 96, 48, 0, 502, 503, 5, 34, 0, 0, 503, 505, 3, 96, 48, 0, 504, 502, 1, 0, 0, 0, 505, 508, 1, 0, 0, 0, 506, 504, 1, 0, 0, 0, 506, 507, 1, 0, 0, 0, 507, 510, 1, 0, 0, 0, 508, 506, 1, 0, 0, 0, 509, 500, 1, 0, 0, 0, 509, 510, 1, 0, 0, 0, 510, 95, 1, 0, 0, 0, 511, 512, 3, 44, 22, 0, 512, 513, 5, 33, 0, 0, 513, 515, 1, 0, 0, 0, 514, 511, 1, 0, 0, 0, 514, 515, 1, 0, 0, 0, 515, 516, 1, 0, 0, 0, 516, 517, 3, 44, 22, 0, 517, 97, 1, 0, 0, 0, 52, 109, 116, 131, 143, 152, 160, 164, 172, 174, 179, 186, 191, 198, 204, 212, 214, 224, 234, 237, 249, 257, 265, 269, 278, 288, 292, 298, 305, 315, 323, 345, 356, 367, 372, 383, 388, 392, 400, 409, 412, 420, 429, 440, 454, 465, 468, 473, 492, 498, 506, 509, 514] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java index d136c346927e6..54ec466de9623 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java @@ -24,12 +24,18 @@ public class EsqlBaseParser extends Parser { PIPE=26, STRING=27, INTEGER_LITERAL=28, DECIMAL_LITERAL=29, BY=30, AND=31, ASC=32, ASSIGN=33, COMMA=34, DESC=35, DOT=36, FALSE=37, FIRST=38, LAST=39, LP=40, IN=41, IS=42, LIKE=43, NOT=44, NULL=45, NULLS=46, OR=47, PARAM=48, - RLIKE=49, RP=50, TRUE=51, INFO=52, FUNCTIONS=53, EQ=54, NEQ=55, LT=56, - LTE=57, GT=58, GTE=59, PLUS=60, MINUS=61, ASTERISK=62, SLASH=63, PERCENT=64, - OPENING_BRACKET=65, CLOSING_BRACKET=66, UNQUOTED_IDENTIFIER=67, QUOTED_IDENTIFIER=68, - EXPR_LINE_COMMENT=69, EXPR_MULTILINE_COMMENT=70, EXPR_WS=71, AS=72, METADATA=73, - ON=74, WITH=75, SRC_UNQUOTED_IDENTIFIER=76, SRC_QUOTED_IDENTIFIER=77, - SRC_LINE_COMMENT=78, SRC_MULTILINE_COMMENT=79, SRC_WS=80, EXPLAIN_PIPE=81; + RLIKE=49, RP=50, TRUE=51, EQ=52, NEQ=53, LT=54, LTE=55, GT=56, GTE=57, + PLUS=58, MINUS=59, ASTERISK=60, SLASH=61, PERCENT=62, OPENING_BRACKET=63, + CLOSING_BRACKET=64, UNQUOTED_IDENTIFIER=65, QUOTED_IDENTIFIER=66, EXPR_LINE_COMMENT=67, + EXPR_MULTILINE_COMMENT=68, EXPR_WS=69, METADATA=70, FROM_UNQUOTED_IDENTIFIER=71, + FROM_LINE_COMMENT=72, FROM_MULTILINE_COMMENT=73, FROM_WS=74, PROJECT_UNQUOTED_IDENTIFIER=75, + PROJECT_LINE_COMMENT=76, PROJECT_MULTILINE_COMMENT=77, PROJECT_WS=78, + AS=79, RENAME_LINE_COMMENT=80, RENAME_MULTILINE_COMMENT=81, RENAME_WS=82, + ON=83, WITH=84, ENRICH_LINE_COMMENT=85, ENRICH_MULTILINE_COMMENT=86, ENRICH_WS=87, + ENRICH_FIELD_LINE_COMMENT=88, ENRICH_FIELD_MULTILINE_COMMENT=89, ENRICH_FIELD_WS=90, + MVEXPAND_LINE_COMMENT=91, MVEXPAND_MULTILINE_COMMENT=92, MVEXPAND_WS=93, + INFO=94, FUNCTIONS=95, SHOW_LINE_COMMENT=96, SHOW_MULTILINE_COMMENT=97, + SHOW_WS=98; public static final int RULE_singleStatement = 0, RULE_query = 1, RULE_sourceCommand = 2, RULE_processingCommand = 3, RULE_whereCommand = 4, RULE_booleanExpression = 5, RULE_regexBooleanExpression = 6, @@ -37,26 +43,28 @@ public class EsqlBaseParser extends Parser { RULE_functionExpression = 10, RULE_rowCommand = 11, RULE_fields = 12, RULE_field = 13, RULE_fromCommand = 14, RULE_metadata = 15, RULE_evalCommand = 16, RULE_statsCommand = 17, RULE_inlinestatsCommand = 18, RULE_grouping = 19, - RULE_sourceIdentifier = 20, RULE_qualifiedName = 21, RULE_identifier = 22, - RULE_constant = 23, RULE_limitCommand = 24, RULE_sortCommand = 25, RULE_orderExpression = 26, - RULE_keepCommand = 27, RULE_dropCommand = 28, RULE_renameCommand = 29, - RULE_renameClause = 30, RULE_dissectCommand = 31, RULE_grokCommand = 32, - RULE_mvExpandCommand = 33, RULE_commandOptions = 34, RULE_commandOption = 35, - RULE_booleanValue = 36, RULE_numericValue = 37, RULE_decimalValue = 38, - RULE_integerValue = 39, RULE_string = 40, RULE_comparisonOperator = 41, - RULE_explainCommand = 42, RULE_subqueryExpression = 43, RULE_showCommand = 44, - RULE_enrichCommand = 45, RULE_enrichWithClause = 46; + RULE_fromIdentifier = 20, RULE_qualifiedName = 21, RULE_qualifiedNamePattern = 22, + RULE_identifier = 23, RULE_identifierPattern = 24, RULE_constant = 25, + RULE_limitCommand = 26, RULE_sortCommand = 27, RULE_orderExpression = 28, + RULE_keepCommand = 29, RULE_dropCommand = 30, RULE_renameCommand = 31, + RULE_renameClause = 32, RULE_dissectCommand = 33, RULE_grokCommand = 34, + RULE_mvExpandCommand = 35, RULE_commandOptions = 36, RULE_commandOption = 37, + RULE_booleanValue = 38, RULE_numericValue = 39, RULE_decimalValue = 40, + RULE_integerValue = 41, RULE_string = 42, RULE_comparisonOperator = 43, + RULE_explainCommand = 44, RULE_subqueryExpression = 45, RULE_showCommand = 46, + RULE_enrichCommand = 47, RULE_enrichWithClause = 48; private static String[] makeRuleNames() { return new String[] { "singleStatement", "query", "sourceCommand", "processingCommand", "whereCommand", "booleanExpression", "regexBooleanExpression", "valueExpression", "operatorExpression", "primaryExpression", "functionExpression", "rowCommand", "fields", "field", "fromCommand", "metadata", "evalCommand", "statsCommand", "inlinestatsCommand", - "grouping", "sourceIdentifier", "qualifiedName", "identifier", "constant", - "limitCommand", "sortCommand", "orderExpression", "keepCommand", "dropCommand", - "renameCommand", "renameClause", "dissectCommand", "grokCommand", "mvExpandCommand", - "commandOptions", "commandOption", "booleanValue", "numericValue", "decimalValue", - "integerValue", "string", "comparisonOperator", "explainCommand", "subqueryExpression", + "grouping", "fromIdentifier", "qualifiedName", "qualifiedNamePattern", + "identifier", "identifierPattern", "constant", "limitCommand", "sortCommand", + "orderExpression", "keepCommand", "dropCommand", "renameCommand", "renameClause", + "dissectCommand", "grokCommand", "mvExpandCommand", "commandOptions", + "commandOption", "booleanValue", "numericValue", "decimalValue", "integerValue", + "string", "comparisonOperator", "explainCommand", "subqueryExpression", "showCommand", "enrichCommand", "enrichWithClause" }; } @@ -67,12 +75,14 @@ private static String[] makeLiteralNames() { null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", "'grok'", "'inlinestats'", "'keep'", "'limit'", "'mv_expand'", "'project'", "'rename'", "'row'", "'show'", "'sort'", "'stats'", "'where'", null, - null, null, null, null, null, null, null, null, null, null, "'by'", "'and'", - "'asc'", null, null, "'desc'", "'.'", "'false'", "'first'", "'last'", - "'('", "'in'", "'is'", "'like'", "'not'", "'null'", "'nulls'", "'or'", - "'?'", "'rlike'", "')'", "'true'", "'info'", "'functions'", "'=='", "'!='", - "'<'", "'<='", "'>'", "'>='", "'+'", "'-'", "'*'", "'/'", "'%'", null, - "']'", null, null, null, null, null, "'as'", "'metadata'", "'on'", "'with'" + null, null, null, null, null, null, "'|'", null, null, null, "'by'", + "'and'", "'asc'", "'='", "','", "'desc'", "'.'", "'false'", "'first'", + "'last'", "'('", "'in'", "'is'", "'like'", "'not'", "'null'", "'nulls'", + "'or'", "'?'", "'rlike'", "')'", "'true'", "'=='", "'!='", "'<'", "'<='", + "'>'", "'>='", "'+'", "'-'", "'*'", "'/'", "'%'", null, "']'", null, + null, null, null, null, "'metadata'", null, null, null, null, null, null, + null, null, "'as'", null, null, null, "'on'", "'with'", null, null, null, + null, null, null, null, null, null, "'info'", "'functions'" }; } private static final String[] _LITERAL_NAMES = makeLiteralNames(); @@ -85,12 +95,17 @@ private static String[] makeSymbolicNames() { "PIPE", "STRING", "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "COMMA", "DESC", "DOT", "FALSE", "FIRST", "LAST", "LP", "IN", "IS", "LIKE", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", - "TRUE", "INFO", "FUNCTIONS", "EQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", - "MINUS", "ASTERISK", "SLASH", "PERCENT", "OPENING_BRACKET", "CLOSING_BRACKET", - "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", - "EXPR_WS", "AS", "METADATA", "ON", "WITH", "SRC_UNQUOTED_IDENTIFIER", - "SRC_QUOTED_IDENTIFIER", "SRC_LINE_COMMENT", "SRC_MULTILINE_COMMENT", - "SRC_WS", "EXPLAIN_PIPE" + "TRUE", "EQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", + "SLASH", "PERCENT", "OPENING_BRACKET", "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", + "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", + "METADATA", "FROM_UNQUOTED_IDENTIFIER", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", + "FROM_WS", "PROJECT_UNQUOTED_IDENTIFIER", "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", + "PROJECT_WS", "AS", "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", + "RENAME_WS", "ON", "WITH", "ENRICH_LINE_COMMENT", "ENRICH_MULTILINE_COMMENT", + "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", "ENRICH_FIELD_MULTILINE_COMMENT", + "ENRICH_FIELD_WS", "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", + "MVEXPAND_WS", "INFO", "FUNCTIONS", "SHOW_LINE_COMMENT", "SHOW_MULTILINE_COMMENT", + "SHOW_WS" }; } private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames(); @@ -177,9 +192,9 @@ public final SingleStatementContext singleStatement() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(94); + setState(98); query(0); - setState(95); + setState(99); match(EOF); } } @@ -275,11 +290,11 @@ private QueryContext query(int _p) throws RecognitionException { _ctx = _localctx; _prevctx = _localctx; - setState(98); + setState(102); sourceCommand(); } _ctx.stop = _input.LT(-1); - setState(105); + setState(109); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -290,16 +305,16 @@ private QueryContext query(int _p) throws RecognitionException { { _localctx = new CompositeQueryContext(new QueryContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_query); - setState(100); + setState(104); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(101); + setState(105); match(PIPE); - setState(102); + setState(106); processingCommand(); } } } - setState(107); + setState(111); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); } @@ -354,34 +369,34 @@ public final SourceCommandContext sourceCommand() throws RecognitionException { SourceCommandContext _localctx = new SourceCommandContext(_ctx, getState()); enterRule(_localctx, 4, RULE_sourceCommand); try { - setState(112); + setState(116); _errHandler.sync(this); switch (_input.LA(1)) { case EXPLAIN: enterOuterAlt(_localctx, 1); { - setState(108); + setState(112); explainCommand(); } break; case FROM: enterOuterAlt(_localctx, 2); { - setState(109); + setState(113); fromCommand(); } break; case ROW: enterOuterAlt(_localctx, 3); { - setState(110); + setState(114); rowCommand(); } break; case SHOW: enterOuterAlt(_localctx, 4); { - setState(111); + setState(115); showCommand(); } break; @@ -465,27 +480,27 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce ProcessingCommandContext _localctx = new ProcessingCommandContext(_ctx, getState()); enterRule(_localctx, 6, RULE_processingCommand); try { - setState(127); + setState(131); _errHandler.sync(this); switch (_input.LA(1)) { case EVAL: enterOuterAlt(_localctx, 1); { - setState(114); + setState(118); evalCommand(); } break; case INLINESTATS: enterOuterAlt(_localctx, 2); { - setState(115); + setState(119); inlinestatsCommand(); } break; case LIMIT: enterOuterAlt(_localctx, 3); { - setState(116); + setState(120); limitCommand(); } break; @@ -493,70 +508,70 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce case PROJECT: enterOuterAlt(_localctx, 4); { - setState(117); + setState(121); keepCommand(); } break; case SORT: enterOuterAlt(_localctx, 5); { - setState(118); + setState(122); sortCommand(); } break; case STATS: enterOuterAlt(_localctx, 6); { - setState(119); + setState(123); statsCommand(); } break; case WHERE: enterOuterAlt(_localctx, 7); { - setState(120); + setState(124); whereCommand(); } break; case DROP: enterOuterAlt(_localctx, 8); { - setState(121); + setState(125); dropCommand(); } break; case RENAME: enterOuterAlt(_localctx, 9); { - setState(122); + setState(126); renameCommand(); } break; case DISSECT: enterOuterAlt(_localctx, 10); { - setState(123); + setState(127); dissectCommand(); } break; case GROK: enterOuterAlt(_localctx, 11); { - setState(124); + setState(128); grokCommand(); } break; case ENRICH: enterOuterAlt(_localctx, 12); { - setState(125); + setState(129); enrichCommand(); } break; case MV_EXPAND: enterOuterAlt(_localctx, 13); { - setState(126); + setState(130); mvExpandCommand(); } break; @@ -607,9 +622,9 @@ public final WhereCommandContext whereCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(129); + setState(133); match(WHERE); - setState(130); + setState(134); booleanExpression(0); } } @@ -804,7 +819,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc int _alt; enterOuterAlt(_localctx, 1); { - setState(160); + setState(164); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) { case 1: @@ -813,9 +828,9 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _ctx = _localctx; _prevctx = _localctx; - setState(133); + setState(137); match(NOT); - setState(134); + setState(138); booleanExpression(7); } break; @@ -824,7 +839,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new BooleanDefaultContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(135); + setState(139); valueExpression(); } break; @@ -833,7 +848,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new RegexExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(136); + setState(140); regexBooleanExpression(); } break; @@ -842,41 +857,41 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalInContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(137); + setState(141); valueExpression(); - setState(139); + setState(143); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(138); + setState(142); match(NOT); } } - setState(141); + setState(145); match(IN); - setState(142); + setState(146); match(LP); - setState(143); + setState(147); valueExpression(); - setState(148); + setState(152); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(144); + setState(148); match(COMMA); - setState(145); + setState(149); valueExpression(); } } - setState(150); + setState(154); _errHandler.sync(this); _la = _input.LA(1); } - setState(151); + setState(155); match(RP); } break; @@ -885,27 +900,27 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new IsNullContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(153); + setState(157); valueExpression(); - setState(154); + setState(158); match(IS); - setState(156); + setState(160); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(155); + setState(159); match(NOT); } } - setState(158); + setState(162); match(NULL); } break; } _ctx.stop = _input.LT(-1); - setState(170); + setState(174); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -913,7 +928,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(168); + setState(172); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) { case 1: @@ -921,11 +936,11 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(162); + setState(166); if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)"); - setState(163); + setState(167); ((LogicalBinaryContext)_localctx).operator = match(AND); - setState(164); + setState(168); ((LogicalBinaryContext)_localctx).right = booleanExpression(5); } break; @@ -934,18 +949,18 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(165); + setState(169); if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)"); - setState(166); + setState(170); ((LogicalBinaryContext)_localctx).operator = match(OR); - setState(167); + setState(171); ((LogicalBinaryContext)_localctx).right = booleanExpression(4); } break; } } } - setState(172); + setState(176); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); } @@ -1000,48 +1015,48 @@ public final RegexBooleanExpressionContext regexBooleanExpression() throws Recog enterRule(_localctx, 12, RULE_regexBooleanExpression); int _la; try { - setState(187); + setState(191); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(173); + setState(177); valueExpression(); - setState(175); + setState(179); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(174); + setState(178); match(NOT); } } - setState(177); + setState(181); ((RegexBooleanExpressionContext)_localctx).kind = match(LIKE); - setState(178); + setState(182); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(180); + setState(184); valueExpression(); - setState(182); + setState(186); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(181); + setState(185); match(NOT); } } - setState(184); + setState(188); ((RegexBooleanExpressionContext)_localctx).kind = match(RLIKE); - setState(185); + setState(189); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; @@ -1127,14 +1142,14 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio ValueExpressionContext _localctx = new ValueExpressionContext(_ctx, getState()); enterRule(_localctx, 14, RULE_valueExpression); try { - setState(194); + setState(198); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) { case 1: _localctx = new ValueExpressionDefaultContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(189); + setState(193); operatorExpression(0); } break; @@ -1142,11 +1157,11 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio _localctx = new ComparisonContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(190); + setState(194); ((ComparisonContext)_localctx).left = operatorExpression(0); - setState(191); + setState(195); comparisonOperator(); - setState(192); + setState(196); ((ComparisonContext)_localctx).right = operatorExpression(0); } break; @@ -1271,7 +1286,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE int _alt; enterOuterAlt(_localctx, 1); { - setState(200); + setState(204); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) { case 1: @@ -1280,7 +1295,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _ctx = _localctx; _prevctx = _localctx; - setState(197); + setState(201); primaryExpression(); } break; @@ -1289,7 +1304,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticUnaryContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(198); + setState(202); ((ArithmeticUnaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1300,13 +1315,13 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(199); + setState(203); operatorExpression(3); } break; } _ctx.stop = _input.LT(-1); - setState(210); + setState(214); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,15,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1314,7 +1329,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(208); + setState(212); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) { case 1: @@ -1322,12 +1337,12 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(202); + setState(206); if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); - setState(203); + setState(207); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); - if ( !((((_la - 62)) & ~0x3f) == 0 && ((1L << (_la - 62)) & 7L) != 0) ) { + if ( !(((_la) & ~0x3f) == 0 && ((1L << _la) & 8070450532247928832L) != 0) ) { ((ArithmeticBinaryContext)_localctx).operator = (Token)_errHandler.recoverInline(this); } else { @@ -1335,7 +1350,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(204); + setState(208); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(3); } break; @@ -1344,9 +1359,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(205); + setState(209); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(206); + setState(210); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1357,14 +1372,14 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(207); + setState(211); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(2); } break; } } } - setState(212); + setState(216); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,15,_ctx); } @@ -1486,14 +1501,14 @@ public final PrimaryExpressionContext primaryExpression() throws RecognitionExce PrimaryExpressionContext _localctx = new PrimaryExpressionContext(_ctx, getState()); enterRule(_localctx, 18, RULE_primaryExpression); try { - setState(220); + setState(224); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) { case 1: _localctx = new ConstantDefaultContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(213); + setState(217); constant(); } break; @@ -1501,7 +1516,7 @@ public final PrimaryExpressionContext primaryExpression() throws RecognitionExce _localctx = new DereferenceContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(214); + setState(218); qualifiedName(); } break; @@ -1509,7 +1524,7 @@ public final PrimaryExpressionContext primaryExpression() throws RecognitionExce _localctx = new FunctionContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(215); + setState(219); functionExpression(); } break; @@ -1517,11 +1532,11 @@ public final PrimaryExpressionContext primaryExpression() throws RecognitionExce _localctx = new ParenthesizedExpressionContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(216); + setState(220); match(LP); - setState(217); + setState(221); booleanExpression(0); - setState(218); + setState(222); match(RP); } break; @@ -1583,16 +1598,16 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(222); + setState(226); identifier(); - setState(223); + setState(227); match(LP); - setState(233); + setState(237); _errHandler.sync(this); switch (_input.LA(1)) { case ASTERISK: { - setState(224); + setState(228); match(ASTERISK); } break; @@ -1612,21 +1627,21 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx case QUOTED_IDENTIFIER: { { - setState(225); + setState(229); booleanExpression(0); - setState(230); + setState(234); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(226); + setState(230); match(COMMA); - setState(227); + setState(231); booleanExpression(0); } } - setState(232); + setState(236); _errHandler.sync(this); _la = _input.LA(1); } @@ -1638,7 +1653,7 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx default: break; } - setState(235); + setState(239); match(RP); } } @@ -1685,9 +1700,9 @@ public final RowCommandContext rowCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(237); + setState(241); match(ROW); - setState(238); + setState(242); fields(); } } @@ -1741,23 +1756,23 @@ public final FieldsContext fields() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(240); + setState(244); field(); - setState(245); + setState(249); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,19,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(241); + setState(245); match(COMMA); - setState(242); + setState(246); field(); } } } - setState(247); + setState(251); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,19,_ctx); } @@ -1807,24 +1822,24 @@ public final FieldContext field() throws RecognitionException { FieldContext _localctx = new FieldContext(_ctx, getState()); enterRule(_localctx, 26, RULE_field); try { - setState(253); + setState(257); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(248); + setState(252); booleanExpression(0); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(249); + setState(253); qualifiedName(); - setState(250); + setState(254); match(ASSIGN); - setState(251); + setState(255); booleanExpression(0); } break; @@ -1844,11 +1859,11 @@ public final FieldContext field() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class FromCommandContext extends ParserRuleContext { public TerminalNode FROM() { return getToken(EsqlBaseParser.FROM, 0); } - public List sourceIdentifier() { - return getRuleContexts(SourceIdentifierContext.class); + public List fromIdentifier() { + return getRuleContexts(FromIdentifierContext.class); } - public SourceIdentifierContext sourceIdentifier(int i) { - return getRuleContext(SourceIdentifierContext.class,i); + public FromIdentifierContext fromIdentifier(int i) { + return getRuleContext(FromIdentifierContext.class,i); } public List COMMA() { return getTokens(EsqlBaseParser.COMMA); } public TerminalNode COMMA(int i) { @@ -1884,34 +1899,34 @@ public final FromCommandContext fromCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(255); + setState(259); match(FROM); - setState(256); - sourceIdentifier(); - setState(261); + setState(260); + fromIdentifier(); + setState(265); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,21,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(257); + setState(261); match(COMMA); - setState(258); - sourceIdentifier(); + setState(262); + fromIdentifier(); } } } - setState(263); + setState(267); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,21,_ctx); } - setState(265); + setState(269); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) { case 1: { - setState(264); + setState(268); metadata(); } break; @@ -1933,11 +1948,11 @@ public final FromCommandContext fromCommand() throws RecognitionException { public static class MetadataContext extends ParserRuleContext { public TerminalNode OPENING_BRACKET() { return getToken(EsqlBaseParser.OPENING_BRACKET, 0); } public TerminalNode METADATA() { return getToken(EsqlBaseParser.METADATA, 0); } - public List sourceIdentifier() { - return getRuleContexts(SourceIdentifierContext.class); + public List fromIdentifier() { + return getRuleContexts(FromIdentifierContext.class); } - public SourceIdentifierContext sourceIdentifier(int i) { - return getRuleContext(SourceIdentifierContext.class,i); + public FromIdentifierContext fromIdentifier(int i) { + return getRuleContext(FromIdentifierContext.class,i); } public TerminalNode CLOSING_BRACKET() { return getToken(EsqlBaseParser.CLOSING_BRACKET, 0); } public List COMMA() { return getTokens(EsqlBaseParser.COMMA); } @@ -1971,29 +1986,29 @@ public final MetadataContext metadata() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(267); + setState(271); match(OPENING_BRACKET); - setState(268); + setState(272); match(METADATA); - setState(269); - sourceIdentifier(); - setState(274); + setState(273); + fromIdentifier(); + setState(278); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(270); + setState(274); match(COMMA); - setState(271); - sourceIdentifier(); + setState(275); + fromIdentifier(); } } - setState(276); + setState(280); _errHandler.sync(this); _la = _input.LA(1); } - setState(277); + setState(281); match(CLOSING_BRACKET); } } @@ -2040,9 +2055,9 @@ public final EvalCommandContext evalCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(279); + setState(283); match(EVAL); - setState(280); + setState(284); fields(); } } @@ -2093,26 +2108,26 @@ public final StatsCommandContext statsCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(282); + setState(286); match(STATS); - setState(284); + setState(288); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) { case 1: { - setState(283); + setState(287); fields(); } break; } - setState(288); + setState(292); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) { case 1: { - setState(286); + setState(290); match(BY); - setState(287); + setState(291); grouping(); } break; @@ -2166,18 +2181,18 @@ public final InlinestatsCommandContext inlinestatsCommand() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(290); + setState(294); match(INLINESTATS); - setState(291); + setState(295); fields(); - setState(294); + setState(298); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) { case 1: { - setState(292); + setState(296); match(BY); - setState(293); + setState(297); grouping(); } break; @@ -2234,23 +2249,23 @@ public final GroupingContext grouping() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(296); + setState(300); qualifiedName(); - setState(301); + setState(305); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,27,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(297); + setState(301); match(COMMA); - setState(298); + setState(302); qualifiedName(); } } } - setState(303); + setState(307); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,27,_ctx); } @@ -2268,39 +2283,39 @@ public final GroupingContext grouping() throws RecognitionException { } @SuppressWarnings("CheckReturnValue") - public static class SourceIdentifierContext extends ParserRuleContext { - public TerminalNode SRC_UNQUOTED_IDENTIFIER() { return getToken(EsqlBaseParser.SRC_UNQUOTED_IDENTIFIER, 0); } - public TerminalNode SRC_QUOTED_IDENTIFIER() { return getToken(EsqlBaseParser.SRC_QUOTED_IDENTIFIER, 0); } + public static class FromIdentifierContext extends ParserRuleContext { + public TerminalNode FROM_UNQUOTED_IDENTIFIER() { return getToken(EsqlBaseParser.FROM_UNQUOTED_IDENTIFIER, 0); } + public TerminalNode QUOTED_IDENTIFIER() { return getToken(EsqlBaseParser.QUOTED_IDENTIFIER, 0); } @SuppressWarnings("this-escape") - public SourceIdentifierContext(ParserRuleContext parent, int invokingState) { + public FromIdentifierContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); } - @Override public int getRuleIndex() { return RULE_sourceIdentifier; } + @Override public int getRuleIndex() { return RULE_fromIdentifier; } @Override public void enterRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterSourceIdentifier(this); + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterFromIdentifier(this); } @Override public void exitRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitSourceIdentifier(this); + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitFromIdentifier(this); } @Override public T accept(ParseTreeVisitor visitor) { - if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitSourceIdentifier(this); + if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitFromIdentifier(this); else return visitor.visitChildren(this); } } - public final SourceIdentifierContext sourceIdentifier() throws RecognitionException { - SourceIdentifierContext _localctx = new SourceIdentifierContext(_ctx, getState()); - enterRule(_localctx, 40, RULE_sourceIdentifier); + public final FromIdentifierContext fromIdentifier() throws RecognitionException { + FromIdentifierContext _localctx = new FromIdentifierContext(_ctx, getState()); + enterRule(_localctx, 40, RULE_fromIdentifier); int _la; try { enterOuterAlt(_localctx, 1); { - setState(304); + setState(308); _la = _input.LA(1); - if ( !(_la==SRC_UNQUOTED_IDENTIFIER || _la==SRC_QUOTED_IDENTIFIER) ) { + if ( !(_la==QUOTED_IDENTIFIER || _la==FROM_UNQUOTED_IDENTIFIER) ) { _errHandler.recoverInline(this); } else { @@ -2360,23 +2375,23 @@ public final QualifiedNameContext qualifiedName() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(306); + setState(310); identifier(); - setState(311); + setState(315); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,28,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(307); + setState(311); match(DOT); - setState(308); + setState(312); identifier(); } } } - setState(313); + setState(317); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,28,_ctx); } @@ -2393,6 +2408,78 @@ public final QualifiedNameContext qualifiedName() throws RecognitionException { return _localctx; } + @SuppressWarnings("CheckReturnValue") + public static class QualifiedNamePatternContext extends ParserRuleContext { + public List identifierPattern() { + return getRuleContexts(IdentifierPatternContext.class); + } + public IdentifierPatternContext identifierPattern(int i) { + return getRuleContext(IdentifierPatternContext.class,i); + } + public List DOT() { return getTokens(EsqlBaseParser.DOT); } + public TerminalNode DOT(int i) { + return getToken(EsqlBaseParser.DOT, i); + } + @SuppressWarnings("this-escape") + public QualifiedNamePatternContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_qualifiedNamePattern; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterQualifiedNamePattern(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitQualifiedNamePattern(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitQualifiedNamePattern(this); + else return visitor.visitChildren(this); + } + } + + public final QualifiedNamePatternContext qualifiedNamePattern() throws RecognitionException { + QualifiedNamePatternContext _localctx = new QualifiedNamePatternContext(_ctx, getState()); + enterRule(_localctx, 44, RULE_qualifiedNamePattern); + try { + int _alt; + enterOuterAlt(_localctx, 1); + { + setState(318); + identifierPattern(); + setState(323); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,29,_ctx); + while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { + if ( _alt==1 ) { + { + { + setState(319); + match(DOT); + setState(320); + identifierPattern(); + } + } + } + setState(325); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,29,_ctx); + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + @SuppressWarnings("CheckReturnValue") public static class IdentifierContext extends ParserRuleContext { public TerminalNode UNQUOTED_IDENTIFIER() { return getToken(EsqlBaseParser.UNQUOTED_IDENTIFIER, 0); } @@ -2419,12 +2506,12 @@ public T accept(ParseTreeVisitor visitor) { public final IdentifierContext identifier() throws RecognitionException { IdentifierContext _localctx = new IdentifierContext(_ctx, getState()); - enterRule(_localctx, 44, RULE_identifier); + enterRule(_localctx, 46, RULE_identifier); int _la; try { enterOuterAlt(_localctx, 1); { - setState(314); + setState(326); _la = _input.LA(1); if ( !(_la==UNQUOTED_IDENTIFIER || _la==QUOTED_IDENTIFIER) ) { _errHandler.recoverInline(this); @@ -2447,6 +2534,60 @@ public final IdentifierContext identifier() throws RecognitionException { return _localctx; } + @SuppressWarnings("CheckReturnValue") + public static class IdentifierPatternContext extends ParserRuleContext { + public TerminalNode PROJECT_UNQUOTED_IDENTIFIER() { return getToken(EsqlBaseParser.PROJECT_UNQUOTED_IDENTIFIER, 0); } + public TerminalNode QUOTED_IDENTIFIER() { return getToken(EsqlBaseParser.QUOTED_IDENTIFIER, 0); } + @SuppressWarnings("this-escape") + public IdentifierPatternContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_identifierPattern; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterIdentifierPattern(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitIdentifierPattern(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitIdentifierPattern(this); + else return visitor.visitChildren(this); + } + } + + public final IdentifierPatternContext identifierPattern() throws RecognitionException { + IdentifierPatternContext _localctx = new IdentifierPatternContext(_ctx, getState()); + enterRule(_localctx, 48, RULE_identifierPattern); + int _la; + try { + enterOuterAlt(_localctx, 1); + { + setState(328); + _la = _input.LA(1); + if ( !(_la==QUOTED_IDENTIFIER || _la==PROJECT_UNQUOTED_IDENTIFIER) ) { + _errHandler.recoverInline(this); + } + else { + if ( _input.LA(1)==Token.EOF ) matchedEOF = true; + _errHandler.reportMatch(this); + consume(); + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + @SuppressWarnings("CheckReturnValue") public static class ConstantContext extends ParserRuleContext { @SuppressWarnings("this-escape") @@ -2698,17 +2839,17 @@ public T accept(ParseTreeVisitor visitor) { public final ConstantContext constant() throws RecognitionException { ConstantContext _localctx = new ConstantContext(_ctx, getState()); - enterRule(_localctx, 46, RULE_constant); + enterRule(_localctx, 50, RULE_constant); int _la; try { - setState(358); + setState(372); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) { case 1: _localctx = new NullLiteralContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(316); + setState(330); match(NULL); } break; @@ -2716,9 +2857,9 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new QualifiedIntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(317); + setState(331); integerValue(); - setState(318); + setState(332); match(UNQUOTED_IDENTIFIER); } break; @@ -2726,7 +2867,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new DecimalLiteralContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(320); + setState(334); decimalValue(); } break; @@ -2734,7 +2875,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new IntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(321); + setState(335); integerValue(); } break; @@ -2742,7 +2883,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanLiteralContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(322); + setState(336); booleanValue(); } break; @@ -2750,7 +2891,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new InputParamContext(_localctx); enterOuterAlt(_localctx, 6); { - setState(323); + setState(337); match(PARAM); } break; @@ -2758,7 +2899,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringLiteralContext(_localctx); enterOuterAlt(_localctx, 7); { - setState(324); + setState(338); string(); } break; @@ -2766,27 +2907,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new NumericArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 8); { - setState(325); + setState(339); match(OPENING_BRACKET); - setState(326); + setState(340); numericValue(); - setState(331); + setState(345); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(327); + setState(341); match(COMMA); - setState(328); + setState(342); numericValue(); } } - setState(333); + setState(347); _errHandler.sync(this); _la = _input.LA(1); } - setState(334); + setState(348); match(CLOSING_BRACKET); } break; @@ -2794,27 +2935,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 9); { - setState(336); + setState(350); match(OPENING_BRACKET); - setState(337); + setState(351); booleanValue(); - setState(342); + setState(356); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(338); + setState(352); match(COMMA); - setState(339); + setState(353); booleanValue(); } } - setState(344); + setState(358); _errHandler.sync(this); _la = _input.LA(1); } - setState(345); + setState(359); match(CLOSING_BRACKET); } break; @@ -2822,27 +2963,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 10); { - setState(347); + setState(361); match(OPENING_BRACKET); - setState(348); + setState(362); string(); - setState(353); + setState(367); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(349); + setState(363); match(COMMA); - setState(350); + setState(364); string(); } } - setState(355); + setState(369); _errHandler.sync(this); _la = _input.LA(1); } - setState(356); + setState(370); match(CLOSING_BRACKET); } break; @@ -2885,13 +3026,13 @@ public T accept(ParseTreeVisitor visitor) { public final LimitCommandContext limitCommand() throws RecognitionException { LimitCommandContext _localctx = new LimitCommandContext(_ctx, getState()); - enterRule(_localctx, 48, RULE_limitCommand); + enterRule(_localctx, 52, RULE_limitCommand); try { enterOuterAlt(_localctx, 1); { - setState(360); + setState(374); match(LIMIT); - setState(361); + setState(375); match(INTEGER_LITERAL); } } @@ -2941,32 +3082,32 @@ public T accept(ParseTreeVisitor visitor) { public final SortCommandContext sortCommand() throws RecognitionException { SortCommandContext _localctx = new SortCommandContext(_ctx, getState()); - enterRule(_localctx, 50, RULE_sortCommand); + enterRule(_localctx, 54, RULE_sortCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(363); + setState(377); match(SORT); - setState(364); + setState(378); orderExpression(); - setState(369); + setState(383); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,33,_ctx); + _alt = getInterpreter().adaptivePredict(_input,34,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(365); + setState(379); match(COMMA); - setState(366); + setState(380); orderExpression(); } } } - setState(371); + setState(385); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,33,_ctx); + _alt = getInterpreter().adaptivePredict(_input,34,_ctx); } } } @@ -3015,19 +3156,19 @@ public T accept(ParseTreeVisitor visitor) { public final OrderExpressionContext orderExpression() throws RecognitionException { OrderExpressionContext _localctx = new OrderExpressionContext(_ctx, getState()); - enterRule(_localctx, 52, RULE_orderExpression); + enterRule(_localctx, 56, RULE_orderExpression); int _la; try { enterOuterAlt(_localctx, 1); { - setState(372); + setState(386); booleanExpression(0); - setState(374); + setState(388); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,35,_ctx) ) { case 1: { - setState(373); + setState(387); ((OrderExpressionContext)_localctx).ordering = _input.LT(1); _la = _input.LA(1); if ( !(_la==ASC || _la==DESC) ) { @@ -3041,14 +3182,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio } break; } - setState(378); + setState(392); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,35,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,36,_ctx) ) { case 1: { - setState(376); + setState(390); match(NULLS); - setState(377); + setState(391); ((OrderExpressionContext)_localctx).nullOrdering = _input.LT(1); _la = _input.LA(1); if ( !(_la==FIRST || _la==LAST) ) { @@ -3078,11 +3219,11 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio @SuppressWarnings("CheckReturnValue") public static class KeepCommandContext extends ParserRuleContext { public TerminalNode KEEP() { return getToken(EsqlBaseParser.KEEP, 0); } - public List sourceIdentifier() { - return getRuleContexts(SourceIdentifierContext.class); + public List qualifiedNamePattern() { + return getRuleContexts(QualifiedNamePatternContext.class); } - public SourceIdentifierContext sourceIdentifier(int i) { - return getRuleContext(SourceIdentifierContext.class,i); + public QualifiedNamePatternContext qualifiedNamePattern(int i) { + return getRuleContext(QualifiedNamePatternContext.class,i); } public List COMMA() { return getTokens(EsqlBaseParser.COMMA); } public TerminalNode COMMA(int i) { @@ -3111,63 +3252,63 @@ public T accept(ParseTreeVisitor visitor) { public final KeepCommandContext keepCommand() throws RecognitionException { KeepCommandContext _localctx = new KeepCommandContext(_ctx, getState()); - enterRule(_localctx, 54, RULE_keepCommand); + enterRule(_localctx, 58, RULE_keepCommand); try { int _alt; - setState(398); + setState(412); _errHandler.sync(this); switch (_input.LA(1)) { case KEEP: enterOuterAlt(_localctx, 1); { - setState(380); + setState(394); match(KEEP); - setState(381); - sourceIdentifier(); - setState(386); + setState(395); + qualifiedNamePattern(); + setState(400); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,36,_ctx); + _alt = getInterpreter().adaptivePredict(_input,37,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(382); + setState(396); match(COMMA); - setState(383); - sourceIdentifier(); + setState(397); + qualifiedNamePattern(); } } } - setState(388); + setState(402); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,36,_ctx); + _alt = getInterpreter().adaptivePredict(_input,37,_ctx); } } break; case PROJECT: enterOuterAlt(_localctx, 2); { - setState(389); + setState(403); match(PROJECT); - setState(390); - sourceIdentifier(); - setState(395); + setState(404); + qualifiedNamePattern(); + setState(409); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,37,_ctx); + _alt = getInterpreter().adaptivePredict(_input,38,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(391); + setState(405); match(COMMA); - setState(392); - sourceIdentifier(); + setState(406); + qualifiedNamePattern(); } } } - setState(397); + setState(411); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,37,_ctx); + _alt = getInterpreter().adaptivePredict(_input,38,_ctx); } } break; @@ -3189,11 +3330,11 @@ public final KeepCommandContext keepCommand() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class DropCommandContext extends ParserRuleContext { public TerminalNode DROP() { return getToken(EsqlBaseParser.DROP, 0); } - public List sourceIdentifier() { - return getRuleContexts(SourceIdentifierContext.class); + public List qualifiedNamePattern() { + return getRuleContexts(QualifiedNamePatternContext.class); } - public SourceIdentifierContext sourceIdentifier(int i) { - return getRuleContext(SourceIdentifierContext.class,i); + public QualifiedNamePatternContext qualifiedNamePattern(int i) { + return getRuleContext(QualifiedNamePatternContext.class,i); } public List COMMA() { return getTokens(EsqlBaseParser.COMMA); } public TerminalNode COMMA(int i) { @@ -3221,32 +3362,32 @@ public T accept(ParseTreeVisitor visitor) { public final DropCommandContext dropCommand() throws RecognitionException { DropCommandContext _localctx = new DropCommandContext(_ctx, getState()); - enterRule(_localctx, 56, RULE_dropCommand); + enterRule(_localctx, 60, RULE_dropCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(400); + setState(414); match(DROP); - setState(401); - sourceIdentifier(); - setState(406); + setState(415); + qualifiedNamePattern(); + setState(420); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,39,_ctx); + _alt = getInterpreter().adaptivePredict(_input,40,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(402); + setState(416); match(COMMA); - setState(403); - sourceIdentifier(); + setState(417); + qualifiedNamePattern(); } } } - setState(408); + setState(422); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,39,_ctx); + _alt = getInterpreter().adaptivePredict(_input,40,_ctx); } } } @@ -3296,32 +3437,32 @@ public T accept(ParseTreeVisitor visitor) { public final RenameCommandContext renameCommand() throws RecognitionException { RenameCommandContext _localctx = new RenameCommandContext(_ctx, getState()); - enterRule(_localctx, 58, RULE_renameCommand); + enterRule(_localctx, 62, RULE_renameCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(409); + setState(423); match(RENAME); - setState(410); + setState(424); renameClause(); - setState(415); + setState(429); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,40,_ctx); + _alt = getInterpreter().adaptivePredict(_input,41,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(411); + setState(425); match(COMMA); - setState(412); + setState(426); renameClause(); } } } - setState(417); + setState(431); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,40,_ctx); + _alt = getInterpreter().adaptivePredict(_input,41,_ctx); } } } @@ -3338,14 +3479,14 @@ public final RenameCommandContext renameCommand() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class RenameClauseContext extends ParserRuleContext { - public SourceIdentifierContext oldName; - public SourceIdentifierContext newName; + public QualifiedNamePatternContext oldName; + public QualifiedNamePatternContext newName; public TerminalNode AS() { return getToken(EsqlBaseParser.AS, 0); } - public List sourceIdentifier() { - return getRuleContexts(SourceIdentifierContext.class); + public List qualifiedNamePattern() { + return getRuleContexts(QualifiedNamePatternContext.class); } - public SourceIdentifierContext sourceIdentifier(int i) { - return getRuleContext(SourceIdentifierContext.class,i); + public QualifiedNamePatternContext qualifiedNamePattern(int i) { + return getRuleContext(QualifiedNamePatternContext.class,i); } @SuppressWarnings("this-escape") public RenameClauseContext(ParserRuleContext parent, int invokingState) { @@ -3369,16 +3510,16 @@ public T accept(ParseTreeVisitor visitor) { public final RenameClauseContext renameClause() throws RecognitionException { RenameClauseContext _localctx = new RenameClauseContext(_ctx, getState()); - enterRule(_localctx, 60, RULE_renameClause); + enterRule(_localctx, 64, RULE_renameClause); try { enterOuterAlt(_localctx, 1); { - setState(418); - ((RenameClauseContext)_localctx).oldName = sourceIdentifier(); - setState(419); + setState(432); + ((RenameClauseContext)_localctx).oldName = qualifiedNamePattern(); + setState(433); match(AS); - setState(420); - ((RenameClauseContext)_localctx).newName = sourceIdentifier(); + setState(434); + ((RenameClauseContext)_localctx).newName = qualifiedNamePattern(); } } catch (RecognitionException re) { @@ -3426,22 +3567,22 @@ public T accept(ParseTreeVisitor visitor) { public final DissectCommandContext dissectCommand() throws RecognitionException { DissectCommandContext _localctx = new DissectCommandContext(_ctx, getState()); - enterRule(_localctx, 62, RULE_dissectCommand); + enterRule(_localctx, 66, RULE_dissectCommand); try { enterOuterAlt(_localctx, 1); { - setState(422); + setState(436); match(DISSECT); - setState(423); + setState(437); primaryExpression(); - setState(424); + setState(438); string(); - setState(426); + setState(440); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,41,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,42,_ctx) ) { case 1: { - setState(425); + setState(439); commandOptions(); } break; @@ -3490,15 +3631,15 @@ public T accept(ParseTreeVisitor visitor) { public final GrokCommandContext grokCommand() throws RecognitionException { GrokCommandContext _localctx = new GrokCommandContext(_ctx, getState()); - enterRule(_localctx, 64, RULE_grokCommand); + enterRule(_localctx, 68, RULE_grokCommand); try { enterOuterAlt(_localctx, 1); { - setState(428); + setState(442); match(GROK); - setState(429); + setState(443); primaryExpression(); - setState(430); + setState(444); string(); } } @@ -3516,8 +3657,8 @@ public final GrokCommandContext grokCommand() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class MvExpandCommandContext extends ParserRuleContext { public TerminalNode MV_EXPAND() { return getToken(EsqlBaseParser.MV_EXPAND, 0); } - public SourceIdentifierContext sourceIdentifier() { - return getRuleContext(SourceIdentifierContext.class,0); + public QualifiedNameContext qualifiedName() { + return getRuleContext(QualifiedNameContext.class,0); } @SuppressWarnings("this-escape") public MvExpandCommandContext(ParserRuleContext parent, int invokingState) { @@ -3541,14 +3682,14 @@ public T accept(ParseTreeVisitor visitor) { public final MvExpandCommandContext mvExpandCommand() throws RecognitionException { MvExpandCommandContext _localctx = new MvExpandCommandContext(_ctx, getState()); - enterRule(_localctx, 66, RULE_mvExpandCommand); + enterRule(_localctx, 70, RULE_mvExpandCommand); try { enterOuterAlt(_localctx, 1); { - setState(432); + setState(446); match(MV_EXPAND); - setState(433); - sourceIdentifier(); + setState(447); + qualifiedName(); } } catch (RecognitionException re) { @@ -3596,30 +3737,30 @@ public T accept(ParseTreeVisitor visitor) { public final CommandOptionsContext commandOptions() throws RecognitionException { CommandOptionsContext _localctx = new CommandOptionsContext(_ctx, getState()); - enterRule(_localctx, 68, RULE_commandOptions); + enterRule(_localctx, 72, RULE_commandOptions); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(435); + setState(449); commandOption(); - setState(440); + setState(454); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,42,_ctx); + _alt = getInterpreter().adaptivePredict(_input,43,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(436); + setState(450); match(COMMA); - setState(437); + setState(451); commandOption(); } } } - setState(442); + setState(456); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,42,_ctx); + _alt = getInterpreter().adaptivePredict(_input,43,_ctx); } } } @@ -3665,15 +3806,15 @@ public T accept(ParseTreeVisitor visitor) { public final CommandOptionContext commandOption() throws RecognitionException { CommandOptionContext _localctx = new CommandOptionContext(_ctx, getState()); - enterRule(_localctx, 70, RULE_commandOption); + enterRule(_localctx, 74, RULE_commandOption); try { enterOuterAlt(_localctx, 1); { - setState(443); + setState(457); identifier(); - setState(444); + setState(458); match(ASSIGN); - setState(445); + setState(459); constant(); } } @@ -3714,12 +3855,12 @@ public T accept(ParseTreeVisitor visitor) { public final BooleanValueContext booleanValue() throws RecognitionException { BooleanValueContext _localctx = new BooleanValueContext(_ctx, getState()); - enterRule(_localctx, 72, RULE_booleanValue); + enterRule(_localctx, 76, RULE_booleanValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(447); + setState(461); _la = _input.LA(1); if ( !(_la==FALSE || _la==TRUE) ) { _errHandler.recoverInline(this); @@ -3772,22 +3913,22 @@ public T accept(ParseTreeVisitor visitor) { public final NumericValueContext numericValue() throws RecognitionException { NumericValueContext _localctx = new NumericValueContext(_ctx, getState()); - enterRule(_localctx, 74, RULE_numericValue); + enterRule(_localctx, 78, RULE_numericValue); try { - setState(451); + setState(465); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,43,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,44,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(449); + setState(463); decimalValue(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(450); + setState(464); integerValue(); } break; @@ -3831,17 +3972,17 @@ public T accept(ParseTreeVisitor visitor) { public final DecimalValueContext decimalValue() throws RecognitionException { DecimalValueContext _localctx = new DecimalValueContext(_ctx, getState()); - enterRule(_localctx, 76, RULE_decimalValue); + enterRule(_localctx, 80, RULE_decimalValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(454); + setState(468); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(453); + setState(467); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -3854,7 +3995,7 @@ public final DecimalValueContext decimalValue() throws RecognitionException { } } - setState(456); + setState(470); match(DECIMAL_LITERAL); } } @@ -3896,17 +4037,17 @@ public T accept(ParseTreeVisitor visitor) { public final IntegerValueContext integerValue() throws RecognitionException { IntegerValueContext _localctx = new IntegerValueContext(_ctx, getState()); - enterRule(_localctx, 78, RULE_integerValue); + enterRule(_localctx, 82, RULE_integerValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(459); + setState(473); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(458); + setState(472); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -3919,7 +4060,7 @@ public final IntegerValueContext integerValue() throws RecognitionException { } } - setState(461); + setState(475); match(INTEGER_LITERAL); } } @@ -3959,11 +4100,11 @@ public T accept(ParseTreeVisitor visitor) { public final StringContext string() throws RecognitionException { StringContext _localctx = new StringContext(_ctx, getState()); - enterRule(_localctx, 80, RULE_string); + enterRule(_localctx, 84, RULE_string); try { enterOuterAlt(_localctx, 1); { - setState(463); + setState(477); match(STRING); } } @@ -4008,14 +4149,14 @@ public T accept(ParseTreeVisitor visitor) { public final ComparisonOperatorContext comparisonOperator() throws RecognitionException { ComparisonOperatorContext _localctx = new ComparisonOperatorContext(_ctx, getState()); - enterRule(_localctx, 82, RULE_comparisonOperator); + enterRule(_localctx, 86, RULE_comparisonOperator); int _la; try { enterOuterAlt(_localctx, 1); { - setState(465); + setState(479); _la = _input.LA(1); - if ( !(((_la) & ~0x3f) == 0 && ((1L << _la) & 1134907106097364992L) != 0) ) { + if ( !(((_la) & ~0x3f) == 0 && ((1L << _la) & 283726776524341248L) != 0) ) { _errHandler.recoverInline(this); } else { @@ -4064,13 +4205,13 @@ public T accept(ParseTreeVisitor visitor) { public final ExplainCommandContext explainCommand() throws RecognitionException { ExplainCommandContext _localctx = new ExplainCommandContext(_ctx, getState()); - enterRule(_localctx, 84, RULE_explainCommand); + enterRule(_localctx, 88, RULE_explainCommand); try { enterOuterAlt(_localctx, 1); { - setState(467); + setState(481); match(EXPLAIN); - setState(468); + setState(482); subqueryExpression(); } } @@ -4114,15 +4255,15 @@ public T accept(ParseTreeVisitor visitor) { public final SubqueryExpressionContext subqueryExpression() throws RecognitionException { SubqueryExpressionContext _localctx = new SubqueryExpressionContext(_ctx, getState()); - enterRule(_localctx, 86, RULE_subqueryExpression); + enterRule(_localctx, 90, RULE_subqueryExpression); try { enterOuterAlt(_localctx, 1); { - setState(470); + setState(484); match(OPENING_BRACKET); - setState(471); + setState(485); query(0); - setState(472); + setState(486); match(CLOSING_BRACKET); } } @@ -4194,18 +4335,18 @@ public T accept(ParseTreeVisitor visitor) { public final ShowCommandContext showCommand() throws RecognitionException { ShowCommandContext _localctx = new ShowCommandContext(_ctx, getState()); - enterRule(_localctx, 88, RULE_showCommand); + enterRule(_localctx, 92, RULE_showCommand); try { - setState(478); + setState(492); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) { case 1: _localctx = new ShowInfoContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(474); + setState(488); match(SHOW); - setState(475); + setState(489); match(INFO); } break; @@ -4213,9 +4354,9 @@ public final ShowCommandContext showCommand() throws RecognitionException { _localctx = new ShowFunctionsContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(476); + setState(490); match(SHOW); - setState(477); + setState(491); match(FUNCTIONS); } break; @@ -4234,14 +4375,11 @@ public final ShowCommandContext showCommand() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class EnrichCommandContext extends ParserRuleContext { - public SourceIdentifierContext policyName; - public SourceIdentifierContext matchField; + public FromIdentifierContext policyName; + public QualifiedNamePatternContext matchField; public TerminalNode ENRICH() { return getToken(EsqlBaseParser.ENRICH, 0); } - public List sourceIdentifier() { - return getRuleContexts(SourceIdentifierContext.class); - } - public SourceIdentifierContext sourceIdentifier(int i) { - return getRuleContext(SourceIdentifierContext.class,i); + public FromIdentifierContext fromIdentifier() { + return getRuleContext(FromIdentifierContext.class,0); } public TerminalNode ON() { return getToken(EsqlBaseParser.ON, 0); } public TerminalNode WITH() { return getToken(EsqlBaseParser.WITH, 0); } @@ -4251,6 +4389,9 @@ public List enrichWithClause() { public EnrichWithClauseContext enrichWithClause(int i) { return getRuleContext(EnrichWithClauseContext.class,i); } + public QualifiedNamePatternContext qualifiedNamePattern() { + return getRuleContext(QualifiedNamePatternContext.class,0); + } public List COMMA() { return getTokens(EsqlBaseParser.COMMA); } public TerminalNode COMMA(int i) { return getToken(EsqlBaseParser.COMMA, i); @@ -4277,53 +4418,53 @@ public T accept(ParseTreeVisitor visitor) { public final EnrichCommandContext enrichCommand() throws RecognitionException { EnrichCommandContext _localctx = new EnrichCommandContext(_ctx, getState()); - enterRule(_localctx, 90, RULE_enrichCommand); + enterRule(_localctx, 94, RULE_enrichCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(480); + setState(494); match(ENRICH); - setState(481); - ((EnrichCommandContext)_localctx).policyName = sourceIdentifier(); - setState(484); + setState(495); + ((EnrichCommandContext)_localctx).policyName = fromIdentifier(); + setState(498); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,48,_ctx) ) { case 1: { - setState(482); + setState(496); match(ON); - setState(483); - ((EnrichCommandContext)_localctx).matchField = sourceIdentifier(); + setState(497); + ((EnrichCommandContext)_localctx).matchField = qualifiedNamePattern(); } break; } - setState(495); + setState(509); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,49,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,50,_ctx) ) { case 1: { - setState(486); + setState(500); match(WITH); - setState(487); + setState(501); enrichWithClause(); - setState(492); + setState(506); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,48,_ctx); + _alt = getInterpreter().adaptivePredict(_input,49,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(488); + setState(502); match(COMMA); - setState(489); + setState(503); enrichWithClause(); } } } - setState(494); + setState(508); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,48,_ctx); + _alt = getInterpreter().adaptivePredict(_input,49,_ctx); } } break; @@ -4343,13 +4484,13 @@ public final EnrichCommandContext enrichCommand() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class EnrichWithClauseContext extends ParserRuleContext { - public SourceIdentifierContext newName; - public SourceIdentifierContext enrichField; - public List sourceIdentifier() { - return getRuleContexts(SourceIdentifierContext.class); + public QualifiedNamePatternContext newName; + public QualifiedNamePatternContext enrichField; + public List qualifiedNamePattern() { + return getRuleContexts(QualifiedNamePatternContext.class); } - public SourceIdentifierContext sourceIdentifier(int i) { - return getRuleContext(SourceIdentifierContext.class,i); + public QualifiedNamePatternContext qualifiedNamePattern(int i) { + return getRuleContext(QualifiedNamePatternContext.class,i); } public TerminalNode ASSIGN() { return getToken(EsqlBaseParser.ASSIGN, 0); } @SuppressWarnings("this-escape") @@ -4374,24 +4515,24 @@ public T accept(ParseTreeVisitor visitor) { public final EnrichWithClauseContext enrichWithClause() throws RecognitionException { EnrichWithClauseContext _localctx = new EnrichWithClauseContext(_ctx, getState()); - enterRule(_localctx, 92, RULE_enrichWithClause); + enterRule(_localctx, 96, RULE_enrichWithClause); try { enterOuterAlt(_localctx, 1); { - setState(500); + setState(514); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,50,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,51,_ctx) ) { case 1: { - setState(497); - ((EnrichWithClauseContext)_localctx).newName = sourceIdentifier(); - setState(498); + setState(511); + ((EnrichWithClauseContext)_localctx).newName = qualifiedNamePattern(); + setState(512); match(ASSIGN); } break; } - setState(502); - ((EnrichWithClauseContext)_localctx).enrichField = sourceIdentifier(); + setState(516); + ((EnrichWithClauseContext)_localctx).enrichField = qualifiedNamePattern(); } } catch (RecognitionException re) { @@ -4443,7 +4584,7 @@ private boolean operatorExpression_sempred(OperatorExpressionContext _localctx, } public static final String _serializedATN = - "\u0004\u0001Q\u01f9\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+ + "\u0004\u0001b\u0207\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+ "\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002"+ "\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007\u0002"+ "\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b\u0002"+ @@ -4456,319 +4597,327 @@ private boolean operatorExpression_sempred(OperatorExpressionContext _localctx, "\u0002\u001f\u0007\u001f\u0002 \u0007 \u0002!\u0007!\u0002\"\u0007\"\u0002"+ "#\u0007#\u0002$\u0007$\u0002%\u0007%\u0002&\u0007&\u0002\'\u0007\'\u0002"+ "(\u0007(\u0002)\u0007)\u0002*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002"+ - "-\u0007-\u0002.\u0007.\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0001"+ - "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0005\u0001"+ - "h\b\u0001\n\u0001\f\u0001k\t\u0001\u0001\u0002\u0001\u0002\u0001\u0002"+ - "\u0001\u0002\u0003\u0002q\b\u0002\u0001\u0003\u0001\u0003\u0001\u0003"+ + "-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u0001\u0000\u0001\u0000"+ + "\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ + "\u0001\u0001\u0005\u0001l\b\u0001\n\u0001\f\u0001o\t\u0001\u0001\u0002"+ + "\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002u\b\u0002\u0001\u0003"+ + "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ - "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0003\u0003\u0080\b\u0003"+ - "\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005"+ - "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u008c\b\u0005"+ + "\u0003\u0003\u0084\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005"+ + "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ + "\u0003\u0005\u0090\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ + "\u0001\u0005\u0005\u0005\u0097\b\u0005\n\u0005\f\u0005\u009a\t\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00a1"+ + "\b\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00a5\b\u0005\u0001\u0005"+ "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005"+ - "\u0093\b\u0005\n\u0005\f\u0005\u0096\t\u0005\u0001\u0005\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u009d\b\u0005\u0001\u0005\u0001"+ - "\u0005\u0003\u0005\u00a1\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00a9\b\u0005\n\u0005\f\u0005"+ - "\u00ac\t\u0005\u0001\u0006\u0001\u0006\u0003\u0006\u00b0\b\u0006\u0001"+ - "\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00b7"+ - "\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00bc\b\u0006"+ - "\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0003\u0007"+ - "\u00c3\b\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0003\b\u00c9\b\b\u0001"+ - "\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0005\b\u00d1\b\b\n\b\f\b\u00d4"+ - "\t\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0003\t\u00dd"+ - "\b\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0005\n\u00e5\b\n"+ - "\n\n\f\n\u00e8\t\n\u0003\n\u00ea\b\n\u0001\n\u0001\n\u0001\u000b\u0001"+ - "\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0005\f\u00f4\b\f\n\f\f\f\u00f7"+ - "\t\f\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0003\r\u00fe\b\r\u0001\u000e"+ - "\u0001\u000e\u0001\u000e\u0001\u000e\u0005\u000e\u0104\b\u000e\n\u000e"+ - "\f\u000e\u0107\t\u000e\u0001\u000e\u0003\u000e\u010a\b\u000e\u0001\u000f"+ - "\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0005\u000f\u0111\b\u000f"+ - "\n\u000f\f\u000f\u0114\t\u000f\u0001\u000f\u0001\u000f\u0001\u0010\u0001"+ - "\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0003\u0011\u011d\b\u0011\u0001"+ - "\u0011\u0001\u0011\u0003\u0011\u0121\b\u0011\u0001\u0012\u0001\u0012\u0001"+ - "\u0012\u0001\u0012\u0003\u0012\u0127\b\u0012\u0001\u0013\u0001\u0013\u0001"+ - "\u0013\u0005\u0013\u012c\b\u0013\n\u0013\f\u0013\u012f\t\u0013\u0001\u0014"+ - "\u0001\u0014\u0001\u0015\u0001\u0015\u0001\u0015\u0005\u0015\u0136\b\u0015"+ - "\n\u0015\f\u0015\u0139\t\u0015\u0001\u0016\u0001\u0016\u0001\u0017\u0001"+ - "\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001"+ - "\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0005"+ - "\u0017\u014a\b\u0017\n\u0017\f\u0017\u014d\t\u0017\u0001\u0017\u0001\u0017"+ - "\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0005\u0017\u0155\b\u0017"+ - "\n\u0017\f\u0017\u0158\t\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001"+ - "\u0017\u0001\u0017\u0001\u0017\u0005\u0017\u0160\b\u0017\n\u0017\f\u0017"+ - "\u0163\t\u0017\u0001\u0017\u0001\u0017\u0003\u0017\u0167\b\u0017\u0001"+ - "\u0018\u0001\u0018\u0001\u0018\u0001\u0019\u0001\u0019\u0001\u0019\u0001"+ - "\u0019\u0005\u0019\u0170\b\u0019\n\u0019\f\u0019\u0173\t\u0019\u0001\u001a"+ - "\u0001\u001a\u0003\u001a\u0177\b\u001a\u0001\u001a\u0001\u001a\u0003\u001a"+ - "\u017b\b\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0005\u001b"+ - "\u0181\b\u001b\n\u001b\f\u001b\u0184\t\u001b\u0001\u001b\u0001\u001b\u0001"+ - "\u001b\u0001\u001b\u0005\u001b\u018a\b\u001b\n\u001b\f\u001b\u018d\t\u001b"+ - "\u0003\u001b\u018f\b\u001b\u0001\u001c\u0001\u001c\u0001\u001c\u0001\u001c"+ - "\u0005\u001c\u0195\b\u001c\n\u001c\f\u001c\u0198\t\u001c\u0001\u001d\u0001"+ - "\u001d\u0001\u001d\u0001\u001d\u0005\u001d\u019e\b\u001d\n\u001d\f\u001d"+ - "\u01a1\t\u001d\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001f"+ - "\u0001\u001f\u0001\u001f\u0001\u001f\u0003\u001f\u01ab\b\u001f\u0001 "+ - "\u0001 \u0001 \u0001 \u0001!\u0001!\u0001!\u0001\"\u0001\"\u0001\"\u0005"+ - "\"\u01b7\b\"\n\"\f\"\u01ba\t\"\u0001#\u0001#\u0001#\u0001#\u0001$\u0001"+ - "$\u0001%\u0001%\u0003%\u01c4\b%\u0001&\u0003&\u01c7\b&\u0001&\u0001&\u0001"+ - "\'\u0003\'\u01cc\b\'\u0001\'\u0001\'\u0001(\u0001(\u0001)\u0001)\u0001"+ - "*\u0001*\u0001*\u0001+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001,\u0001"+ - ",\u0003,\u01df\b,\u0001-\u0001-\u0001-\u0001-\u0003-\u01e5\b-\u0001-\u0001"+ - "-\u0001-\u0001-\u0005-\u01eb\b-\n-\f-\u01ee\t-\u0003-\u01f0\b-\u0001."+ - "\u0001.\u0001.\u0003.\u01f5\b.\u0001.\u0001.\u0001.\u0000\u0003\u0002"+ - "\n\u0010/\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012\u0014\u0016"+ - "\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPRTVXZ\\\u0000\b\u0001"+ - "\u0000<=\u0001\u0000>@\u0001\u0000LM\u0001\u0000CD\u0002\u0000 ##\u0001"+ - "\u0000&\'\u0002\u0000%%33\u0001\u00006;\u0217\u0000^\u0001\u0000\u0000"+ - "\u0000\u0002a\u0001\u0000\u0000\u0000\u0004p\u0001\u0000\u0000\u0000\u0006"+ - "\u007f\u0001\u0000\u0000\u0000\b\u0081\u0001\u0000\u0000\u0000\n\u00a0"+ - "\u0001\u0000\u0000\u0000\f\u00bb\u0001\u0000\u0000\u0000\u000e\u00c2\u0001"+ - "\u0000\u0000\u0000\u0010\u00c8\u0001\u0000\u0000\u0000\u0012\u00dc\u0001"+ - "\u0000\u0000\u0000\u0014\u00de\u0001\u0000\u0000\u0000\u0016\u00ed\u0001"+ - "\u0000\u0000\u0000\u0018\u00f0\u0001\u0000\u0000\u0000\u001a\u00fd\u0001"+ - "\u0000\u0000\u0000\u001c\u00ff\u0001\u0000\u0000\u0000\u001e\u010b\u0001"+ - "\u0000\u0000\u0000 \u0117\u0001\u0000\u0000\u0000\"\u011a\u0001\u0000"+ - "\u0000\u0000$\u0122\u0001\u0000\u0000\u0000&\u0128\u0001\u0000\u0000\u0000"+ - "(\u0130\u0001\u0000\u0000\u0000*\u0132\u0001\u0000\u0000\u0000,\u013a"+ - "\u0001\u0000\u0000\u0000.\u0166\u0001\u0000\u0000\u00000\u0168\u0001\u0000"+ - "\u0000\u00002\u016b\u0001\u0000\u0000\u00004\u0174\u0001\u0000\u0000\u0000"+ - "6\u018e\u0001\u0000\u0000\u00008\u0190\u0001\u0000\u0000\u0000:\u0199"+ - "\u0001\u0000\u0000\u0000<\u01a2\u0001\u0000\u0000\u0000>\u01a6\u0001\u0000"+ - "\u0000\u0000@\u01ac\u0001\u0000\u0000\u0000B\u01b0\u0001\u0000\u0000\u0000"+ - "D\u01b3\u0001\u0000\u0000\u0000F\u01bb\u0001\u0000\u0000\u0000H\u01bf"+ - "\u0001\u0000\u0000\u0000J\u01c3\u0001\u0000\u0000\u0000L\u01c6\u0001\u0000"+ - "\u0000\u0000N\u01cb\u0001\u0000\u0000\u0000P\u01cf\u0001\u0000\u0000\u0000"+ - "R\u01d1\u0001\u0000\u0000\u0000T\u01d3\u0001\u0000\u0000\u0000V\u01d6"+ - "\u0001\u0000\u0000\u0000X\u01de\u0001\u0000\u0000\u0000Z\u01e0\u0001\u0000"+ - "\u0000\u0000\\\u01f4\u0001\u0000\u0000\u0000^_\u0003\u0002\u0001\u0000"+ - "_`\u0005\u0000\u0000\u0001`\u0001\u0001\u0000\u0000\u0000ab\u0006\u0001"+ - "\uffff\uffff\u0000bc\u0003\u0004\u0002\u0000ci\u0001\u0000\u0000\u0000"+ - "de\n\u0001\u0000\u0000ef\u0005\u001a\u0000\u0000fh\u0003\u0006\u0003\u0000"+ - "gd\u0001\u0000\u0000\u0000hk\u0001\u0000\u0000\u0000ig\u0001\u0000\u0000"+ - "\u0000ij\u0001\u0000\u0000\u0000j\u0003\u0001\u0000\u0000\u0000ki\u0001"+ - "\u0000\u0000\u0000lq\u0003T*\u0000mq\u0003\u001c\u000e\u0000nq\u0003\u0016"+ - "\u000b\u0000oq\u0003X,\u0000pl\u0001\u0000\u0000\u0000pm\u0001\u0000\u0000"+ - "\u0000pn\u0001\u0000\u0000\u0000po\u0001\u0000\u0000\u0000q\u0005\u0001"+ - "\u0000\u0000\u0000r\u0080\u0003 \u0010\u0000s\u0080\u0003$\u0012\u0000"+ - "t\u0080\u00030\u0018\u0000u\u0080\u00036\u001b\u0000v\u0080\u00032\u0019"+ - "\u0000w\u0080\u0003\"\u0011\u0000x\u0080\u0003\b\u0004\u0000y\u0080\u0003"+ - "8\u001c\u0000z\u0080\u0003:\u001d\u0000{\u0080\u0003>\u001f\u0000|\u0080"+ - "\u0003@ \u0000}\u0080\u0003Z-\u0000~\u0080\u0003B!\u0000\u007fr\u0001"+ - "\u0000\u0000\u0000\u007fs\u0001\u0000\u0000\u0000\u007ft\u0001\u0000\u0000"+ - "\u0000\u007fu\u0001\u0000\u0000\u0000\u007fv\u0001\u0000\u0000\u0000\u007f"+ - "w\u0001\u0000\u0000\u0000\u007fx\u0001\u0000\u0000\u0000\u007fy\u0001"+ - "\u0000\u0000\u0000\u007fz\u0001\u0000\u0000\u0000\u007f{\u0001\u0000\u0000"+ - "\u0000\u007f|\u0001\u0000\u0000\u0000\u007f}\u0001\u0000\u0000\u0000\u007f"+ - "~\u0001\u0000\u0000\u0000\u0080\u0007\u0001\u0000\u0000\u0000\u0081\u0082"+ - "\u0005\u0012\u0000\u0000\u0082\u0083\u0003\n\u0005\u0000\u0083\t\u0001"+ - "\u0000\u0000\u0000\u0084\u0085\u0006\u0005\uffff\uffff\u0000\u0085\u0086"+ - "\u0005,\u0000\u0000\u0086\u00a1\u0003\n\u0005\u0007\u0087\u00a1\u0003"+ - "\u000e\u0007\u0000\u0088\u00a1\u0003\f\u0006\u0000\u0089\u008b\u0003\u000e"+ - "\u0007\u0000\u008a\u008c\u0005,\u0000\u0000\u008b\u008a\u0001\u0000\u0000"+ - "\u0000\u008b\u008c\u0001\u0000\u0000\u0000\u008c\u008d\u0001\u0000\u0000"+ - "\u0000\u008d\u008e\u0005)\u0000\u0000\u008e\u008f\u0005(\u0000\u0000\u008f"+ - "\u0094\u0003\u000e\u0007\u0000\u0090\u0091\u0005\"\u0000\u0000\u0091\u0093"+ - "\u0003\u000e\u0007\u0000\u0092\u0090\u0001\u0000\u0000\u0000\u0093\u0096"+ - "\u0001\u0000\u0000\u0000\u0094\u0092\u0001\u0000\u0000\u0000\u0094\u0095"+ - "\u0001\u0000\u0000\u0000\u0095\u0097\u0001\u0000\u0000\u0000\u0096\u0094"+ - "\u0001\u0000\u0000\u0000\u0097\u0098\u00052\u0000\u0000\u0098\u00a1\u0001"+ - "\u0000\u0000\u0000\u0099\u009a\u0003\u000e\u0007\u0000\u009a\u009c\u0005"+ - "*\u0000\u0000\u009b\u009d\u0005,\u0000\u0000\u009c\u009b\u0001\u0000\u0000"+ - "\u0000\u009c\u009d\u0001\u0000\u0000\u0000\u009d\u009e\u0001\u0000\u0000"+ - "\u0000\u009e\u009f\u0005-\u0000\u0000\u009f\u00a1\u0001\u0000\u0000\u0000"+ - "\u00a0\u0084\u0001\u0000\u0000\u0000\u00a0\u0087\u0001\u0000\u0000\u0000"+ - "\u00a0\u0088\u0001\u0000\u0000\u0000\u00a0\u0089\u0001\u0000\u0000\u0000"+ - "\u00a0\u0099\u0001\u0000\u0000\u0000\u00a1\u00aa\u0001\u0000\u0000\u0000"+ - "\u00a2\u00a3\n\u0004\u0000\u0000\u00a3\u00a4\u0005\u001f\u0000\u0000\u00a4"+ - "\u00a9\u0003\n\u0005\u0005\u00a5\u00a6\n\u0003\u0000\u0000\u00a6\u00a7"+ - "\u0005/\u0000\u0000\u00a7\u00a9\u0003\n\u0005\u0004\u00a8\u00a2\u0001"+ - "\u0000\u0000\u0000\u00a8\u00a5\u0001\u0000\u0000\u0000\u00a9\u00ac\u0001"+ - "\u0000\u0000\u0000\u00aa\u00a8\u0001\u0000\u0000\u0000\u00aa\u00ab\u0001"+ - "\u0000\u0000\u0000\u00ab\u000b\u0001\u0000\u0000\u0000\u00ac\u00aa\u0001"+ - "\u0000\u0000\u0000\u00ad\u00af\u0003\u000e\u0007\u0000\u00ae\u00b0\u0005"+ - ",\u0000\u0000\u00af\u00ae\u0001\u0000\u0000\u0000\u00af\u00b0\u0001\u0000"+ - "\u0000\u0000\u00b0\u00b1\u0001\u0000\u0000\u0000\u00b1\u00b2\u0005+\u0000"+ - "\u0000\u00b2\u00b3\u0003P(\u0000\u00b3\u00bc\u0001\u0000\u0000\u0000\u00b4"+ - "\u00b6\u0003\u000e\u0007\u0000\u00b5\u00b7\u0005,\u0000\u0000\u00b6\u00b5"+ - "\u0001\u0000\u0000\u0000\u00b6\u00b7\u0001\u0000\u0000\u0000\u00b7\u00b8"+ - "\u0001\u0000\u0000\u0000\u00b8\u00b9\u00051\u0000\u0000\u00b9\u00ba\u0003"+ - "P(\u0000\u00ba\u00bc\u0001\u0000\u0000\u0000\u00bb\u00ad\u0001\u0000\u0000"+ - "\u0000\u00bb\u00b4\u0001\u0000\u0000\u0000\u00bc\r\u0001\u0000\u0000\u0000"+ - "\u00bd\u00c3\u0003\u0010\b\u0000\u00be\u00bf\u0003\u0010\b\u0000\u00bf"+ - "\u00c0\u0003R)\u0000\u00c0\u00c1\u0003\u0010\b\u0000\u00c1\u00c3\u0001"+ - "\u0000\u0000\u0000\u00c2\u00bd\u0001\u0000\u0000\u0000\u00c2\u00be\u0001"+ - "\u0000\u0000\u0000\u00c3\u000f\u0001\u0000\u0000\u0000\u00c4\u00c5\u0006"+ - "\b\uffff\uffff\u0000\u00c5\u00c9\u0003\u0012\t\u0000\u00c6\u00c7\u0007"+ - "\u0000\u0000\u0000\u00c7\u00c9\u0003\u0010\b\u0003\u00c8\u00c4\u0001\u0000"+ - "\u0000\u0000\u00c8\u00c6\u0001\u0000\u0000\u0000\u00c9\u00d2\u0001\u0000"+ - "\u0000\u0000\u00ca\u00cb\n\u0002\u0000\u0000\u00cb\u00cc\u0007\u0001\u0000"+ - "\u0000\u00cc\u00d1\u0003\u0010\b\u0003\u00cd\u00ce\n\u0001\u0000\u0000"+ - "\u00ce\u00cf\u0007\u0000\u0000\u0000\u00cf\u00d1\u0003\u0010\b\u0002\u00d0"+ - "\u00ca\u0001\u0000\u0000\u0000\u00d0\u00cd\u0001\u0000\u0000\u0000\u00d1"+ - "\u00d4\u0001\u0000\u0000\u0000\u00d2\u00d0\u0001\u0000\u0000\u0000\u00d2"+ - "\u00d3\u0001\u0000\u0000\u0000\u00d3\u0011\u0001\u0000\u0000\u0000\u00d4"+ - "\u00d2\u0001\u0000\u0000\u0000\u00d5\u00dd\u0003.\u0017\u0000\u00d6\u00dd"+ - "\u0003*\u0015\u0000\u00d7\u00dd\u0003\u0014\n\u0000\u00d8\u00d9\u0005"+ - "(\u0000\u0000\u00d9\u00da\u0003\n\u0005\u0000\u00da\u00db\u00052\u0000"+ - "\u0000\u00db\u00dd\u0001\u0000\u0000\u0000\u00dc\u00d5\u0001\u0000\u0000"+ - "\u0000\u00dc\u00d6\u0001\u0000\u0000\u0000\u00dc\u00d7\u0001\u0000\u0000"+ - "\u0000\u00dc\u00d8\u0001\u0000\u0000\u0000\u00dd\u0013\u0001\u0000\u0000"+ - "\u0000\u00de\u00df\u0003,\u0016\u0000\u00df\u00e9\u0005(\u0000\u0000\u00e0"+ - "\u00ea\u0005>\u0000\u0000\u00e1\u00e6\u0003\n\u0005\u0000\u00e2\u00e3"+ - "\u0005\"\u0000\u0000\u00e3\u00e5\u0003\n\u0005\u0000\u00e4\u00e2\u0001"+ - "\u0000\u0000\u0000\u00e5\u00e8\u0001\u0000\u0000\u0000\u00e6\u00e4\u0001"+ - "\u0000\u0000\u0000\u00e6\u00e7\u0001\u0000\u0000\u0000\u00e7\u00ea\u0001"+ - "\u0000\u0000\u0000\u00e8\u00e6\u0001\u0000\u0000\u0000\u00e9\u00e0\u0001"+ - "\u0000\u0000\u0000\u00e9\u00e1\u0001\u0000\u0000\u0000\u00e9\u00ea\u0001"+ - "\u0000\u0000\u0000\u00ea\u00eb\u0001\u0000\u0000\u0000\u00eb\u00ec\u0005"+ - "2\u0000\u0000\u00ec\u0015\u0001\u0000\u0000\u0000\u00ed\u00ee\u0005\u000e"+ - "\u0000\u0000\u00ee\u00ef\u0003\u0018\f\u0000\u00ef\u0017\u0001\u0000\u0000"+ - "\u0000\u00f0\u00f5\u0003\u001a\r\u0000\u00f1\u00f2\u0005\"\u0000\u0000"+ - "\u00f2\u00f4\u0003\u001a\r\u0000\u00f3\u00f1\u0001\u0000\u0000\u0000\u00f4"+ - "\u00f7\u0001\u0000\u0000\u0000\u00f5\u00f3\u0001\u0000\u0000\u0000\u00f5"+ - "\u00f6\u0001\u0000\u0000\u0000\u00f6\u0019\u0001\u0000\u0000\u0000\u00f7"+ - "\u00f5\u0001\u0000\u0000\u0000\u00f8\u00fe\u0003\n\u0005\u0000\u00f9\u00fa"+ - "\u0003*\u0015\u0000\u00fa\u00fb\u0005!\u0000\u0000\u00fb\u00fc\u0003\n"+ - "\u0005\u0000\u00fc\u00fe\u0001\u0000\u0000\u0000\u00fd\u00f8\u0001\u0000"+ - "\u0000\u0000\u00fd\u00f9\u0001\u0000\u0000\u0000\u00fe\u001b\u0001\u0000"+ - "\u0000\u0000\u00ff\u0100\u0005\u0006\u0000\u0000\u0100\u0105\u0003(\u0014"+ - "\u0000\u0101\u0102\u0005\"\u0000\u0000\u0102\u0104\u0003(\u0014\u0000"+ - "\u0103\u0101\u0001\u0000\u0000\u0000\u0104\u0107\u0001\u0000\u0000\u0000"+ - "\u0105\u0103\u0001\u0000\u0000\u0000\u0105\u0106\u0001\u0000\u0000\u0000"+ - "\u0106\u0109\u0001\u0000\u0000\u0000\u0107\u0105\u0001\u0000\u0000\u0000"+ - "\u0108\u010a\u0003\u001e\u000f\u0000\u0109\u0108\u0001\u0000\u0000\u0000"+ - "\u0109\u010a\u0001\u0000\u0000\u0000\u010a\u001d\u0001\u0000\u0000\u0000"+ - "\u010b\u010c\u0005A\u0000\u0000\u010c\u010d\u0005I\u0000\u0000\u010d\u0112"+ - "\u0003(\u0014\u0000\u010e\u010f\u0005\"\u0000\u0000\u010f\u0111\u0003"+ - "(\u0014\u0000\u0110\u010e\u0001\u0000\u0000\u0000\u0111\u0114\u0001\u0000"+ - "\u0000\u0000\u0112\u0110\u0001\u0000\u0000\u0000\u0112\u0113\u0001\u0000"+ - "\u0000\u0000\u0113\u0115\u0001\u0000\u0000\u0000\u0114\u0112\u0001\u0000"+ - "\u0000\u0000\u0115\u0116\u0005B\u0000\u0000\u0116\u001f\u0001\u0000\u0000"+ - "\u0000\u0117\u0118\u0005\u0004\u0000\u0000\u0118\u0119\u0003\u0018\f\u0000"+ - "\u0119!\u0001\u0000\u0000\u0000\u011a\u011c\u0005\u0011\u0000\u0000\u011b"+ - "\u011d\u0003\u0018\f\u0000\u011c\u011b\u0001\u0000\u0000\u0000\u011c\u011d"+ - "\u0001\u0000\u0000\u0000\u011d\u0120\u0001\u0000\u0000\u0000\u011e\u011f"+ - "\u0005\u001e\u0000\u0000\u011f\u0121\u0003&\u0013\u0000\u0120\u011e\u0001"+ - "\u0000\u0000\u0000\u0120\u0121\u0001\u0000\u0000\u0000\u0121#\u0001\u0000"+ - "\u0000\u0000\u0122\u0123\u0005\b\u0000\u0000\u0123\u0126\u0003\u0018\f"+ - "\u0000\u0124\u0125\u0005\u001e\u0000\u0000\u0125\u0127\u0003&\u0013\u0000"+ - "\u0126\u0124\u0001\u0000\u0000\u0000\u0126\u0127\u0001\u0000\u0000\u0000"+ - "\u0127%\u0001\u0000\u0000\u0000\u0128\u012d\u0003*\u0015\u0000\u0129\u012a"+ - "\u0005\"\u0000\u0000\u012a\u012c\u0003*\u0015\u0000\u012b\u0129\u0001"+ - "\u0000\u0000\u0000\u012c\u012f\u0001\u0000\u0000\u0000\u012d\u012b\u0001"+ - "\u0000\u0000\u0000\u012d\u012e\u0001\u0000\u0000\u0000\u012e\'\u0001\u0000"+ - "\u0000\u0000\u012f\u012d\u0001\u0000\u0000\u0000\u0130\u0131\u0007\u0002"+ - "\u0000\u0000\u0131)\u0001\u0000\u0000\u0000\u0132\u0137\u0003,\u0016\u0000"+ - "\u0133\u0134\u0005$\u0000\u0000\u0134\u0136\u0003,\u0016\u0000\u0135\u0133"+ - "\u0001\u0000\u0000\u0000\u0136\u0139\u0001\u0000\u0000\u0000\u0137\u0135"+ - "\u0001\u0000\u0000\u0000\u0137\u0138\u0001\u0000\u0000\u0000\u0138+\u0001"+ - "\u0000\u0000\u0000\u0139\u0137\u0001\u0000\u0000\u0000\u013a\u013b\u0007"+ - "\u0003\u0000\u0000\u013b-\u0001\u0000\u0000\u0000\u013c\u0167\u0005-\u0000"+ - "\u0000\u013d\u013e\u0003N\'\u0000\u013e\u013f\u0005C\u0000\u0000\u013f"+ - "\u0167\u0001\u0000\u0000\u0000\u0140\u0167\u0003L&\u0000\u0141\u0167\u0003"+ - "N\'\u0000\u0142\u0167\u0003H$\u0000\u0143\u0167\u00050\u0000\u0000\u0144"+ - "\u0167\u0003P(\u0000\u0145\u0146\u0005A\u0000\u0000\u0146\u014b\u0003"+ - "J%\u0000\u0147\u0148\u0005\"\u0000\u0000\u0148\u014a\u0003J%\u0000\u0149"+ - "\u0147\u0001\u0000\u0000\u0000\u014a\u014d\u0001\u0000\u0000\u0000\u014b"+ - "\u0149\u0001\u0000\u0000\u0000\u014b\u014c\u0001\u0000\u0000\u0000\u014c"+ - "\u014e\u0001\u0000\u0000\u0000\u014d\u014b\u0001\u0000\u0000\u0000\u014e"+ - "\u014f\u0005B\u0000\u0000\u014f\u0167\u0001\u0000\u0000\u0000\u0150\u0151"+ - "\u0005A\u0000\u0000\u0151\u0156\u0003H$\u0000\u0152\u0153\u0005\"\u0000"+ - "\u0000\u0153\u0155\u0003H$\u0000\u0154\u0152\u0001\u0000\u0000\u0000\u0155"+ - "\u0158\u0001\u0000\u0000\u0000\u0156\u0154\u0001\u0000\u0000\u0000\u0156"+ - "\u0157\u0001\u0000\u0000\u0000\u0157\u0159\u0001\u0000\u0000\u0000\u0158"+ - "\u0156\u0001\u0000\u0000\u0000\u0159\u015a\u0005B\u0000\u0000\u015a\u0167"+ - "\u0001\u0000\u0000\u0000\u015b\u015c\u0005A\u0000\u0000\u015c\u0161\u0003"+ - "P(\u0000\u015d\u015e\u0005\"\u0000\u0000\u015e\u0160\u0003P(\u0000\u015f"+ - "\u015d\u0001\u0000\u0000\u0000\u0160\u0163\u0001\u0000\u0000\u0000\u0161"+ - "\u015f\u0001\u0000\u0000\u0000\u0161\u0162\u0001\u0000\u0000\u0000\u0162"+ - "\u0164\u0001\u0000\u0000\u0000\u0163\u0161\u0001\u0000\u0000\u0000\u0164"+ - "\u0165\u0005B\u0000\u0000\u0165\u0167\u0001\u0000\u0000\u0000\u0166\u013c"+ - "\u0001\u0000\u0000\u0000\u0166\u013d\u0001\u0000\u0000\u0000\u0166\u0140"+ - "\u0001\u0000\u0000\u0000\u0166\u0141\u0001\u0000\u0000\u0000\u0166\u0142"+ - "\u0001\u0000\u0000\u0000\u0166\u0143\u0001\u0000\u0000\u0000\u0166\u0144"+ - "\u0001\u0000\u0000\u0000\u0166\u0145\u0001\u0000\u0000\u0000\u0166\u0150"+ - "\u0001\u0000\u0000\u0000\u0166\u015b\u0001\u0000\u0000\u0000\u0167/\u0001"+ - "\u0000\u0000\u0000\u0168\u0169\u0005\n\u0000\u0000\u0169\u016a\u0005\u001c"+ - "\u0000\u0000\u016a1\u0001\u0000\u0000\u0000\u016b\u016c\u0005\u0010\u0000"+ - "\u0000\u016c\u0171\u00034\u001a\u0000\u016d\u016e\u0005\"\u0000\u0000"+ - "\u016e\u0170\u00034\u001a\u0000\u016f\u016d\u0001\u0000\u0000\u0000\u0170"+ - "\u0173\u0001\u0000\u0000\u0000\u0171\u016f\u0001\u0000\u0000\u0000\u0171"+ - "\u0172\u0001\u0000\u0000\u0000\u01723\u0001\u0000\u0000\u0000\u0173\u0171"+ - "\u0001\u0000\u0000\u0000\u0174\u0176\u0003\n\u0005\u0000\u0175\u0177\u0007"+ - "\u0004\u0000\u0000\u0176\u0175\u0001\u0000\u0000\u0000\u0176\u0177\u0001"+ - "\u0000\u0000\u0000\u0177\u017a\u0001\u0000\u0000\u0000\u0178\u0179\u0005"+ - ".\u0000\u0000\u0179\u017b\u0007\u0005\u0000\u0000\u017a\u0178\u0001\u0000"+ - "\u0000\u0000\u017a\u017b\u0001\u0000\u0000\u0000\u017b5\u0001\u0000\u0000"+ - "\u0000\u017c\u017d\u0005\t\u0000\u0000\u017d\u0182\u0003(\u0014\u0000"+ - "\u017e\u017f\u0005\"\u0000\u0000\u017f\u0181\u0003(\u0014\u0000\u0180"+ - "\u017e\u0001\u0000\u0000\u0000\u0181\u0184\u0001\u0000\u0000\u0000\u0182"+ - "\u0180\u0001\u0000\u0000\u0000\u0182\u0183\u0001\u0000\u0000\u0000\u0183"+ - "\u018f\u0001\u0000\u0000\u0000\u0184\u0182\u0001\u0000\u0000\u0000\u0185"+ - "\u0186\u0005\f\u0000\u0000\u0186\u018b\u0003(\u0014\u0000\u0187\u0188"+ - "\u0005\"\u0000\u0000\u0188\u018a\u0003(\u0014\u0000\u0189\u0187\u0001"+ - "\u0000\u0000\u0000\u018a\u018d\u0001\u0000\u0000\u0000\u018b\u0189\u0001"+ - "\u0000\u0000\u0000\u018b\u018c\u0001\u0000\u0000\u0000\u018c\u018f\u0001"+ - "\u0000\u0000\u0000\u018d\u018b\u0001\u0000\u0000\u0000\u018e\u017c\u0001"+ - "\u0000\u0000\u0000\u018e\u0185\u0001\u0000\u0000\u0000\u018f7\u0001\u0000"+ - "\u0000\u0000\u0190\u0191\u0005\u0002\u0000\u0000\u0191\u0196\u0003(\u0014"+ - "\u0000\u0192\u0193\u0005\"\u0000\u0000\u0193\u0195\u0003(\u0014\u0000"+ - "\u0194\u0192\u0001\u0000\u0000\u0000\u0195\u0198\u0001\u0000\u0000\u0000"+ - "\u0196\u0194\u0001\u0000\u0000\u0000\u0196\u0197\u0001\u0000\u0000\u0000"+ - "\u01979\u0001\u0000\u0000\u0000\u0198\u0196\u0001\u0000\u0000\u0000\u0199"+ - "\u019a\u0005\r\u0000\u0000\u019a\u019f\u0003<\u001e\u0000\u019b\u019c"+ - "\u0005\"\u0000\u0000\u019c\u019e\u0003<\u001e\u0000\u019d\u019b\u0001"+ - "\u0000\u0000\u0000\u019e\u01a1\u0001\u0000\u0000\u0000\u019f\u019d\u0001"+ - "\u0000\u0000\u0000\u019f\u01a0\u0001\u0000\u0000\u0000\u01a0;\u0001\u0000"+ - "\u0000\u0000\u01a1\u019f\u0001\u0000\u0000\u0000\u01a2\u01a3\u0003(\u0014"+ - "\u0000\u01a3\u01a4\u0005H\u0000\u0000\u01a4\u01a5\u0003(\u0014\u0000\u01a5"+ - "=\u0001\u0000\u0000\u0000\u01a6\u01a7\u0005\u0001\u0000\u0000\u01a7\u01a8"+ - "\u0003\u0012\t\u0000\u01a8\u01aa\u0003P(\u0000\u01a9\u01ab\u0003D\"\u0000"+ - "\u01aa\u01a9\u0001\u0000\u0000\u0000\u01aa\u01ab\u0001\u0000\u0000\u0000"+ - "\u01ab?\u0001\u0000\u0000\u0000\u01ac\u01ad\u0005\u0007\u0000\u0000\u01ad"+ - "\u01ae\u0003\u0012\t\u0000\u01ae\u01af\u0003P(\u0000\u01afA\u0001\u0000"+ - "\u0000\u0000\u01b0\u01b1\u0005\u000b\u0000\u0000\u01b1\u01b2\u0003(\u0014"+ - "\u0000\u01b2C\u0001\u0000\u0000\u0000\u01b3\u01b8\u0003F#\u0000\u01b4"+ - "\u01b5\u0005\"\u0000\u0000\u01b5\u01b7\u0003F#\u0000\u01b6\u01b4\u0001"+ - "\u0000\u0000\u0000\u01b7\u01ba\u0001\u0000\u0000\u0000\u01b8\u01b6\u0001"+ - "\u0000\u0000\u0000\u01b8\u01b9\u0001\u0000\u0000\u0000\u01b9E\u0001\u0000"+ - "\u0000\u0000\u01ba\u01b8\u0001\u0000\u0000\u0000\u01bb\u01bc\u0003,\u0016"+ - "\u0000\u01bc\u01bd\u0005!\u0000\u0000\u01bd\u01be\u0003.\u0017\u0000\u01be"+ - "G\u0001\u0000\u0000\u0000\u01bf\u01c0\u0007\u0006\u0000\u0000\u01c0I\u0001"+ - "\u0000\u0000\u0000\u01c1\u01c4\u0003L&\u0000\u01c2\u01c4\u0003N\'\u0000"+ - "\u01c3\u01c1\u0001\u0000\u0000\u0000\u01c3\u01c2\u0001\u0000\u0000\u0000"+ - "\u01c4K\u0001\u0000\u0000\u0000\u01c5\u01c7\u0007\u0000\u0000\u0000\u01c6"+ - "\u01c5\u0001\u0000\u0000\u0000\u01c6\u01c7\u0001\u0000\u0000\u0000\u01c7"+ - "\u01c8\u0001\u0000\u0000\u0000\u01c8\u01c9\u0005\u001d\u0000\u0000\u01c9"+ - "M\u0001\u0000\u0000\u0000\u01ca\u01cc\u0007\u0000\u0000\u0000\u01cb\u01ca"+ - "\u0001\u0000\u0000\u0000\u01cb\u01cc\u0001\u0000\u0000\u0000\u01cc\u01cd"+ - "\u0001\u0000\u0000\u0000\u01cd\u01ce\u0005\u001c\u0000\u0000\u01ceO\u0001"+ - "\u0000\u0000\u0000\u01cf\u01d0\u0005\u001b\u0000\u0000\u01d0Q\u0001\u0000"+ - "\u0000\u0000\u01d1\u01d2\u0007\u0007\u0000\u0000\u01d2S\u0001\u0000\u0000"+ - "\u0000\u01d3\u01d4\u0005\u0005\u0000\u0000\u01d4\u01d5\u0003V+\u0000\u01d5"+ - "U\u0001\u0000\u0000\u0000\u01d6\u01d7\u0005A\u0000\u0000\u01d7\u01d8\u0003"+ - "\u0002\u0001\u0000\u01d8\u01d9\u0005B\u0000\u0000\u01d9W\u0001\u0000\u0000"+ - "\u0000\u01da\u01db\u0005\u000f\u0000\u0000\u01db\u01df\u00054\u0000\u0000"+ - "\u01dc\u01dd\u0005\u000f\u0000\u0000\u01dd\u01df\u00055\u0000\u0000\u01de"+ - "\u01da\u0001\u0000\u0000\u0000\u01de\u01dc\u0001\u0000\u0000\u0000\u01df"+ - "Y\u0001\u0000\u0000\u0000\u01e0\u01e1\u0005\u0003\u0000\u0000\u01e1\u01e4"+ - "\u0003(\u0014\u0000\u01e2\u01e3\u0005J\u0000\u0000\u01e3\u01e5\u0003("+ - "\u0014\u0000\u01e4\u01e2\u0001\u0000\u0000\u0000\u01e4\u01e5\u0001\u0000"+ - "\u0000\u0000\u01e5\u01ef\u0001\u0000\u0000\u0000\u01e6\u01e7\u0005K\u0000"+ - "\u0000\u01e7\u01ec\u0003\\.\u0000\u01e8\u01e9\u0005\"\u0000\u0000\u01e9"+ - "\u01eb\u0003\\.\u0000\u01ea\u01e8\u0001\u0000\u0000\u0000\u01eb\u01ee"+ - "\u0001\u0000\u0000\u0000\u01ec\u01ea\u0001\u0000\u0000\u0000\u01ec\u01ed"+ - "\u0001\u0000\u0000\u0000\u01ed\u01f0\u0001\u0000\u0000\u0000\u01ee\u01ec"+ - "\u0001\u0000\u0000\u0000\u01ef\u01e6\u0001\u0000\u0000\u0000\u01ef\u01f0"+ - "\u0001\u0000\u0000\u0000\u01f0[\u0001\u0000\u0000\u0000\u01f1\u01f2\u0003"+ - "(\u0014\u0000\u01f2\u01f3\u0005!\u0000\u0000\u01f3\u01f5\u0001\u0000\u0000"+ - "\u0000\u01f4\u01f1\u0001\u0000\u0000\u0000\u01f4\u01f5\u0001\u0000\u0000"+ - "\u0000\u01f5\u01f6\u0001\u0000\u0000\u0000\u01f6\u01f7\u0003(\u0014\u0000"+ - "\u01f7]\u0001\u0000\u0000\u00003ip\u007f\u008b\u0094\u009c\u00a0\u00a8"+ - "\u00aa\u00af\u00b6\u00bb\u00c2\u00c8\u00d0\u00d2\u00dc\u00e6\u00e9\u00f5"+ - "\u00fd\u0105\u0109\u0112\u011c\u0120\u0126\u012d\u0137\u014b\u0156\u0161"+ - "\u0166\u0171\u0176\u017a\u0182\u018b\u018e\u0196\u019f\u01aa\u01b8\u01c3"+ - "\u01c6\u01cb\u01de\u01e4\u01ec\u01ef\u01f4"; + "\u00ad\b\u0005\n\u0005\f\u0005\u00b0\t\u0005\u0001\u0006\u0001\u0006\u0003"+ + "\u0006\u00b4\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001"+ + "\u0006\u0003\u0006\u00bb\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003"+ + "\u0006\u00c0\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+ + "\u0007\u0003\u0007\u00c7\b\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0003"+ + "\b\u00cd\b\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0005\b\u00d5"+ + "\b\b\n\b\f\b\u00d8\t\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t"+ + "\u0001\t\u0003\t\u00e1\b\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+ + "\n\u0005\n\u00e9\b\n\n\n\f\n\u00ec\t\n\u0003\n\u00ee\b\n\u0001\n\u0001"+ + "\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0005\f"+ + "\u00f8\b\f\n\f\f\f\u00fb\t\f\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0003"+ + "\r\u0102\b\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0005\u000e"+ + "\u0108\b\u000e\n\u000e\f\u000e\u010b\t\u000e\u0001\u000e\u0003\u000e\u010e"+ + "\b\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0005"+ + "\u000f\u0115\b\u000f\n\u000f\f\u000f\u0118\t\u000f\u0001\u000f\u0001\u000f"+ + "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0003\u0011"+ + "\u0121\b\u0011\u0001\u0011\u0001\u0011\u0003\u0011\u0125\b\u0011\u0001"+ + "\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0003\u0012\u012b\b\u0012\u0001"+ + "\u0013\u0001\u0013\u0001\u0013\u0005\u0013\u0130\b\u0013\n\u0013\f\u0013"+ + "\u0133\t\u0013\u0001\u0014\u0001\u0014\u0001\u0015\u0001\u0015\u0001\u0015"+ + "\u0005\u0015\u013a\b\u0015\n\u0015\f\u0015\u013d\t\u0015\u0001\u0016\u0001"+ + "\u0016\u0001\u0016\u0005\u0016\u0142\b\u0016\n\u0016\f\u0016\u0145\t\u0016"+ + "\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0019\u0001\u0019"+ + "\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019"+ + "\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0005\u0019"+ + "\u0158\b\u0019\n\u0019\f\u0019\u015b\t\u0019\u0001\u0019\u0001\u0019\u0001"+ + "\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0005\u0019\u0163\b\u0019\n"+ + "\u0019\f\u0019\u0166\t\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0001"+ + "\u0019\u0001\u0019\u0001\u0019\u0005\u0019\u016e\b\u0019\n\u0019\f\u0019"+ + "\u0171\t\u0019\u0001\u0019\u0001\u0019\u0003\u0019\u0175\b\u0019\u0001"+ + "\u001a\u0001\u001a\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0001"+ + "\u001b\u0005\u001b\u017e\b\u001b\n\u001b\f\u001b\u0181\t\u001b\u0001\u001c"+ + "\u0001\u001c\u0003\u001c\u0185\b\u001c\u0001\u001c\u0001\u001c\u0003\u001c"+ + "\u0189\b\u001c\u0001\u001d\u0001\u001d\u0001\u001d\u0001\u001d\u0005\u001d"+ + "\u018f\b\u001d\n\u001d\f\u001d\u0192\t\u001d\u0001\u001d\u0001\u001d\u0001"+ + "\u001d\u0001\u001d\u0005\u001d\u0198\b\u001d\n\u001d\f\u001d\u019b\t\u001d"+ + "\u0003\u001d\u019d\b\u001d\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001e"+ + "\u0005\u001e\u01a3\b\u001e\n\u001e\f\u001e\u01a6\t\u001e\u0001\u001f\u0001"+ + "\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u01ac\b\u001f\n\u001f\f\u001f"+ + "\u01af\t\u001f\u0001 \u0001 \u0001 \u0001 \u0001!\u0001!\u0001!\u0001"+ + "!\u0003!\u01b9\b!\u0001\"\u0001\"\u0001\"\u0001\"\u0001#\u0001#\u0001"+ + "#\u0001$\u0001$\u0001$\u0005$\u01c5\b$\n$\f$\u01c8\t$\u0001%\u0001%\u0001"+ + "%\u0001%\u0001&\u0001&\u0001\'\u0001\'\u0003\'\u01d2\b\'\u0001(\u0003"+ + "(\u01d5\b(\u0001(\u0001(\u0001)\u0003)\u01da\b)\u0001)\u0001)\u0001*\u0001"+ + "*\u0001+\u0001+\u0001,\u0001,\u0001,\u0001-\u0001-\u0001-\u0001-\u0001"+ + ".\u0001.\u0001.\u0001.\u0003.\u01ed\b.\u0001/\u0001/\u0001/\u0001/\u0003"+ + "/\u01f3\b/\u0001/\u0001/\u0001/\u0001/\u0005/\u01f9\b/\n/\f/\u01fc\t/"+ + "\u0003/\u01fe\b/\u00010\u00010\u00010\u00030\u0203\b0\u00010\u00010\u0001"+ + "0\u0000\u0003\u0002\n\u00101\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010"+ + "\u0012\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPR"+ + "TVXZ\\^`\u0000\t\u0001\u0000:;\u0001\u0000<>\u0002\u0000BBGG\u0001\u0000"+ + "AB\u0002\u0000BBKK\u0002\u0000 ##\u0001\u0000&\'\u0002\u0000%%33\u0001"+ + "\u000049\u0224\u0000b\u0001\u0000\u0000\u0000\u0002e\u0001\u0000\u0000"+ + "\u0000\u0004t\u0001\u0000\u0000\u0000\u0006\u0083\u0001\u0000\u0000\u0000"+ + "\b\u0085\u0001\u0000\u0000\u0000\n\u00a4\u0001\u0000\u0000\u0000\f\u00bf"+ + "\u0001\u0000\u0000\u0000\u000e\u00c6\u0001\u0000\u0000\u0000\u0010\u00cc"+ + "\u0001\u0000\u0000\u0000\u0012\u00e0\u0001\u0000\u0000\u0000\u0014\u00e2"+ + "\u0001\u0000\u0000\u0000\u0016\u00f1\u0001\u0000\u0000\u0000\u0018\u00f4"+ + "\u0001\u0000\u0000\u0000\u001a\u0101\u0001\u0000\u0000\u0000\u001c\u0103"+ + "\u0001\u0000\u0000\u0000\u001e\u010f\u0001\u0000\u0000\u0000 \u011b\u0001"+ + "\u0000\u0000\u0000\"\u011e\u0001\u0000\u0000\u0000$\u0126\u0001\u0000"+ + "\u0000\u0000&\u012c\u0001\u0000\u0000\u0000(\u0134\u0001\u0000\u0000\u0000"+ + "*\u0136\u0001\u0000\u0000\u0000,\u013e\u0001\u0000\u0000\u0000.\u0146"+ + "\u0001\u0000\u0000\u00000\u0148\u0001\u0000\u0000\u00002\u0174\u0001\u0000"+ + "\u0000\u00004\u0176\u0001\u0000\u0000\u00006\u0179\u0001\u0000\u0000\u0000"+ + "8\u0182\u0001\u0000\u0000\u0000:\u019c\u0001\u0000\u0000\u0000<\u019e"+ + "\u0001\u0000\u0000\u0000>\u01a7\u0001\u0000\u0000\u0000@\u01b0\u0001\u0000"+ + "\u0000\u0000B\u01b4\u0001\u0000\u0000\u0000D\u01ba\u0001\u0000\u0000\u0000"+ + "F\u01be\u0001\u0000\u0000\u0000H\u01c1\u0001\u0000\u0000\u0000J\u01c9"+ + "\u0001\u0000\u0000\u0000L\u01cd\u0001\u0000\u0000\u0000N\u01d1\u0001\u0000"+ + "\u0000\u0000P\u01d4\u0001\u0000\u0000\u0000R\u01d9\u0001\u0000\u0000\u0000"+ + "T\u01dd\u0001\u0000\u0000\u0000V\u01df\u0001\u0000\u0000\u0000X\u01e1"+ + "\u0001\u0000\u0000\u0000Z\u01e4\u0001\u0000\u0000\u0000\\\u01ec\u0001"+ + "\u0000\u0000\u0000^\u01ee\u0001\u0000\u0000\u0000`\u0202\u0001\u0000\u0000"+ + "\u0000bc\u0003\u0002\u0001\u0000cd\u0005\u0000\u0000\u0001d\u0001\u0001"+ + "\u0000\u0000\u0000ef\u0006\u0001\uffff\uffff\u0000fg\u0003\u0004\u0002"+ + "\u0000gm\u0001\u0000\u0000\u0000hi\n\u0001\u0000\u0000ij\u0005\u001a\u0000"+ + "\u0000jl\u0003\u0006\u0003\u0000kh\u0001\u0000\u0000\u0000lo\u0001\u0000"+ + "\u0000\u0000mk\u0001\u0000\u0000\u0000mn\u0001\u0000\u0000\u0000n\u0003"+ + "\u0001\u0000\u0000\u0000om\u0001\u0000\u0000\u0000pu\u0003X,\u0000qu\u0003"+ + "\u001c\u000e\u0000ru\u0003\u0016\u000b\u0000su\u0003\\.\u0000tp\u0001"+ + "\u0000\u0000\u0000tq\u0001\u0000\u0000\u0000tr\u0001\u0000\u0000\u0000"+ + "ts\u0001\u0000\u0000\u0000u\u0005\u0001\u0000\u0000\u0000v\u0084\u0003"+ + " \u0010\u0000w\u0084\u0003$\u0012\u0000x\u0084\u00034\u001a\u0000y\u0084"+ + "\u0003:\u001d\u0000z\u0084\u00036\u001b\u0000{\u0084\u0003\"\u0011\u0000"+ + "|\u0084\u0003\b\u0004\u0000}\u0084\u0003<\u001e\u0000~\u0084\u0003>\u001f"+ + "\u0000\u007f\u0084\u0003B!\u0000\u0080\u0084\u0003D\"\u0000\u0081\u0084"+ + "\u0003^/\u0000\u0082\u0084\u0003F#\u0000\u0083v\u0001\u0000\u0000\u0000"+ + "\u0083w\u0001\u0000\u0000\u0000\u0083x\u0001\u0000\u0000\u0000\u0083y"+ + "\u0001\u0000\u0000\u0000\u0083z\u0001\u0000\u0000\u0000\u0083{\u0001\u0000"+ + "\u0000\u0000\u0083|\u0001\u0000\u0000\u0000\u0083}\u0001\u0000\u0000\u0000"+ + "\u0083~\u0001\u0000\u0000\u0000\u0083\u007f\u0001\u0000\u0000\u0000\u0083"+ + "\u0080\u0001\u0000\u0000\u0000\u0083\u0081\u0001\u0000\u0000\u0000\u0083"+ + "\u0082\u0001\u0000\u0000\u0000\u0084\u0007\u0001\u0000\u0000\u0000\u0085"+ + "\u0086\u0005\u0012\u0000\u0000\u0086\u0087\u0003\n\u0005\u0000\u0087\t"+ + "\u0001\u0000\u0000\u0000\u0088\u0089\u0006\u0005\uffff\uffff\u0000\u0089"+ + "\u008a\u0005,\u0000\u0000\u008a\u00a5\u0003\n\u0005\u0007\u008b\u00a5"+ + "\u0003\u000e\u0007\u0000\u008c\u00a5\u0003\f\u0006\u0000\u008d\u008f\u0003"+ + "\u000e\u0007\u0000\u008e\u0090\u0005,\u0000\u0000\u008f\u008e\u0001\u0000"+ + "\u0000\u0000\u008f\u0090\u0001\u0000\u0000\u0000\u0090\u0091\u0001\u0000"+ + "\u0000\u0000\u0091\u0092\u0005)\u0000\u0000\u0092\u0093\u0005(\u0000\u0000"+ + "\u0093\u0098\u0003\u000e\u0007\u0000\u0094\u0095\u0005\"\u0000\u0000\u0095"+ + "\u0097\u0003\u000e\u0007\u0000\u0096\u0094\u0001\u0000\u0000\u0000\u0097"+ + "\u009a\u0001\u0000\u0000\u0000\u0098\u0096\u0001\u0000\u0000\u0000\u0098"+ + "\u0099\u0001\u0000\u0000\u0000\u0099\u009b\u0001\u0000\u0000\u0000\u009a"+ + "\u0098\u0001\u0000\u0000\u0000\u009b\u009c\u00052\u0000\u0000\u009c\u00a5"+ + "\u0001\u0000\u0000\u0000\u009d\u009e\u0003\u000e\u0007\u0000\u009e\u00a0"+ + "\u0005*\u0000\u0000\u009f\u00a1\u0005,\u0000\u0000\u00a0\u009f\u0001\u0000"+ + "\u0000\u0000\u00a0\u00a1\u0001\u0000\u0000\u0000\u00a1\u00a2\u0001\u0000"+ + "\u0000\u0000\u00a2\u00a3\u0005-\u0000\u0000\u00a3\u00a5\u0001\u0000\u0000"+ + "\u0000\u00a4\u0088\u0001\u0000\u0000\u0000\u00a4\u008b\u0001\u0000\u0000"+ + "\u0000\u00a4\u008c\u0001\u0000\u0000\u0000\u00a4\u008d\u0001\u0000\u0000"+ + "\u0000\u00a4\u009d\u0001\u0000\u0000\u0000\u00a5\u00ae\u0001\u0000\u0000"+ + "\u0000\u00a6\u00a7\n\u0004\u0000\u0000\u00a7\u00a8\u0005\u001f\u0000\u0000"+ + "\u00a8\u00ad\u0003\n\u0005\u0005\u00a9\u00aa\n\u0003\u0000\u0000\u00aa"+ + "\u00ab\u0005/\u0000\u0000\u00ab\u00ad\u0003\n\u0005\u0004\u00ac\u00a6"+ + "\u0001\u0000\u0000\u0000\u00ac\u00a9\u0001\u0000\u0000\u0000\u00ad\u00b0"+ + "\u0001\u0000\u0000\u0000\u00ae\u00ac\u0001\u0000\u0000\u0000\u00ae\u00af"+ + "\u0001\u0000\u0000\u0000\u00af\u000b\u0001\u0000\u0000\u0000\u00b0\u00ae"+ + "\u0001\u0000\u0000\u0000\u00b1\u00b3\u0003\u000e\u0007\u0000\u00b2\u00b4"+ + "\u0005,\u0000\u0000\u00b3\u00b2\u0001\u0000\u0000\u0000\u00b3\u00b4\u0001"+ + "\u0000\u0000\u0000\u00b4\u00b5\u0001\u0000\u0000\u0000\u00b5\u00b6\u0005"+ + "+\u0000\u0000\u00b6\u00b7\u0003T*\u0000\u00b7\u00c0\u0001\u0000\u0000"+ + "\u0000\u00b8\u00ba\u0003\u000e\u0007\u0000\u00b9\u00bb\u0005,\u0000\u0000"+ + "\u00ba\u00b9\u0001\u0000\u0000\u0000\u00ba\u00bb\u0001\u0000\u0000\u0000"+ + "\u00bb\u00bc\u0001\u0000\u0000\u0000\u00bc\u00bd\u00051\u0000\u0000\u00bd"+ + "\u00be\u0003T*\u0000\u00be\u00c0\u0001\u0000\u0000\u0000\u00bf\u00b1\u0001"+ + "\u0000\u0000\u0000\u00bf\u00b8\u0001\u0000\u0000\u0000\u00c0\r\u0001\u0000"+ + "\u0000\u0000\u00c1\u00c7\u0003\u0010\b\u0000\u00c2\u00c3\u0003\u0010\b"+ + "\u0000\u00c3\u00c4\u0003V+\u0000\u00c4\u00c5\u0003\u0010\b\u0000\u00c5"+ + "\u00c7\u0001\u0000\u0000\u0000\u00c6\u00c1\u0001\u0000\u0000\u0000\u00c6"+ + "\u00c2\u0001\u0000\u0000\u0000\u00c7\u000f\u0001\u0000\u0000\u0000\u00c8"+ + "\u00c9\u0006\b\uffff\uffff\u0000\u00c9\u00cd\u0003\u0012\t\u0000\u00ca"+ + "\u00cb\u0007\u0000\u0000\u0000\u00cb\u00cd\u0003\u0010\b\u0003\u00cc\u00c8"+ + "\u0001\u0000\u0000\u0000\u00cc\u00ca\u0001\u0000\u0000\u0000\u00cd\u00d6"+ + "\u0001\u0000\u0000\u0000\u00ce\u00cf\n\u0002\u0000\u0000\u00cf\u00d0\u0007"+ + "\u0001\u0000\u0000\u00d0\u00d5\u0003\u0010\b\u0003\u00d1\u00d2\n\u0001"+ + "\u0000\u0000\u00d2\u00d3\u0007\u0000\u0000\u0000\u00d3\u00d5\u0003\u0010"+ + "\b\u0002\u00d4\u00ce\u0001\u0000\u0000\u0000\u00d4\u00d1\u0001\u0000\u0000"+ + "\u0000\u00d5\u00d8\u0001\u0000\u0000\u0000\u00d6\u00d4\u0001\u0000\u0000"+ + "\u0000\u00d6\u00d7\u0001\u0000\u0000\u0000\u00d7\u0011\u0001\u0000\u0000"+ + "\u0000\u00d8\u00d6\u0001\u0000\u0000\u0000\u00d9\u00e1\u00032\u0019\u0000"+ + "\u00da\u00e1\u0003*\u0015\u0000\u00db\u00e1\u0003\u0014\n\u0000\u00dc"+ + "\u00dd\u0005(\u0000\u0000\u00dd\u00de\u0003\n\u0005\u0000\u00de\u00df"+ + "\u00052\u0000\u0000\u00df\u00e1\u0001\u0000\u0000\u0000\u00e0\u00d9\u0001"+ + "\u0000\u0000\u0000\u00e0\u00da\u0001\u0000\u0000\u0000\u00e0\u00db\u0001"+ + "\u0000\u0000\u0000\u00e0\u00dc\u0001\u0000\u0000\u0000\u00e1\u0013\u0001"+ + "\u0000\u0000\u0000\u00e2\u00e3\u0003.\u0017\u0000\u00e3\u00ed\u0005(\u0000"+ + "\u0000\u00e4\u00ee\u0005<\u0000\u0000\u00e5\u00ea\u0003\n\u0005\u0000"+ + "\u00e6\u00e7\u0005\"\u0000\u0000\u00e7\u00e9\u0003\n\u0005\u0000\u00e8"+ + "\u00e6\u0001\u0000\u0000\u0000\u00e9\u00ec\u0001\u0000\u0000\u0000\u00ea"+ + "\u00e8\u0001\u0000\u0000\u0000\u00ea\u00eb\u0001\u0000\u0000\u0000\u00eb"+ + "\u00ee\u0001\u0000\u0000\u0000\u00ec\u00ea\u0001\u0000\u0000\u0000\u00ed"+ + "\u00e4\u0001\u0000\u0000\u0000\u00ed\u00e5\u0001\u0000\u0000\u0000\u00ed"+ + "\u00ee\u0001\u0000\u0000\u0000\u00ee\u00ef\u0001\u0000\u0000\u0000\u00ef"+ + "\u00f0\u00052\u0000\u0000\u00f0\u0015\u0001\u0000\u0000\u0000\u00f1\u00f2"+ + "\u0005\u000e\u0000\u0000\u00f2\u00f3\u0003\u0018\f\u0000\u00f3\u0017\u0001"+ + "\u0000\u0000\u0000\u00f4\u00f9\u0003\u001a\r\u0000\u00f5\u00f6\u0005\""+ + "\u0000\u0000\u00f6\u00f8\u0003\u001a\r\u0000\u00f7\u00f5\u0001\u0000\u0000"+ + "\u0000\u00f8\u00fb\u0001\u0000\u0000\u0000\u00f9\u00f7\u0001\u0000\u0000"+ + "\u0000\u00f9\u00fa\u0001\u0000\u0000\u0000\u00fa\u0019\u0001\u0000\u0000"+ + "\u0000\u00fb\u00f9\u0001\u0000\u0000\u0000\u00fc\u0102\u0003\n\u0005\u0000"+ + "\u00fd\u00fe\u0003*\u0015\u0000\u00fe\u00ff\u0005!\u0000\u0000\u00ff\u0100"+ + "\u0003\n\u0005\u0000\u0100\u0102\u0001\u0000\u0000\u0000\u0101\u00fc\u0001"+ + "\u0000\u0000\u0000\u0101\u00fd\u0001\u0000\u0000\u0000\u0102\u001b\u0001"+ + "\u0000\u0000\u0000\u0103\u0104\u0005\u0006\u0000\u0000\u0104\u0109\u0003"+ + "(\u0014\u0000\u0105\u0106\u0005\"\u0000\u0000\u0106\u0108\u0003(\u0014"+ + "\u0000\u0107\u0105\u0001\u0000\u0000\u0000\u0108\u010b\u0001\u0000\u0000"+ + "\u0000\u0109\u0107\u0001\u0000\u0000\u0000\u0109\u010a\u0001\u0000\u0000"+ + "\u0000\u010a\u010d\u0001\u0000\u0000\u0000\u010b\u0109\u0001\u0000\u0000"+ + "\u0000\u010c\u010e\u0003\u001e\u000f\u0000\u010d\u010c\u0001\u0000\u0000"+ + "\u0000\u010d\u010e\u0001\u0000\u0000\u0000\u010e\u001d\u0001\u0000\u0000"+ + "\u0000\u010f\u0110\u0005?\u0000\u0000\u0110\u0111\u0005F\u0000\u0000\u0111"+ + "\u0116\u0003(\u0014\u0000\u0112\u0113\u0005\"\u0000\u0000\u0113\u0115"+ + "\u0003(\u0014\u0000\u0114\u0112\u0001\u0000\u0000\u0000\u0115\u0118\u0001"+ + "\u0000\u0000\u0000\u0116\u0114\u0001\u0000\u0000\u0000\u0116\u0117\u0001"+ + "\u0000\u0000\u0000\u0117\u0119\u0001\u0000\u0000\u0000\u0118\u0116\u0001"+ + "\u0000\u0000\u0000\u0119\u011a\u0005@\u0000\u0000\u011a\u001f\u0001\u0000"+ + "\u0000\u0000\u011b\u011c\u0005\u0004\u0000\u0000\u011c\u011d\u0003\u0018"+ + "\f\u0000\u011d!\u0001\u0000\u0000\u0000\u011e\u0120\u0005\u0011\u0000"+ + "\u0000\u011f\u0121\u0003\u0018\f\u0000\u0120\u011f\u0001\u0000\u0000\u0000"+ + "\u0120\u0121\u0001\u0000\u0000\u0000\u0121\u0124\u0001\u0000\u0000\u0000"+ + "\u0122\u0123\u0005\u001e\u0000\u0000\u0123\u0125\u0003&\u0013\u0000\u0124"+ + "\u0122\u0001\u0000\u0000\u0000\u0124\u0125\u0001\u0000\u0000\u0000\u0125"+ + "#\u0001\u0000\u0000\u0000\u0126\u0127\u0005\b\u0000\u0000\u0127\u012a"+ + "\u0003\u0018\f\u0000\u0128\u0129\u0005\u001e\u0000\u0000\u0129\u012b\u0003"+ + "&\u0013\u0000\u012a\u0128\u0001\u0000\u0000\u0000\u012a\u012b\u0001\u0000"+ + "\u0000\u0000\u012b%\u0001\u0000\u0000\u0000\u012c\u0131\u0003*\u0015\u0000"+ + "\u012d\u012e\u0005\"\u0000\u0000\u012e\u0130\u0003*\u0015\u0000\u012f"+ + "\u012d\u0001\u0000\u0000\u0000\u0130\u0133\u0001\u0000\u0000\u0000\u0131"+ + "\u012f\u0001\u0000\u0000\u0000\u0131\u0132\u0001\u0000\u0000\u0000\u0132"+ + "\'\u0001\u0000\u0000\u0000\u0133\u0131\u0001\u0000\u0000\u0000\u0134\u0135"+ + "\u0007\u0002\u0000\u0000\u0135)\u0001\u0000\u0000\u0000\u0136\u013b\u0003"+ + ".\u0017\u0000\u0137\u0138\u0005$\u0000\u0000\u0138\u013a\u0003.\u0017"+ + "\u0000\u0139\u0137\u0001\u0000\u0000\u0000\u013a\u013d\u0001\u0000\u0000"+ + "\u0000\u013b\u0139\u0001\u0000\u0000\u0000\u013b\u013c\u0001\u0000\u0000"+ + "\u0000\u013c+\u0001\u0000\u0000\u0000\u013d\u013b\u0001\u0000\u0000\u0000"+ + "\u013e\u0143\u00030\u0018\u0000\u013f\u0140\u0005$\u0000\u0000\u0140\u0142"+ + "\u00030\u0018\u0000\u0141\u013f\u0001\u0000\u0000\u0000\u0142\u0145\u0001"+ + "\u0000\u0000\u0000\u0143\u0141\u0001\u0000\u0000\u0000\u0143\u0144\u0001"+ + "\u0000\u0000\u0000\u0144-\u0001\u0000\u0000\u0000\u0145\u0143\u0001\u0000"+ + "\u0000\u0000\u0146\u0147\u0007\u0003\u0000\u0000\u0147/\u0001\u0000\u0000"+ + "\u0000\u0148\u0149\u0007\u0004\u0000\u0000\u01491\u0001\u0000\u0000\u0000"+ + "\u014a\u0175\u0005-\u0000\u0000\u014b\u014c\u0003R)\u0000\u014c\u014d"+ + "\u0005A\u0000\u0000\u014d\u0175\u0001\u0000\u0000\u0000\u014e\u0175\u0003"+ + "P(\u0000\u014f\u0175\u0003R)\u0000\u0150\u0175\u0003L&\u0000\u0151\u0175"+ + "\u00050\u0000\u0000\u0152\u0175\u0003T*\u0000\u0153\u0154\u0005?\u0000"+ + "\u0000\u0154\u0159\u0003N\'\u0000\u0155\u0156\u0005\"\u0000\u0000\u0156"+ + "\u0158\u0003N\'\u0000\u0157\u0155\u0001\u0000\u0000\u0000\u0158\u015b"+ + "\u0001\u0000\u0000\u0000\u0159\u0157\u0001\u0000\u0000\u0000\u0159\u015a"+ + "\u0001\u0000\u0000\u0000\u015a\u015c\u0001\u0000\u0000\u0000\u015b\u0159"+ + "\u0001\u0000\u0000\u0000\u015c\u015d\u0005@\u0000\u0000\u015d\u0175\u0001"+ + "\u0000\u0000\u0000\u015e\u015f\u0005?\u0000\u0000\u015f\u0164\u0003L&"+ + "\u0000\u0160\u0161\u0005\"\u0000\u0000\u0161\u0163\u0003L&\u0000\u0162"+ + "\u0160\u0001\u0000\u0000\u0000\u0163\u0166\u0001\u0000\u0000\u0000\u0164"+ + "\u0162\u0001\u0000\u0000\u0000\u0164\u0165\u0001\u0000\u0000\u0000\u0165"+ + "\u0167\u0001\u0000\u0000\u0000\u0166\u0164\u0001\u0000\u0000\u0000\u0167"+ + "\u0168\u0005@\u0000\u0000\u0168\u0175\u0001\u0000\u0000\u0000\u0169\u016a"+ + "\u0005?\u0000\u0000\u016a\u016f\u0003T*\u0000\u016b\u016c\u0005\"\u0000"+ + "\u0000\u016c\u016e\u0003T*\u0000\u016d\u016b\u0001\u0000\u0000\u0000\u016e"+ + "\u0171\u0001\u0000\u0000\u0000\u016f\u016d\u0001\u0000\u0000\u0000\u016f"+ + "\u0170\u0001\u0000\u0000\u0000\u0170\u0172\u0001\u0000\u0000\u0000\u0171"+ + "\u016f\u0001\u0000\u0000\u0000\u0172\u0173\u0005@\u0000\u0000\u0173\u0175"+ + "\u0001\u0000\u0000\u0000\u0174\u014a\u0001\u0000\u0000\u0000\u0174\u014b"+ + "\u0001\u0000\u0000\u0000\u0174\u014e\u0001\u0000\u0000\u0000\u0174\u014f"+ + "\u0001\u0000\u0000\u0000\u0174\u0150\u0001\u0000\u0000\u0000\u0174\u0151"+ + "\u0001\u0000\u0000\u0000\u0174\u0152\u0001\u0000\u0000\u0000\u0174\u0153"+ + "\u0001\u0000\u0000\u0000\u0174\u015e\u0001\u0000\u0000\u0000\u0174\u0169"+ + "\u0001\u0000\u0000\u0000\u01753\u0001\u0000\u0000\u0000\u0176\u0177\u0005"+ + "\n\u0000\u0000\u0177\u0178\u0005\u001c\u0000\u0000\u01785\u0001\u0000"+ + "\u0000\u0000\u0179\u017a\u0005\u0010\u0000\u0000\u017a\u017f\u00038\u001c"+ + "\u0000\u017b\u017c\u0005\"\u0000\u0000\u017c\u017e\u00038\u001c\u0000"+ + "\u017d\u017b\u0001\u0000\u0000\u0000\u017e\u0181\u0001\u0000\u0000\u0000"+ + "\u017f\u017d\u0001\u0000\u0000\u0000\u017f\u0180\u0001\u0000\u0000\u0000"+ + "\u01807\u0001\u0000\u0000\u0000\u0181\u017f\u0001\u0000\u0000\u0000\u0182"+ + "\u0184\u0003\n\u0005\u0000\u0183\u0185\u0007\u0005\u0000\u0000\u0184\u0183"+ + "\u0001\u0000\u0000\u0000\u0184\u0185\u0001\u0000\u0000\u0000\u0185\u0188"+ + "\u0001\u0000\u0000\u0000\u0186\u0187\u0005.\u0000\u0000\u0187\u0189\u0007"+ + "\u0006\u0000\u0000\u0188\u0186\u0001\u0000\u0000\u0000\u0188\u0189\u0001"+ + "\u0000\u0000\u0000\u01899\u0001\u0000\u0000\u0000\u018a\u018b\u0005\t"+ + "\u0000\u0000\u018b\u0190\u0003,\u0016\u0000\u018c\u018d\u0005\"\u0000"+ + "\u0000\u018d\u018f\u0003,\u0016\u0000\u018e\u018c\u0001\u0000\u0000\u0000"+ + "\u018f\u0192\u0001\u0000\u0000\u0000\u0190\u018e\u0001\u0000\u0000\u0000"+ + "\u0190\u0191\u0001\u0000\u0000\u0000\u0191\u019d\u0001\u0000\u0000\u0000"+ + "\u0192\u0190\u0001\u0000\u0000\u0000\u0193\u0194\u0005\f\u0000\u0000\u0194"+ + "\u0199\u0003,\u0016\u0000\u0195\u0196\u0005\"\u0000\u0000\u0196\u0198"+ + "\u0003,\u0016\u0000\u0197\u0195\u0001\u0000\u0000\u0000\u0198\u019b\u0001"+ + "\u0000\u0000\u0000\u0199\u0197\u0001\u0000\u0000\u0000\u0199\u019a\u0001"+ + "\u0000\u0000\u0000\u019a\u019d\u0001\u0000\u0000\u0000\u019b\u0199\u0001"+ + "\u0000\u0000\u0000\u019c\u018a\u0001\u0000\u0000\u0000\u019c\u0193\u0001"+ + "\u0000\u0000\u0000\u019d;\u0001\u0000\u0000\u0000\u019e\u019f\u0005\u0002"+ + "\u0000\u0000\u019f\u01a4\u0003,\u0016\u0000\u01a0\u01a1\u0005\"\u0000"+ + "\u0000\u01a1\u01a3\u0003,\u0016\u0000\u01a2\u01a0\u0001\u0000\u0000\u0000"+ + "\u01a3\u01a6\u0001\u0000\u0000\u0000\u01a4\u01a2\u0001\u0000\u0000\u0000"+ + "\u01a4\u01a5\u0001\u0000\u0000\u0000\u01a5=\u0001\u0000\u0000\u0000\u01a6"+ + "\u01a4\u0001\u0000\u0000\u0000\u01a7\u01a8\u0005\r\u0000\u0000\u01a8\u01ad"+ + "\u0003@ \u0000\u01a9\u01aa\u0005\"\u0000\u0000\u01aa\u01ac\u0003@ \u0000"+ + "\u01ab\u01a9\u0001\u0000\u0000\u0000\u01ac\u01af\u0001\u0000\u0000\u0000"+ + "\u01ad\u01ab\u0001\u0000\u0000\u0000\u01ad\u01ae\u0001\u0000\u0000\u0000"+ + "\u01ae?\u0001\u0000\u0000\u0000\u01af\u01ad\u0001\u0000\u0000\u0000\u01b0"+ + "\u01b1\u0003,\u0016\u0000\u01b1\u01b2\u0005O\u0000\u0000\u01b2\u01b3\u0003"+ + ",\u0016\u0000\u01b3A\u0001\u0000\u0000\u0000\u01b4\u01b5\u0005\u0001\u0000"+ + "\u0000\u01b5\u01b6\u0003\u0012\t\u0000\u01b6\u01b8\u0003T*\u0000\u01b7"+ + "\u01b9\u0003H$\u0000\u01b8\u01b7\u0001\u0000\u0000\u0000\u01b8\u01b9\u0001"+ + "\u0000\u0000\u0000\u01b9C\u0001\u0000\u0000\u0000\u01ba\u01bb\u0005\u0007"+ + "\u0000\u0000\u01bb\u01bc\u0003\u0012\t\u0000\u01bc\u01bd\u0003T*\u0000"+ + "\u01bdE\u0001\u0000\u0000\u0000\u01be\u01bf\u0005\u000b\u0000\u0000\u01bf"+ + "\u01c0\u0003*\u0015\u0000\u01c0G\u0001\u0000\u0000\u0000\u01c1\u01c6\u0003"+ + "J%\u0000\u01c2\u01c3\u0005\"\u0000\u0000\u01c3\u01c5\u0003J%\u0000\u01c4"+ + "\u01c2\u0001\u0000\u0000\u0000\u01c5\u01c8\u0001\u0000\u0000\u0000\u01c6"+ + "\u01c4\u0001\u0000\u0000\u0000\u01c6\u01c7\u0001\u0000\u0000\u0000\u01c7"+ + "I\u0001\u0000\u0000\u0000\u01c8\u01c6\u0001\u0000\u0000\u0000\u01c9\u01ca"+ + "\u0003.\u0017\u0000\u01ca\u01cb\u0005!\u0000\u0000\u01cb\u01cc\u00032"+ + "\u0019\u0000\u01ccK\u0001\u0000\u0000\u0000\u01cd\u01ce\u0007\u0007\u0000"+ + "\u0000\u01ceM\u0001\u0000\u0000\u0000\u01cf\u01d2\u0003P(\u0000\u01d0"+ + "\u01d2\u0003R)\u0000\u01d1\u01cf\u0001\u0000\u0000\u0000\u01d1\u01d0\u0001"+ + "\u0000\u0000\u0000\u01d2O\u0001\u0000\u0000\u0000\u01d3\u01d5\u0007\u0000"+ + "\u0000\u0000\u01d4\u01d3\u0001\u0000\u0000\u0000\u01d4\u01d5\u0001\u0000"+ + "\u0000\u0000\u01d5\u01d6\u0001\u0000\u0000\u0000\u01d6\u01d7\u0005\u001d"+ + "\u0000\u0000\u01d7Q\u0001\u0000\u0000\u0000\u01d8\u01da\u0007\u0000\u0000"+ + "\u0000\u01d9\u01d8\u0001\u0000\u0000\u0000\u01d9\u01da\u0001\u0000\u0000"+ + "\u0000\u01da\u01db\u0001\u0000\u0000\u0000\u01db\u01dc\u0005\u001c\u0000"+ + "\u0000\u01dcS\u0001\u0000\u0000\u0000\u01dd\u01de\u0005\u001b\u0000\u0000"+ + "\u01deU\u0001\u0000\u0000\u0000\u01df\u01e0\u0007\b\u0000\u0000\u01e0"+ + "W\u0001\u0000\u0000\u0000\u01e1\u01e2\u0005\u0005\u0000\u0000\u01e2\u01e3"+ + "\u0003Z-\u0000\u01e3Y\u0001\u0000\u0000\u0000\u01e4\u01e5\u0005?\u0000"+ + "\u0000\u01e5\u01e6\u0003\u0002\u0001\u0000\u01e6\u01e7\u0005@\u0000\u0000"+ + "\u01e7[\u0001\u0000\u0000\u0000\u01e8\u01e9\u0005\u000f\u0000\u0000\u01e9"+ + "\u01ed\u0005^\u0000\u0000\u01ea\u01eb\u0005\u000f\u0000\u0000\u01eb\u01ed"+ + "\u0005_\u0000\u0000\u01ec\u01e8\u0001\u0000\u0000\u0000\u01ec\u01ea\u0001"+ + "\u0000\u0000\u0000\u01ed]\u0001\u0000\u0000\u0000\u01ee\u01ef\u0005\u0003"+ + "\u0000\u0000\u01ef\u01f2\u0003(\u0014\u0000\u01f0\u01f1\u0005S\u0000\u0000"+ + "\u01f1\u01f3\u0003,\u0016\u0000\u01f2\u01f0\u0001\u0000\u0000\u0000\u01f2"+ + "\u01f3\u0001\u0000\u0000\u0000\u01f3\u01fd\u0001\u0000\u0000\u0000\u01f4"+ + "\u01f5\u0005T\u0000\u0000\u01f5\u01fa\u0003`0\u0000\u01f6\u01f7\u0005"+ + "\"\u0000\u0000\u01f7\u01f9\u0003`0\u0000\u01f8\u01f6\u0001\u0000\u0000"+ + "\u0000\u01f9\u01fc\u0001\u0000\u0000\u0000\u01fa\u01f8\u0001\u0000\u0000"+ + "\u0000\u01fa\u01fb\u0001\u0000\u0000\u0000\u01fb\u01fe\u0001\u0000\u0000"+ + "\u0000\u01fc\u01fa\u0001\u0000\u0000\u0000\u01fd\u01f4\u0001\u0000\u0000"+ + "\u0000\u01fd\u01fe\u0001\u0000\u0000\u0000\u01fe_\u0001\u0000\u0000\u0000"+ + "\u01ff\u0200\u0003,\u0016\u0000\u0200\u0201\u0005!\u0000\u0000\u0201\u0203"+ + "\u0001\u0000\u0000\u0000\u0202\u01ff\u0001\u0000\u0000\u0000\u0202\u0203"+ + "\u0001\u0000\u0000\u0000\u0203\u0204\u0001\u0000\u0000\u0000\u0204\u0205"+ + "\u0003,\u0016\u0000\u0205a\u0001\u0000\u0000\u00004mt\u0083\u008f\u0098"+ + "\u00a0\u00a4\u00ac\u00ae\u00b3\u00ba\u00bf\u00c6\u00cc\u00d4\u00d6\u00e0"+ + "\u00ea\u00ed\u00f9\u0101\u0109\u010d\u0116\u0120\u0124\u012a\u0131\u013b"+ + "\u0143\u0159\u0164\u016f\u0174\u017f\u0184\u0188\u0190\u0199\u019c\u01a4"+ + "\u01ad\u01b8\u01c6\u01d1\u01d4\u01d9\u01ec\u01f2\u01fa\u01fd\u0202"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java index 3137eff0b6550..73b529cd2be92 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java @@ -401,13 +401,13 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener { * *

The default implementation does nothing.

*/ - @Override public void enterSourceIdentifier(EsqlBaseParser.SourceIdentifierContext ctx) { } + @Override public void enterFromIdentifier(EsqlBaseParser.FromIdentifierContext ctx) { } /** * {@inheritDoc} * *

The default implementation does nothing.

*/ - @Override public void exitSourceIdentifier(EsqlBaseParser.SourceIdentifierContext ctx) { } + @Override public void exitFromIdentifier(EsqlBaseParser.FromIdentifierContext ctx) { } /** * {@inheritDoc} * @@ -420,6 +420,18 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener { *

The default implementation does nothing.

*/ @Override public void exitQualifiedName(EsqlBaseParser.QualifiedNameContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx) { } /** * {@inheritDoc} * @@ -432,6 +444,18 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener { *

The default implementation does nothing.

*/ @Override public void exitIdentifier(EsqlBaseParser.IdentifierContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterIdentifierPattern(EsqlBaseParser.IdentifierPatternContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitIdentifierPattern(EsqlBaseParser.IdentifierPatternContext ctx) { } /** * {@inheritDoc} * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java index d7b2f359e3c83..d35481745cecc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java @@ -242,7 +242,7 @@ public class EsqlBaseParserBaseVisitor extends AbstractParseTreeVisitor im *

The default implementation returns the result of calling * {@link #visitChildren} on {@code ctx}.

*/ - @Override public T visitSourceIdentifier(EsqlBaseParser.SourceIdentifierContext ctx) { return visitChildren(ctx); } + @Override public T visitFromIdentifier(EsqlBaseParser.FromIdentifierContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * @@ -250,6 +250,13 @@ public class EsqlBaseParserBaseVisitor extends AbstractParseTreeVisitor im * {@link #visitChildren} on {@code ctx}.

*/ @Override public T visitQualifiedName(EsqlBaseParser.QualifiedNameContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * @@ -257,6 +264,13 @@ public class EsqlBaseParserBaseVisitor extends AbstractParseTreeVisitor im * {@link #visitChildren} on {@code ctx}.

*/ @Override public T visitIdentifier(EsqlBaseParser.IdentifierContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitIdentifierPattern(EsqlBaseParser.IdentifierPatternContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java index dd6cdaacddbef..6c8cd7272d8dc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java @@ -362,15 +362,15 @@ public interface EsqlBaseParserListener extends ParseTreeListener { */ void exitGrouping(EsqlBaseParser.GroupingContext ctx); /** - * Enter a parse tree produced by {@link EsqlBaseParser#sourceIdentifier}. + * Enter a parse tree produced by {@link EsqlBaseParser#fromIdentifier}. * @param ctx the parse tree */ - void enterSourceIdentifier(EsqlBaseParser.SourceIdentifierContext ctx); + void enterFromIdentifier(EsqlBaseParser.FromIdentifierContext ctx); /** - * Exit a parse tree produced by {@link EsqlBaseParser#sourceIdentifier}. + * Exit a parse tree produced by {@link EsqlBaseParser#fromIdentifier}. * @param ctx the parse tree */ - void exitSourceIdentifier(EsqlBaseParser.SourceIdentifierContext ctx); + void exitFromIdentifier(EsqlBaseParser.FromIdentifierContext ctx); /** * Enter a parse tree produced by {@link EsqlBaseParser#qualifiedName}. * @param ctx the parse tree @@ -381,6 +381,16 @@ public interface EsqlBaseParserListener extends ParseTreeListener { * @param ctx the parse tree */ void exitQualifiedName(EsqlBaseParser.QualifiedNameContext ctx); + /** + * Enter a parse tree produced by {@link EsqlBaseParser#qualifiedNamePattern}. + * @param ctx the parse tree + */ + void enterQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx); + /** + * Exit a parse tree produced by {@link EsqlBaseParser#qualifiedNamePattern}. + * @param ctx the parse tree + */ + void exitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx); /** * Enter a parse tree produced by {@link EsqlBaseParser#identifier}. * @param ctx the parse tree @@ -391,6 +401,16 @@ public interface EsqlBaseParserListener extends ParseTreeListener { * @param ctx the parse tree */ void exitIdentifier(EsqlBaseParser.IdentifierContext ctx); + /** + * Enter a parse tree produced by {@link EsqlBaseParser#identifierPattern}. + * @param ctx the parse tree + */ + void enterIdentifierPattern(EsqlBaseParser.IdentifierPatternContext ctx); + /** + * Exit a parse tree produced by {@link EsqlBaseParser#identifierPattern}. + * @param ctx the parse tree + */ + void exitIdentifierPattern(EsqlBaseParser.IdentifierPatternContext ctx); /** * Enter a parse tree produced by the {@code nullLiteral} * labeled alternative in {@link EsqlBaseParser#constant}. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java index 35297f3d4f336..2fe5de566dbaf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java @@ -220,23 +220,35 @@ public interface EsqlBaseParserVisitor extends ParseTreeVisitor { */ T visitGrouping(EsqlBaseParser.GroupingContext ctx); /** - * Visit a parse tree produced by {@link EsqlBaseParser#sourceIdentifier}. + * Visit a parse tree produced by {@link EsqlBaseParser#fromIdentifier}. * @param ctx the parse tree * @return the visitor result */ - T visitSourceIdentifier(EsqlBaseParser.SourceIdentifierContext ctx); + T visitFromIdentifier(EsqlBaseParser.FromIdentifierContext ctx); /** * Visit a parse tree produced by {@link EsqlBaseParser#qualifiedName}. * @param ctx the parse tree * @return the visitor result */ T visitQualifiedName(EsqlBaseParser.QualifiedNameContext ctx); + /** + * Visit a parse tree produced by {@link EsqlBaseParser#qualifiedNamePattern}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx); /** * Visit a parse tree produced by {@link EsqlBaseParser#identifier}. * @param ctx the parse tree * @return the visitor result */ T visitIdentifier(EsqlBaseParser.IdentifierContext ctx); + /** + * Visit a parse tree produced by {@link EsqlBaseParser#identifierPattern}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitIdentifierPattern(EsqlBaseParser.IdentifierPatternContext ctx); /** * Visit a parse tree produced by the {@code nullLiteral} * labeled alternative in {@link EsqlBaseParser#constant}. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index f24324fac2fbd..3b1ef475350b1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -207,10 +207,21 @@ public UnresolvedAttribute visitQualifiedName(EsqlBaseParser.QualifiedNameContex return null; } - return new UnresolvedAttribute( - source(ctx), - Strings.collectionToDelimitedString(visitList(this, ctx.identifier(), String.class), ".") - ); + List strings = visitList(this, ctx.identifier(), String.class); + return new UnresolvedAttribute(source(ctx), Strings.collectionToDelimitedString(strings, ".")); + } + + @Override + public NamedExpression visitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx) { + if (ctx == null) { + return null; + } + + List strings = visitList(this, ctx.identifierPattern(), String.class); + var src = source(ctx); + return strings.size() == 1 && strings.get(0).equals(WILDCARD) + ? new UnresolvedStar(src, null) + : new UnresolvedAttribute(src, Strings.collectionToDelimitedString(strings, ".")); } @Override @@ -366,22 +377,32 @@ public Order visitOrderExpression(EsqlBaseParser.OrderExpressionContext ctx) { ); } - public NamedExpression visitProjectExpression(EsqlBaseParser.SourceIdentifierContext ctx) { - Source src = source(ctx); - String identifier = visitSourceIdentifier(ctx); - return identifier.equals(WILDCARD) ? new UnresolvedStar(src, null) : new UnresolvedAttribute(src, identifier); - } - @Override public Alias visitRenameClause(EsqlBaseParser.RenameClauseContext ctx) { Source src = source(ctx); - String newName = visitSourceIdentifier(ctx.newName); - String oldName = visitSourceIdentifier(ctx.oldName); - if (newName.contains(WILDCARD) || oldName.contains(WILDCARD)) { + NamedExpression newName = visitQualifiedNamePattern(ctx.newName); + NamedExpression oldName = visitQualifiedNamePattern(ctx.oldName); + if (newName.name().contains(WILDCARD) || oldName.name().contains(WILDCARD)) { throw new ParsingException(src, "Using wildcards (*) in renaming projections is not allowed [{}]", src.text()); } - return new Alias(src, newName, new UnresolvedAttribute(source(ctx.oldName), oldName)); + return new Alias(src, newName.name(), oldName); + } + + @Override + public NamedExpression visitEnrichWithClause(EsqlBaseParser.EnrichWithClauseContext ctx) { + Source src = source(ctx); + NamedExpression enrichField = enrichFieldName(ctx.enrichField); + NamedExpression newName = enrichFieldName(ctx.newName); + return newName == null ? enrichField : new Alias(src, newName.name(), enrichField); + } + + private NamedExpression enrichFieldName(EsqlBaseParser.QualifiedNamePatternContext ctx) { + var name = visitQualifiedNamePattern(ctx); + if (name != null && name.name().contains(WILDCARD)) { + throw new ParsingException(source(ctx), "Using wildcards (*) in ENRICH WITH projections is not allowed [{}]", name.name()); + } + return name; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java index 296206b1079b2..2039dc633f6cf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java @@ -9,6 +9,8 @@ import org.antlr.v4.runtime.tree.TerminalNode; import org.elasticsearch.common.Strings; +import org.elasticsearch.xpack.esql.parser.EsqlBaseParser.FromIdentifierContext; +import org.elasticsearch.xpack.esql.parser.EsqlBaseParser.IdentifierContext; import java.util.List; @@ -17,27 +19,32 @@ abstract class IdentifierBuilder extends AbstractBuilder { @Override - public String visitIdentifier(EsqlBaseParser.IdentifierContext ctx) { - return unquoteIdentifier(ctx.QUOTED_IDENTIFIER(), ctx.UNQUOTED_IDENTIFIER()); + public String visitIdentifier(IdentifierContext ctx) { + return ctx == null ? null : unquoteIdentifier(ctx.QUOTED_IDENTIFIER(), ctx.UNQUOTED_IDENTIFIER()); } @Override - public String visitSourceIdentifier(EsqlBaseParser.SourceIdentifierContext ctx) { - return unquoteIdentifier(ctx.SRC_QUOTED_IDENTIFIER(), ctx.SRC_UNQUOTED_IDENTIFIER()); + public String visitIdentifierPattern(EsqlBaseParser.IdentifierPatternContext ctx) { + return unquoteIdentifier(ctx.QUOTED_IDENTIFIER(), ctx.PROJECT_UNQUOTED_IDENTIFIER()); } - private static String unquoteIdentifier(TerminalNode quotedNode, TerminalNode unquotedNode) { + @Override + public String visitFromIdentifier(FromIdentifierContext ctx) { + return ctx == null ? null : unquoteIdentifier(ctx.QUOTED_IDENTIFIER(), ctx.FROM_UNQUOTED_IDENTIFIER()); + } + + static String unquoteIdentifier(TerminalNode quotedNode, TerminalNode unquotedNode) { String result; if (quotedNode != null) { String identifier = quotedNode.getText(); - result = identifier.substring(1, identifier.length() - 1); + result = identifier.substring(1, identifier.length() - 1).replace("``", "`"); } else { result = unquotedNode.getText(); } return result; } - public String visitSourceIdentifiers(List ctx) { + public String visitFromIdentifiers(List ctx) { return Strings.collectionToDelimitedString(visitList(this, ctx, String.class), ","); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index d5763f28f6394..f9d1a252afe42 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -12,6 +12,7 @@ import org.antlr.v4.runtime.tree.ParseTree; import org.elasticsearch.dissect.DissectException; import org.elasticsearch.dissect.DissectParser; +import org.elasticsearch.xpack.esql.parser.EsqlBaseParser.QualifiedNamePatternContext; import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Drop; import org.elasticsearch.xpack.esql.plan.logical.Enrich; @@ -61,7 +62,6 @@ import static org.elasticsearch.xpack.ql.parser.ParserUtils.source; import static org.elasticsearch.xpack.ql.parser.ParserUtils.typedParsing; import static org.elasticsearch.xpack.ql.parser.ParserUtils.visitList; -import static org.elasticsearch.xpack.ql.util.StringUtils.WILDCARD; public class LogicalPlanBuilder extends ExpressionBuilder { @@ -149,9 +149,9 @@ public PlanFactory visitDissectCommand(EsqlBaseParser.DissectCommandContext ctx) @Override public PlanFactory visitMvExpandCommand(EsqlBaseParser.MvExpandCommandContext ctx) { - String identifier = visitSourceIdentifier(ctx.sourceIdentifier()); + UnresolvedAttribute field = visitQualifiedName(ctx.qualifiedName()); Source src = source(ctx); - return child -> new MvExpand(src, child, new UnresolvedAttribute(src, identifier), new UnresolvedAttribute(src, identifier)); + return child -> new MvExpand(src, child, field, new UnresolvedAttribute(src, field.name())); } @@ -175,11 +175,11 @@ public LogicalPlan visitRowCommand(EsqlBaseParser.RowCommandContext ctx) { @Override public LogicalPlan visitFromCommand(EsqlBaseParser.FromCommandContext ctx) { Source source = source(ctx); - TableIdentifier table = new TableIdentifier(source, null, visitSourceIdentifiers(ctx.sourceIdentifier())); + TableIdentifier table = new TableIdentifier(source, null, visitFromIdentifiers(ctx.fromIdentifier())); Map metadataMap = new LinkedHashMap<>(); if (ctx.metadata() != null) { - for (var c : ctx.metadata().sourceIdentifier()) { - String id = visitSourceIdentifier(c); + for (var c : ctx.metadata().fromIdentifier()) { + String id = visitFromIdentifier(c); Source src = source(c); if (MetadataAttribute.isSupported(id) == false) { throw new ParsingException(src, "unsupported metadata field [" + id + "]"); @@ -254,16 +254,16 @@ public Explain visitExplainCommand(EsqlBaseParser.ExplainCommandContext ctx) { @Override public PlanFactory visitDropCommand(EsqlBaseParser.DropCommandContext ctx) { - var identifiers = ctx.sourceIdentifier(); + var identifiers = ctx.qualifiedNamePattern(); List removals = new ArrayList<>(identifiers.size()); - for (EsqlBaseParser.SourceIdentifierContext idCtx : identifiers) { - Source src = source(idCtx); - String identifier = visitSourceIdentifier(idCtx); - if (identifier.equals(WILDCARD)) { + for (QualifiedNamePatternContext patternContext : identifiers) { + NamedExpression ne = visitQualifiedNamePattern(patternContext); + if (ne instanceof UnresolvedStar) { + var src = ne.source(); throw new ParsingException(src, "Removing all fields is not allowed [{}]", src.text()); } - removals.add(new UnresolvedAttribute(src, identifier)); + removals.add(ne); } return child -> new Drop(source(ctx), child, removals); @@ -280,13 +280,15 @@ public PlanFactory visitKeepCommand(EsqlBaseParser.KeepCommandContext ctx) { if (ctx.PROJECT() != null) { addWarning("PROJECT command is no longer supported, please use KEEP instead"); } - List projections = new ArrayList<>(ctx.sourceIdentifier().size()); + var identifiers = ctx.qualifiedNamePattern(); + List projections = new ArrayList<>(identifiers.size()); boolean hasSeenStar = false; - for (var srcIdCtx : ctx.sourceIdentifier()) { - NamedExpression ne = visitProjectExpression(srcIdCtx); + for (QualifiedNamePatternContext patternContext : identifiers) { + NamedExpression ne = visitQualifiedNamePattern(patternContext); if (ne instanceof UnresolvedStar) { if (hasSeenStar) { - throw new ParsingException(ne.source(), "Cannot specify [*] more than once", ne.source().text()); + var src = ne.source(); + throw new ParsingException(src, "Cannot specify [*] more than once", src.text()); } else { hasSeenStar = true; } @@ -309,11 +311,9 @@ public LogicalPlan visitShowFunctions(EsqlBaseParser.ShowFunctionsContext ctx) { @Override public PlanFactory visitEnrichCommand(EsqlBaseParser.EnrichCommandContext ctx) { return p -> { - final String policyName = visitSourceIdentifier(ctx.policyName); + String policyName = visitFromIdentifier(ctx.policyName); var source = source(ctx); - NamedExpression matchField = ctx.ON() != null - ? new UnresolvedAttribute(source(ctx.matchField), visitSourceIdentifier(ctx.matchField)) - : new EmptyAttribute(source); + NamedExpression matchField = ctx.ON() != null ? visitQualifiedNamePattern(ctx.matchField) : new EmptyAttribute(source); if (matchField.name().contains("*")) { throw new ParsingException( source(ctx), @@ -321,6 +321,7 @@ public PlanFactory visitEnrichCommand(EsqlBaseParser.EnrichCommandContext ctx) { matchField.name() ); } + List keepClauses = visitList(this, ctx.enrichWithClause(), NamedExpression.class); return new Enrich( source, @@ -333,22 +334,5 @@ public PlanFactory visitEnrichCommand(EsqlBaseParser.EnrichCommandContext ctx) { }; } - @Override - public NamedExpression visitEnrichWithClause(EsqlBaseParser.EnrichWithClauseContext ctx) { - Source src = source(ctx); - String enrichField = enrichFieldName(ctx.enrichField); - String newName = enrichFieldName(ctx.newName); - UnresolvedAttribute enrichAttr = new UnresolvedAttribute(src, enrichField); - return newName == null ? enrichAttr : new Alias(src, newName, enrichAttr); - } - - private String enrichFieldName(EsqlBaseParser.SourceIdentifierContext ctx) { - String name = ctx == null ? null : visitSourceIdentifier(ctx); - if (name != null && name.contains(WILDCARD)) { - throw new ParsingException(source(ctx), "Using wildcards (*) in ENRICH WITH projections is not allowed [{}]", name); - } - return name; - } - interface PlanFactory extends Function {} } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java index 17f669b5d30b3..0cdcd4af00026 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java @@ -22,13 +22,12 @@ public class MvExpand extends UnaryPlan { private final NamedExpression target; private final Attribute expanded; - private final List output; + private List output; public MvExpand(Source source, LogicalPlan child, NamedExpression target, Attribute expanded) { super(source, child); this.target = target; this.expanded = expanded; - this.output = calculateOutput(child.output(), target, expanded); } public static List calculateOutput(List input, NamedExpression target, Attribute expanded) { @@ -63,6 +62,9 @@ public UnaryPlan replaceChild(LogicalPlan newChild) { @Override public List output() { + if (output == null) { + output = calculateOutput(child().output(), target, expanded); + } return output; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index b7b31868d65e2..20fcc05e80440 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -328,6 +328,7 @@ void runCompute(CancellableTask task, ComputeContext context, PhysicalPlan plan, private void acquireSearchContexts( List shardIds, + EsqlConfiguration configuration, Map aliasFilters, ActionListener> listener ) { @@ -351,11 +352,12 @@ private void acquireSearchContexts( try { for (IndexShard shard : targetShards) { var aliasFilter = aliasFilters.getOrDefault(shard.shardId().getIndex(), AliasFilter.EMPTY); - ShardSearchRequest shardSearchLocalRequest = new ShardSearchRequest(shard.shardId(), 0, aliasFilter); - SearchContext context = searchService.createSearchContext( - shardSearchLocalRequest, - SearchService.NO_TIMEOUT + var shardRequest = new ShardSearchRequest( + shard.shardId(), + configuration.absoluteStartedTimeInMillis(), + aliasFilter ); + SearchContext context = searchService.createSearchContext(shardRequest, SearchService.NO_TIMEOUT); searchContexts.add(context); } for (SearchContext searchContext : searchContexts) { @@ -501,8 +503,9 @@ public void messageReceived(DataNodeRequest request, TransportChannel channel, T final var exchangeSink = exchangeService.getSinkHandler(sessionId); parentTask.addListener(() -> exchangeService.finishSinkHandler(sessionId, new TaskCancelledException("task cancelled"))); final ActionListener listener = new OwningChannelActionListener<>(channel); - acquireSearchContexts(request.shardIds(), request.aliasFilters(), ActionListener.wrap(searchContexts -> { - var computeContext = new ComputeContext(sessionId, searchContexts, request.configuration(), null, exchangeSink); + final EsqlConfiguration configuration = request.configuration(); + acquireSearchContexts(request.shardIds(), configuration, request.aliasFilters(), ActionListener.wrap(searchContexts -> { + var computeContext = new ComputeContext(sessionId, searchContexts, configuration, null, exchangeSink); runCompute(parentTask, computeContext, request.plan(), ActionListener.wrap(driverProfiles -> { // don't return until all pages are fetched exchangeSink.addCompletionListener( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlConfiguration.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlConfiguration.java index ac13f25c2d2a9..ccec6554cb2cb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlConfiguration.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlConfiguration.java @@ -112,6 +112,15 @@ public String query() { return query; } + /** + * Returns the current time in milliseconds from the time epoch for the execution of this request. + * It ensures consistency by using the same value on all nodes involved in the search request. + * Note: Currently, it returns {@link System#currentTimeMillis()}, but this value will be serialized between nodes. + */ + public long absoluteStartedTimeInMillis() { + return System.currentTimeMillis(); + } + /** * Enable profiling, sacrificing performance to return information about * what operations are taking the most time. diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 17ed0c1223636..a5e3033b6e1e3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -229,7 +229,7 @@ protected final boolean enableWarningsCheck() { } public boolean logResults() { - return true; + return false; } private void doTest() throws Exception { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 93456ff30c4cd..90e45a0a8b5a7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolution; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.expression.function.aggregate.Max; +import org.elasticsearch.xpack.esql.parser.ParsingException; import org.elasticsearch.xpack.esql.plan.logical.EsqlUnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Row; @@ -266,11 +267,68 @@ public void testNoProjection() { ); } - public void testProjectOrder() { + public void testDuplicateProjections() { + assertProjection(""" + from test + | keep first_name, first_name + """, "first_name"); + assertProjection(""" + from test + | keep first_name, first_name, last_name, first_name + """, "last_name", "first_name"); + } + + public void testProjectWildcard() { assertProjection(""" from test | keep first_name, *, last_name """, "first_name", "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary", "last_name"); + assertProjection(""" + from test + | keep first_name, last_name, * + """, "first_name", "last_name", "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary"); + assertProjection(""" + from test + | keep *, first_name, last_name + """, "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary", "first_name", "last_name"); + + var e = expectThrows(ParsingException.class, () -> analyze(""" + from test + | keep *, first_name, last_name, * + """)); + assertThat(e.getMessage(), containsString("Cannot specify [*] more than once")); + + } + + public void testProjectMixedWildcard() { + assertProjection(""" + from test + | keep *name, first* + """, "last_name", "first_name"); + assertProjection(""" + from test + | keep first_name, *name, first* + """, "first_name", "last_name"); + assertProjection(""" + from test + | keep *ob*, first_name, *name, first* + """, "job", "job.raw", "first_name", "last_name"); + assertProjection(""" + from test + | keep first_name, *, *name + """, "first_name", "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary", "last_name"); + assertProjection(""" + from test + | keep first*, *, last_name, first_name + """, "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary", "last_name", "first_name"); + assertProjection(""" + from test + | keep first*, *, last_name, fir* + """, "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary", "last_name", "first_name"); + assertProjection(""" + from test + | keep *, job* + """, "_meta_field", "emp_no", "first_name", "gender", "languages", "last_name", "long_noidx", "salary", "job", "job.raw"); } public void testProjectThenDropName() { @@ -1429,9 +1487,6 @@ public void testMissingAttributeException_InChainedEval() { public void testUnresolvedMvExpand() { var e = expectThrows(VerificationException.class, () -> analyze("row foo = 1 | mv_expand bar")); assertThat(e.getMessage(), containsString("Unknown column [bar]")); - - e = expectThrows(VerificationException.class, () -> analyze("row foo = 1 | keep foo, foo | mv_expand foo")); - assertThat(e.getMessage(), containsString("Reference [foo] is ambiguous (to disambiguate use quotes or qualifiers)")); } private void verifyUnsupported(String query, String errorMessage) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java index ebcaf367b1226..0309bcce85581 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java @@ -54,17 +54,13 @@ public static Iterable parameters() { List.of() ); // random strings that don't look like a double - TestCaseSupplier.forUnaryStrings( - suppliers, - evaluatorName.apply("String"), - DataTypes.DOUBLE, - bytesRef -> null, - bytesRef -> List.of( + TestCaseSupplier.forUnaryStrings(suppliers, evaluatorName.apply("String"), DataTypes.DOUBLE, bytesRef -> null, bytesRef -> { + var exception = expectThrows(NumberFormatException.class, () -> Double.parseDouble(bytesRef.utf8ToString())); + return List.of( "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", - "Line -1:-1: java.lang.NumberFormatException: " - + (bytesRef.utf8ToString().isEmpty() ? "empty String" : ("For input string: \"" + bytesRef.utf8ToString() + "\"")) - ) - ); + "Line -1:-1: " + exception + ); + }); TestCaseSupplier.forUnaryUnsignedLong( suppliers, evaluatorName.apply("UnsignedLong"), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java index bc46189e13827..ac2426f485fcc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.esql.analysis.Analyzer; import org.elasticsearch.xpack.esql.analysis.AnalyzerContext; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; @@ -21,9 +22,11 @@ import org.elasticsearch.xpack.ql.expression.Expressions; import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.expression.ReferenceAttribute; +import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.ql.index.EsIndex; import org.elasticsearch.xpack.ql.index.IndexResolution; import org.elasticsearch.xpack.ql.plan.logical.EsRelation; +import org.elasticsearch.xpack.ql.plan.logical.Filter; import org.elasticsearch.xpack.ql.plan.logical.Limit; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.plan.logical.Project; @@ -35,6 +38,7 @@ import java.util.Map; import static org.elasticsearch.xpack.esql.EsqlTestUtils.L; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_SEARCH_STATS; import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER; import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; @@ -263,6 +267,38 @@ public void testMissingFieldInFilterNoProjection() { ); } + public void testIsNotNullOnCoalesce() { + var plan = localPlan(""" + from test + | where coalesce(emp_no, salary) is not null + """); + + var limit = as(plan, Limit.class); + var filter = as(limit.child(), Filter.class); + var inn = as(filter.condition(), IsNotNull.class); + var coalesce = as(inn.children().get(0), Coalesce.class); + assertThat(Expressions.names(coalesce.children()), contains("emp_no", "salary")); + var source = as(filter.child(), EsRelation.class); + } + + public void testIsNotNullOnExpression() { + var plan = localPlan(""" + from test + | eval x = emp_no + 1 + | where x is not null + """); + + var limit = as(plan, Limit.class); + var filter = as(limit.child(), Filter.class); + var inn = as(filter.condition(), IsNotNull.class); + assertThat(Expressions.names(inn.children()), contains("x")); + var eval = as(filter.child(), Eval.class); + filter = as(eval.child(), Filter.class); + inn = as(filter.condition(), IsNotNull.class); + assertThat(Expressions.names(inn.children()), contains("emp_no")); + var source = as(filter.child(), EsRelation.class); + } + private LocalRelation asEmptyRelation(Object o) { var empty = as(o, LocalRelation.class); assertThat(empty.supplier(), is(LocalSupplier.EMPTY)); @@ -285,6 +321,10 @@ private LogicalPlan localPlan(LogicalPlan plan, SearchStats searchStats) { return localPlan; } + private LogicalPlan localPlan(String query) { + return localPlan(plan(query), TEST_SEARCH_STATS); + } + @Override protected List filteredWarnings() { return withDefaultLimitWarning(super.filteredWarnings()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 53e2a2e412fcd..b20d166beb22e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -26,8 +26,10 @@ import org.elasticsearch.xpack.esql.plan.logical.InlineStats; import org.elasticsearch.xpack.esql.plan.logical.MvExpand; import org.elasticsearch.xpack.esql.plan.logical.Row; +import org.elasticsearch.xpack.ql.capabilities.UnresolvedException; import org.elasticsearch.xpack.ql.expression.Alias; import org.elasticsearch.xpack.ql.expression.EmptyAttribute; +import org.elasticsearch.xpack.ql.expression.Expressions; import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.expression.NamedExpression; import org.elasticsearch.xpack.ql.expression.Order; @@ -41,6 +43,7 @@ import org.elasticsearch.xpack.ql.plan.logical.Limit; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.plan.logical.OrderBy; +import org.elasticsearch.xpack.ql.plan.logical.Project; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.versionfield.Version; @@ -53,6 +56,7 @@ import java.util.List; import java.util.function.Function; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.ql.expression.Literal.FALSE; import static org.elasticsearch.xpack.ql.expression.Literal.TRUE; import static org.elasticsearch.xpack.ql.expression.function.FunctionResolutionStrategy.DEFAULT; @@ -60,6 +64,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.util.NumericUtils.asLongUnsigned; import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -576,13 +581,10 @@ public void testDeprecatedIsNullFunction() { public void testMetadataFieldOnOtherSources() { expectError( "row a = 1 [metadata _index]", - "1:11: mismatched input '[' expecting {, PIPE, 'and', COMMA, 'or', '+', '-', '*', '/', '%'}" - ); - expectError("show functions [metadata _index]", "line 1:16: mismatched input '[' expecting {, PIPE}"); - expectError( - "explain [from foo] [metadata _index]", - "line 1:20: mismatched input '[' expecting {PIPE, COMMA, OPENING_BRACKET, ']'}" + "1:11: mismatched input '[' expecting {, '|', 'and', ',', 'or', '+', '-', '*', '/', '%'}" ); + expectError("show functions [metadata _index]", "line 1:16: token recognition error at: '['"); + expectError("explain [from foo] [metadata _index]", "line 1:20: mismatched input '[' expecting {'|', ',', OPENING_BRACKET, ']'}"); } public void testMetadataFieldMultipleDeclarations() { @@ -709,6 +711,17 @@ public void testMvExpand() { assertThat(expand.target(), equalTo(attribute("a"))); } + // see https://github.com/elastic/elasticsearch/issues/103331 + public void testKeepStarMvExpand() { + try { + String query = "from test | keep * | mv_expand a"; + var plan = statement(query); + } catch (UnresolvedException e) { + fail(e, "Regression: https://github.com/elastic/elasticsearch/issues/103331"); + } + + } + public void testUsageOfProject() { processingCommand("project a"); assertWarnings("PROJECT command is no longer supported, please use KEEP instead"); @@ -799,6 +812,29 @@ public void testMissingInputParams() { expectError("row x = ?, y = ?", List.of(new TypedParamValue("integer", 1)), "Not enough actual parameters 1"); } + public void testFieldContainingDotsAndNumbers() { + LogicalPlan where = processingCommand("where `a.b.1m.4321`"); + assertThat(where, instanceOf(Filter.class)); + Filter w = (Filter) where; + assertThat(w.child(), equalTo(PROCESSING_CMD_INPUT)); + assertThat(Expressions.name(w.condition()), equalTo("a.b.1m.4321")); + } + + public void testFieldQualifiedName() { + LogicalPlan where = processingCommand("where a.b.`1m`.`4321`"); + assertThat(where, instanceOf(Filter.class)); + Filter w = (Filter) where; + assertThat(w.child(), equalTo(PROCESSING_CMD_INPUT)); + assertThat(Expressions.name(w.condition()), equalTo("a.b.1m.4321")); + } + + public void testQuotedName() { + // row `my-field`=123 | stats count(`my-field`) | eval x = `count(`my-field`)` + LogicalPlan plan = processingCommand("stats count(`my-field`) | keep `count(``my-field``)`"); + var project = as(plan, Project.class); + assertThat(Expressions.names(project.projections()), contains("count(`my-field`)")); + } + private void assertIdentifierAsIndexPattern(String identifier, String statement) { LogicalPlan from = statement(statement); assertThat(from, instanceOf(EsqlUnresolvedRelation.class)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index 0aaf4a1a18e32..fb5135d1de54c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -198,7 +198,7 @@ public void testEvalDateTruncGrouping() { | eval y = date_trunc(hire_date, 1 year) | stats count(emp_no) by y | sort y - | keep y, count(emp_no) + | keep y, `count(emp_no)` | limit 5""", Set.of("hire_date", "hire_date.*", "emp_no", "emp_no.*")); } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java index 2e61b6e978b61..6d3811fd66d9c 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java @@ -302,6 +302,7 @@ public void testRollupIndexInTheHotPhaseAfterRollover() throws Exception { }); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103439") public void testTsdbDataStreams() throws Exception { // Create the ILM policy DateHistogramInterval fixedInterval = ConfigTestHelpers.randomInterval(); diff --git a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle index 3f2f85e3e09da..64970d18b5c82 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle +++ b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle @@ -1,4 +1,3 @@ -import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.internal.info.BuildParams apply plugin: 'elasticsearch.legacy-java-rest-test' @@ -17,7 +16,6 @@ testClusters.configureEach { setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.history_index_enabled', 'false' setting 'slm.history_index_enabled', 'false' - requiresFeature 'es.learning_to_rank_feature_flag_enabled', Version.fromString("8.12.0") } if (BuildParams.inFipsJvm){ diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle index df2eb2c687fb5..f2ec17093bb93 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle @@ -1,4 +1,3 @@ -import org.elasticsearch.gradle.Version apply plugin: 'elasticsearch.legacy-yaml-rest-test' dependencies { @@ -258,5 +257,4 @@ testClusters.configureEach { user username: "no_ml", password: "x-pack-test-password", role: "minimal" setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' - requiresFeature 'es.learning_to_rank_feature_flag_enabled', Version.fromString("8.12.0") } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java index 113ed9a5aa686..398ef5f2e743a 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java @@ -6,19 +6,20 @@ */ package org.elasticsearch.xpack.ml.integration; +import org.apache.logging.log4j.Level; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ResourceNotFoundException; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.monitor.jvm.HotThreads; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.AggregationBuilders; @@ -468,11 +469,7 @@ public void testRealtime() throws Exception { StopDatafeedAction.Response stopJobResponse = stopDatafeed(datafeedId); assertTrue(stopJobResponse.isStopped()); } catch (Exception e) { - NodesHotThreadsResponse nodesHotThreadsResponse = clusterAdmin().prepareNodesHotThreads().get(); - int i = 0; - for (NodeHotThreads nodeHotThreads : nodesHotThreadsResponse.getNodes()) { - logger.info(i++ + ":\n" + nodeHotThreads.getHotThreads()); - } + HotThreads.logLocalHotThreads(logger, Level.INFO, "hot threads at failure", ReferenceDocs.LOGGING); throw e; } assertBusy(() -> { @@ -491,11 +488,7 @@ public void testCloseJobStopsRealtimeDatafeed() throws Exception { CloseJobAction.Response closeJobResponse = closeJob(jobId); assertTrue(closeJobResponse.isClosed()); } catch (Exception e) { - NodesHotThreadsResponse nodesHotThreadsResponse = clusterAdmin().prepareNodesHotThreads().get(); - int i = 0; - for (NodeHotThreads nodeHotThreads : nodesHotThreadsResponse.getNodes()) { - logger.info(i++ + ":\n" + nodeHotThreads.getHotThreads()); - } + HotThreads.logLocalHotThreads(logger, Level.INFO, "hot threads at failure", ReferenceDocs.LOGGING); throw e; } assertBusy(() -> { @@ -538,11 +531,7 @@ public void testCloseJobStopsLookbackOnlyDatafeed() throws Exception { CloseJobAction.Response closeJobResponse = closeJob(jobId, useForce); assertTrue(closeJobResponse.isClosed()); } catch (Exception e) { - NodesHotThreadsResponse nodesHotThreadsResponse = clusterAdmin().prepareNodesHotThreads().get(); - int i = 0; - for (NodeHotThreads nodeHotThreads : nodesHotThreadsResponse.getNodes()) { - logger.info(i++ + ":\n" + nodeHotThreads.getHotThreads()); - } + HotThreads.logLocalHotThreads(logger, Level.INFO, "hot threads at failure", ReferenceDocs.LOGGING); throw e; } GetDatafeedsStatsAction.Request request = new GetDatafeedsStatsAction.Request(datafeedId); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java index fa94bf96c1167..66f4797ef707c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java @@ -51,6 +51,11 @@ public boolean isNlpEnabled() { return true; } + @Override + public boolean isLearningToRankEnabled() { + return true; + } + @Override public String[] getAnalyticsDestIndexAllowedSettings() { return ANALYTICS_DEST_INDEX_ALLOWED_SETTINGS; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 1031d45facf85..f9c483496445e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -327,7 +327,6 @@ import org.elasticsearch.xpack.ml.inference.ingest.InferenceProcessor; import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService; import org.elasticsearch.xpack.ml.inference.ltr.LearningToRankRescorerBuilder; -import org.elasticsearch.xpack.ml.inference.ltr.LearningToRankRescorerFeature; import org.elasticsearch.xpack.ml.inference.ltr.LearningToRankService; import org.elasticsearch.xpack.ml.inference.modelsize.MlModelSizeNamedXContentProvider; import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider; @@ -373,6 +372,7 @@ import org.elasticsearch.xpack.ml.process.NativeController; import org.elasticsearch.xpack.ml.process.NativeStorageProvider; import org.elasticsearch.xpack.ml.queries.TextExpansionQueryBuilder; +import org.elasticsearch.xpack.ml.queries.WeightedTokensQueryBuilder; import org.elasticsearch.xpack.ml.rest.RestDeleteExpiredDataAction; import org.elasticsearch.xpack.ml.rest.RestMlInfoAction; import org.elasticsearch.xpack.ml.rest.RestMlMemoryAction; @@ -890,7 +890,7 @@ private static void reportClashingNodeAttribute(String attrName) { @Override public List> getRescorers() { - if (enabled && LearningToRankRescorerFeature.isEnabled()) { + if (enabled && machineLearningExtension.get().isLearningToRankEnabled()) { return List.of( new RescorerSpec<>( LearningToRankRescorerBuilder.NAME, @@ -1720,6 +1720,11 @@ public List> getQueries() { TextExpansionQueryBuilder.NAME, TextExpansionQueryBuilder::new, TextExpansionQueryBuilder::fromXContent + ), + new QuerySpec( + WeightedTokensQueryBuilder.NAME, + WeightedTokensQueryBuilder::new, + WeightedTokensQueryBuilder::fromXContent ) ); } @@ -1801,7 +1806,7 @@ public List getNamedXContent() { ); namedXContent.addAll(new CorrelationNamedContentProvider().getNamedXContentParsers()); // LTR Combine with Inference named content provider when feature flag is removed - if (LearningToRankRescorerFeature.isEnabled()) { + if (machineLearningExtension.get().isLearningToRankEnabled()) { namedXContent.addAll(new MlLTRNamedXContentProvider().getNamedXContentParsers()); } return namedXContent; @@ -1889,7 +1894,7 @@ public List getNamedWriteables() { namedWriteables.addAll(new CorrelationNamedContentProvider().getNamedWriteables()); namedWriteables.addAll(new ChangePointNamedContentProvider().getNamedWriteables()); // LTR Combine with Inference named content provider when feature flag is removed - if (LearningToRankRescorerFeature.isEnabled()) { + if (machineLearningExtension.get().isLearningToRankEnabled()) { namedWriteables.addAll(new MlLTRNamedXContentProvider().getNamedWriteables()); } return namedWriteables; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java index 552344b4ef10e..c27568c6e3b5c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java @@ -25,6 +25,10 @@ default void configure(Settings settings) {} boolean isNlpEnabled(); + default boolean isLearningToRankEnabled() { + return false; + } + String[] getAnalyticsDestIndexAllowedSettings(); AbstractNodeAvailabilityZoneMapper getNodeAvailabilityZoneMapper(Settings settings, ClusterSettings clusterSettings); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java index fe4462d6556ee..a1ac1aa55c320 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java @@ -1099,31 +1099,61 @@ static boolean haveMlNodesChanged(ClusterChangedEvent event, TrainedModelAssignm // it may get re-allocated to that node when another node is added/removed... boolean nodesShutdownChanged = event.changedCustomMetadataSet().contains(NodesShutdownMetadata.TYPE); if (event.nodesChanged() || nodesShutdownChanged) { + // This is just to track the various log messages that happen in this function to help with debugging in the future + // so that we can reasonably assume they're all related + // If the log messages printed from this method get interlaced across nodes it can make debugging difficult + var eventIdentity = Long.toHexString(System.nanoTime()); + Set shuttingDownNodes = nodesShuttingDown(event.state()); DiscoveryNodes.Delta nodesDelta = event.nodesDelta(); Set removedNodes = nodesDelta.removedNodes().stream().map(DiscoveryNode::getId).collect(Collectors.toSet()); Set addedNodes = nodesDelta.addedNodes().stream().map(DiscoveryNode::getId).collect(Collectors.toSet()); + logger.debug( + () -> format( + "Initial node change info; identity: %s; removed nodes: %s; added nodes: %s; shutting down nodes: %s", + eventIdentity, + removedNodes, + addedNodes, + shuttingDownNodes + ) + ); + Set exitingShutDownNodes; if (nodesShutdownChanged) { Set previousShuttingDownNodes = nodesShuttingDown(event.previousState()); + Set presentNodes = event.state().nodes().stream().map(DiscoveryNode::getId).collect(Collectors.toSet()); // Add nodes that where marked for shutdown in the previous state // but are no longer marked as shutdown in the current state. - Set returningShutDownNodes = Sets.difference(previousShuttingDownNodes, shuttingDownNodes); + // The intersection is to only include the nodes that actually exist + Set returningShutDownNodes = Sets.intersection( + presentNodes, + Sets.difference(previousShuttingDownNodes, shuttingDownNodes) + ); addedNodes.addAll(returningShutDownNodes); // and nodes that are marked for shutdown in this event only exitingShutDownNodes = Sets.difference(shuttingDownNodes, previousShuttingDownNodes); removedNodes.addAll(exitingShutDownNodes); + + logger.debug( + () -> format( + "Shutting down nodes were changed; identity: %s; previous shutting down nodes: %s; returning nodes: %s", + eventIdentity, + previousShuttingDownNodes, + returningShutDownNodes + ) + ); } else { exitingShutDownNodes = Collections.emptySet(); } logger.debug( () -> format( - "added nodes %s; removed nodes %s; shutting down nodes %s; exiting shutdown nodes %s", + "identity: %s; added nodes %s; removed nodes %s; shutting down nodes %s; exiting shutdown nodes %s", + eventIdentity, addedNodes, removedNodes, shuttingDownNodes, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerFeature.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerFeature.java deleted file mode 100644 index 42598691beec2..0000000000000 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerFeature.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.ml.inference.ltr; - -import org.elasticsearch.common.util.FeatureFlag; - -/** - * Learning to rank feature flag. When the feature is complete, this flag will be removed. - * - * Upon removal, ensure transport serialization is all corrected for future BWC. - * - * See {@link LearningToRankRescorerBuilder} - */ -public class LearningToRankRescorerFeature { - - private LearningToRankRescorerFeature() {} - - private static final FeatureFlag LEARNING_TO_RANK = new FeatureFlag("learning_to_rank"); - - public static boolean isEnabled() { - return LEARNING_TO_RANK.isEnabled(); - } -} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilder.java index 12019e93ba713..24383e51b0ed2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilder.java @@ -15,8 +15,8 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.query.AbstractQueryBuilder; -import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryRewriteContext; @@ -41,6 +41,7 @@ public class TextExpansionQueryBuilder extends AbstractQueryBuilder { public static final String NAME = "text_expansion"; + public static final ParseField PRUNING_CONFIG = new ParseField("pruning_config"); public static final ParseField MODEL_TEXT = new ParseField("model_text"); public static final ParseField MODEL_ID = new ParseField("model_id"); @@ -48,8 +49,13 @@ public class TextExpansionQueryBuilder extends AbstractQueryBuilder weightedTokensSupplier; + private final TokenPruningConfig tokenPruningConfig; public TextExpansionQueryBuilder(String fieldName, String modelText, String modelId) { + this(fieldName, modelText, modelId, null); + } + + public TextExpansionQueryBuilder(String fieldName, String modelText, String modelId, @Nullable TokenPruningConfig tokenPruningConfig) { if (fieldName == null) { throw new IllegalArgumentException("[" + NAME + "] requires a fieldName"); } @@ -59,10 +65,10 @@ public TextExpansionQueryBuilder(String fieldName, String modelText, String mode if (modelId == null) { throw new IllegalArgumentException("[" + NAME + "] requires a " + MODEL_ID.getPreferredName() + " value"); } - this.fieldName = fieldName; this.modelText = modelText; this.modelId = modelId; + this.tokenPruningConfig = tokenPruningConfig; } public TextExpansionQueryBuilder(StreamInput in) throws IOException { @@ -70,12 +76,18 @@ public TextExpansionQueryBuilder(StreamInput in) throws IOException { this.fieldName = in.readString(); this.modelText = in.readString(); this.modelId = in.readString(); + if (in.getTransportVersion().onOrAfter(TransportVersions.TEXT_EXPANSION_TOKEN_PRUNING_CONFIG_ADDED)) { + this.tokenPruningConfig = in.readOptionalWriteable(TokenPruningConfig::new); + } else { + this.tokenPruningConfig = null; + } } private TextExpansionQueryBuilder(TextExpansionQueryBuilder other, SetOnce weightedTokensSupplier) { this.fieldName = other.fieldName; this.modelText = other.modelText; this.modelId = other.modelId; + this.tokenPruningConfig = other.tokenPruningConfig; this.boost = other.boost; this.queryName = other.queryName; this.weightedTokensSupplier = weightedTokensSupplier; @@ -85,6 +97,10 @@ String getFieldName() { return fieldName; } + public TokenPruningConfig getTokenPruningConfig() { + return tokenPruningConfig; + } + @Override public String getWriteableName() { return NAME; @@ -103,6 +119,9 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeString(fieldName); out.writeString(modelText); out.writeString(modelId); + if (out.getTransportVersion().onOrAfter(TransportVersions.TEXT_EXPANSION_TOKEN_PRUNING_CONFIG_ADDED)) { + out.writeOptionalWriteable(tokenPruningConfig); + } } @Override @@ -111,6 +130,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.startObject(fieldName); builder.field(MODEL_TEXT.getPreferredName(), modelText); builder.field(MODEL_ID.getPreferredName(), modelId); + if (tokenPruningConfig != null) { + builder.field(PRUNING_CONFIG.getPreferredName(), tokenPruningConfig); + } boostAndQueryNameToXContent(builder); builder.endObject(); builder.endObject(); @@ -174,21 +196,33 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws return new TextExpansionQueryBuilder(this, textExpansionResultsSupplier); } - static BoolQueryBuilder weightedTokensToQuery( + private QueryBuilder weightedTokensToQuery( String fieldName, TextExpansionResults textExpansionResults, QueryRewriteContext queryRewriteContext - ) throws IOException { + ) { + if (tokenPruningConfig != null) { + WeightedTokensQueryBuilder weightedTokensQueryBuilder = new WeightedTokensQueryBuilder( + fieldName, + textExpansionResults.getWeightedTokens(), + tokenPruningConfig + ); + weightedTokensQueryBuilder.queryName(queryName); + weightedTokensQueryBuilder.boost(boost); + return weightedTokensQueryBuilder; + } var boolQuery = QueryBuilders.boolQuery(); for (var weightedToken : textExpansionResults.getWeightedTokens()) { boolQuery.should(QueryBuilders.termQuery(fieldName, weightedToken.token()).boost(weightedToken.weight())); } boolQuery.minimumShouldMatch(1); + boolQuery.boost(this.boost); + boolQuery.queryName(this.queryName); return boolQuery; } @Override - protected Query doToQuery(SearchExecutionContext context) throws IOException { + protected Query doToQuery(SearchExecutionContext context) { throw new IllegalStateException("text_expansion should have been rewritten to another query type"); } @@ -197,18 +231,20 @@ protected boolean doEquals(TextExpansionQueryBuilder other) { return Objects.equals(fieldName, other.fieldName) && Objects.equals(modelText, other.modelText) && Objects.equals(modelId, other.modelId) + && Objects.equals(tokenPruningConfig, other.tokenPruningConfig) && Objects.equals(weightedTokensSupplier, other.weightedTokensSupplier); } @Override protected int doHashCode() { - return Objects.hash(fieldName, modelText, modelId, weightedTokensSupplier); + return Objects.hash(fieldName, modelText, modelId, tokenPruningConfig, weightedTokensSupplier); } public static TextExpansionQueryBuilder fromXContent(XContentParser parser) throws IOException { String fieldName = null; String modelText = null; String modelId = null; + TokenPruningConfig tokenPruningConfig = null; float boost = AbstractQueryBuilder.DEFAULT_BOOST; String queryName = null; String currentFieldName = null; @@ -222,6 +258,15 @@ public static TextExpansionQueryBuilder fromXContent(XContentParser parser) thro while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + if (PRUNING_CONFIG.match(currentFieldName, parser.getDeprecationHandler())) { + tokenPruningConfig = TokenPruningConfig.fromXContent(parser); + } else { + throw new ParsingException( + parser.getTokenLocation(), + "[" + NAME + "] unknown token [" + token + "] after [" + currentFieldName + "]" + ); + } } else if (token.isValue()) { if (MODEL_TEXT.match(currentFieldName, parser.getDeprecationHandler())) { modelText = parser.text(); @@ -259,7 +304,7 @@ public static TextExpansionQueryBuilder fromXContent(XContentParser parser) thro throw new ParsingException(parser.getTokenLocation(), "No fieldname specified for query"); } - TextExpansionQueryBuilder queryBuilder = new TextExpansionQueryBuilder(fieldName, modelText, modelId); + TextExpansionQueryBuilder queryBuilder = new TextExpansionQueryBuilder(fieldName, modelText, modelId, tokenPruningConfig); queryBuilder.queryName(queryName); queryBuilder.boost(boost); return queryBuilder; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TokenPruningConfig.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TokenPruningConfig.java new file mode 100644 index 0000000000000..d789a645fd9c4 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TokenPruningConfig.java @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.queries; + +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; + +import static org.elasticsearch.xpack.ml.queries.TextExpansionQueryBuilder.PRUNING_CONFIG; + +public class TokenPruningConfig implements Writeable, ToXContentObject { + public static final ParseField TOKENS_FREQ_RATIO_THRESHOLD = new ParseField("tokens_freq_ratio_threshold"); + public static final ParseField TOKENS_WEIGHT_THRESHOLD = new ParseField("tokens_weight_threshold"); + public static final ParseField ONLY_SCORE_PRUNED_TOKENS_FIELD = new ParseField("only_score_pruned_tokens"); + + // Tokens whose frequency is more than 5 times the average frequency of all tokens in the specified field are considered outliers. + public static final float DEFAULT_TOKENS_FREQ_RATIO_THRESHOLD = 5; + public static final float MAX_TOKENS_FREQ_RATIO_THRESHOLD = 100; + // A token's weight should be > 40% of the best weight in the query to be considered significant. + public static final float DEFAULT_TOKENS_WEIGHT_THRESHOLD = 0.4f; + + private final float tokensFreqRatioThreshold; + private final float tokensWeightThreshold; + private final boolean onlyScorePrunedTokens; + + public TokenPruningConfig() { + this(DEFAULT_TOKENS_FREQ_RATIO_THRESHOLD, DEFAULT_TOKENS_WEIGHT_THRESHOLD, false); + } + + public TokenPruningConfig(float tokensFreqRatioThreshold, float tokensWeightThreshold, boolean onlyScorePrunedTokens) { + if (tokensFreqRatioThreshold < 1 || tokensFreqRatioThreshold > MAX_TOKENS_FREQ_RATIO_THRESHOLD) { + throw new IllegalArgumentException( + "[" + + TOKENS_FREQ_RATIO_THRESHOLD.getPreferredName() + + "] must be between [1.0] and [" + + String.format(Locale.ROOT, "%.1f", MAX_TOKENS_FREQ_RATIO_THRESHOLD) + + "], got " + + tokensFreqRatioThreshold + ); + } + if (tokensWeightThreshold < 0 || tokensWeightThreshold > 1) { + throw new IllegalArgumentException("[" + TOKENS_WEIGHT_THRESHOLD.getPreferredName() + "] must be between 0 and 1"); + } + this.tokensFreqRatioThreshold = tokensFreqRatioThreshold; + this.tokensWeightThreshold = tokensWeightThreshold; + this.onlyScorePrunedTokens = onlyScorePrunedTokens; + } + + public TokenPruningConfig(StreamInput in) throws IOException { + this.tokensFreqRatioThreshold = in.readFloat(); + this.tokensWeightThreshold = in.readFloat(); + this.onlyScorePrunedTokens = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeFloat(tokensFreqRatioThreshold); + out.writeFloat(tokensWeightThreshold); + out.writeBoolean(onlyScorePrunedTokens); + } + + /** + * Returns the frequency ratio threshold to apply on the query. + * Tokens whose frequency is more than ratio_threshold times the average frequency of all tokens in the specified + * field are considered outliers and may be subject to removal from the query. + */ + public float getTokensFreqRatioThreshold() { + return tokensFreqRatioThreshold; + } + + /** + * Returns the weight threshold to apply on the query. + * Tokens whose weight is more than (weightThreshold * best_weight) of the highest weight in the query are not + * considered outliers, even if their frequency exceeds the specified ratio_threshold. + * This threshold ensures that important tokens, as indicated by their weight, are retained in the query. + */ + public float getTokensWeightThreshold() { + return tokensWeightThreshold; + } + + /** + * Returns whether the filtering process retains tokens identified as non-relevant based on the specified thresholds + * (ratio and weight). When {@code true}, only non-relevant tokens are considered for matching and scoring documents. + * Enabling this option is valuable for re-scoring top hits retrieved from a {@link WeightedTokensQueryBuilder} with + * active thresholds. + */ + public boolean isOnlyScorePrunedTokens() { + return onlyScorePrunedTokens; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TokenPruningConfig that = (TokenPruningConfig) o; + return Float.compare(that.tokensFreqRatioThreshold, tokensFreqRatioThreshold) == 0 + && Float.compare(that.tokensWeightThreshold, tokensWeightThreshold) == 0 + && onlyScorePrunedTokens == that.onlyScorePrunedTokens; + } + + @Override + public int hashCode() { + return Objects.hash(tokensFreqRatioThreshold, tokensWeightThreshold, onlyScorePrunedTokens); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(TOKENS_FREQ_RATIO_THRESHOLD.getPreferredName(), tokensFreqRatioThreshold); + builder.field(TOKENS_WEIGHT_THRESHOLD.getPreferredName(), tokensWeightThreshold); + if (onlyScorePrunedTokens) { + builder.field(ONLY_SCORE_PRUNED_TOKENS_FIELD.getPreferredName(), onlyScorePrunedTokens); + } + builder.endObject(); + return builder; + } + + public static TokenPruningConfig fromXContent(XContentParser parser) throws IOException { + String currentFieldName = null; + XContentParser.Token token; + float ratioThreshold = DEFAULT_TOKENS_FREQ_RATIO_THRESHOLD; + float weightThreshold = DEFAULT_TOKENS_WEIGHT_THRESHOLD; + boolean onlyScorePrunedTokens = false; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.START_OBJECT) { + continue; + } + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + if (Set.of( + TOKENS_FREQ_RATIO_THRESHOLD.getPreferredName(), + TOKENS_WEIGHT_THRESHOLD.getPreferredName(), + ONLY_SCORE_PRUNED_TOKENS_FIELD.getPreferredName() + ).contains(currentFieldName) == false) { + throw new ParsingException( + parser.getTokenLocation(), + "[" + PRUNING_CONFIG.getPreferredName() + "] unknown token [" + currentFieldName + "]" + ); + } + } else if (token.isValue()) { + if (TOKENS_FREQ_RATIO_THRESHOLD.match(currentFieldName, parser.getDeprecationHandler())) { + ratioThreshold = parser.intValue(); + } else if (TOKENS_WEIGHT_THRESHOLD.match(currentFieldName, parser.getDeprecationHandler())) { + weightThreshold = parser.floatValue(); + } else if (ONLY_SCORE_PRUNED_TOKENS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + onlyScorePrunedTokens = parser.booleanValue(); + } else { + throw new ParsingException( + parser.getTokenLocation(), + "[" + PRUNING_CONFIG.getPreferredName() + "] does not support [" + currentFieldName + "]" + ); + } + } else { + throw new ParsingException( + parser.getTokenLocation(), + "[" + PRUNING_CONFIG.getPreferredName() + "] unknown token [" + token + "] after [" + currentFieldName + "]" + ); + } + } + return new TokenPruningConfig(ratioThreshold, weightThreshold, onlyScorePrunedTokens); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/WeightedTokensQueryBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/WeightedTokensQueryBuilder.java new file mode 100644 index 0000000000000..a09bcadaacfc0 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/WeightedTokensQueryBuilder.java @@ -0,0 +1,264 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.queries; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults.WeightedToken; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.xpack.ml.queries.TextExpansionQueryBuilder.PRUNING_CONFIG; + +public class WeightedTokensQueryBuilder extends AbstractQueryBuilder { + public static final String NAME = "weighted_tokens"; + + public static final ParseField TOKENS_FIELD = new ParseField("tokens"); + private final String fieldName; + private final List tokens; + @Nullable + private final TokenPruningConfig tokenPruningConfig; + + public WeightedTokensQueryBuilder(String fieldName, List tokens) { + this(fieldName, tokens, null); + } + + public WeightedTokensQueryBuilder(String fieldName, List tokens, @Nullable TokenPruningConfig tokenPruningConfig) { + this.fieldName = Objects.requireNonNull(fieldName, "[" + NAME + "] requires a fieldName"); + this.tokens = Objects.requireNonNull(tokens, "[" + NAME + "] requires tokens"); + if (tokens.isEmpty()) { + throw new IllegalArgumentException("[" + NAME + "] requires at least one token"); + } + this.tokenPruningConfig = tokenPruningConfig; + } + + public WeightedTokensQueryBuilder(StreamInput in) throws IOException { + super(in); + this.fieldName = in.readString(); + this.tokens = in.readCollectionAsList(WeightedToken::new); + this.tokenPruningConfig = in.readOptionalWriteable(TokenPruningConfig::new); + } + + public String getFieldName() { + return fieldName; + } + + @Nullable + public TokenPruningConfig getTokenPruningConfig() { + return tokenPruningConfig; + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeString(fieldName); + out.writeCollection(tokens); + out.writeOptionalWriteable(tokenPruningConfig); + } + + @Override + protected void doXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + builder.startObject(fieldName); + builder.startObject(TOKENS_FIELD.getPreferredName()); + for (var token : tokens) { + token.toXContent(builder, params); + } + builder.endObject(); + if (tokenPruningConfig != null) { + builder.field(PRUNING_CONFIG.getPreferredName(), tokenPruningConfig); + } + boostAndQueryNameToXContent(builder); + builder.endObject(); + builder.endObject(); + } + + /** + * We calculate the maximum number of unique tokens for any shard of data. The maximum is used to compute + * average token frequency since we don't have a unique inter-segment token count. + * Once we have the maximum number of unique tokens, we use the total count of tokens in the index to calculate + * the average frequency ratio. + * + * @param reader + * @param fieldDocCount + * @return float + * @throws IOException + */ + private float getAverageTokenFreqRatio(IndexReader reader, int fieldDocCount) throws IOException { + int numUniqueTokens = 0; + for (var leaf : reader.getContext().leaves()) { + var terms = leaf.reader().terms(fieldName); + if (terms != null) { + numUniqueTokens = (int) Math.max(terms.size(), numUniqueTokens); + } + } + if (numUniqueTokens == 0) { + return 0; + } + return (float) reader.getSumDocFreq(fieldName) / fieldDocCount / numUniqueTokens; + } + + /** + * Returns true if the token should be queried based on the {@code tokensFreqRatioThreshold} and {@code tokensWeightThreshold} + * set on the query. + */ + private boolean shouldKeepToken( + IndexReader reader, + WeightedToken token, + int fieldDocCount, + float averageTokenFreqRatio, + float bestWeight + ) throws IOException { + if (this.tokenPruningConfig == null) { + return true; + } + int docFreq = reader.docFreq(new Term(fieldName, token.token())); + if (docFreq == 0) { + return false; + } + float tokenFreqRatio = (float) docFreq / fieldDocCount; + return tokenFreqRatio < this.tokenPruningConfig.getTokensFreqRatioThreshold() * averageTokenFreqRatio + || token.weight() > this.tokenPruningConfig.getTokensWeightThreshold() * bestWeight; + } + + @Override + protected Query doToQuery(SearchExecutionContext context) throws IOException { + final MappedFieldType ft = context.getFieldType(fieldName); + if (ft == null) { + return new MatchNoDocsQuery("The \"" + getName() + "\" query is against a field that does not exist"); + } + var qb = new BooleanQuery.Builder(); + int fieldDocCount = context.getIndexReader().getDocCount(fieldName); + float bestWeight = 0f; + for (var t : tokens) { + bestWeight = Math.max(t.weight(), bestWeight); + } + float averageTokenFreqRatio = getAverageTokenFreqRatio(context.getIndexReader(), fieldDocCount); + if (averageTokenFreqRatio == 0) { + return new MatchNoDocsQuery("The \"" + getName() + "\" query is against an empty field"); + } + for (var token : tokens) { + boolean keep = shouldKeepToken(context.getIndexReader(), token, fieldDocCount, averageTokenFreqRatio, bestWeight); + if (this.tokenPruningConfig != null) { + keep ^= this.tokenPruningConfig.isOnlyScorePrunedTokens(); + } + if (keep) { + qb.add(new BoostQuery(ft.termQuery(token.token(), context), token.weight()), BooleanClause.Occur.SHOULD); + } + } + qb.setMinimumNumberShouldMatch(1); + return qb.build(); + } + + @Override + protected boolean doEquals(WeightedTokensQueryBuilder other) { + return Objects.equals(fieldName, other.fieldName) + && Objects.equals(tokenPruningConfig, other.tokenPruningConfig) + && tokens.equals(other.tokens); + } + + @Override + protected int doHashCode() { + return Objects.hash(fieldName, tokens, tokenPruningConfig); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.TEXT_EXPANSION_TOKEN_PRUNING_CONFIG_ADDED; + } + + private static float parseWeight(String token, Object weight) throws IOException { + if (weight instanceof Number asNumber) { + return asNumber.floatValue(); + } + if (weight instanceof String asString) { + return Float.parseFloat(asString); + } + throw new ElasticsearchParseException( + "Illegal weight for token: [" + token + "], expected floating point got " + weight.getClass().getSimpleName() + ); + } + + public static WeightedTokensQueryBuilder fromXContent(XContentParser parser) throws IOException { + String currentFieldName = null; + String fieldName = null; + List tokens = new ArrayList<>(); + TokenPruningConfig tokenPruningConfig = null; + float boost = AbstractQueryBuilder.DEFAULT_BOOST; + String queryName = null; + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + throwParsingExceptionOnMultipleFields(NAME, parser.getTokenLocation(), fieldName, currentFieldName); + fieldName = currentFieldName; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (PRUNING_CONFIG.match(currentFieldName, parser.getDeprecationHandler())) { + if (token != XContentParser.Token.START_OBJECT) { + throw new ParsingException( + parser.getTokenLocation(), + "[" + PRUNING_CONFIG.getPreferredName() + "] should be an object" + ); + } + tokenPruningConfig = TokenPruningConfig.fromXContent(parser); + } else if (TOKENS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + var tokensMap = parser.map(); + for (var e : tokensMap.entrySet()) { + tokens.add(new WeightedToken(e.getKey(), parseWeight(e.getKey(), e.getValue()))); + } + } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + boost = parser.floatValue(); + } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + queryName = parser.text(); + } else { + throw new ParsingException(parser.getTokenLocation(), "unknown field [" + currentFieldName + "]"); + } + } + } else { + throw new IllegalArgumentException("invalid query"); + } + } + + if (fieldName == null) { + throw new ParsingException(parser.getTokenLocation(), "No fieldname specified for query"); + } + + var qb = new WeightedTokensQueryBuilder(fieldName, tokens, tokenPruningConfig); + qb.queryName(queryName); + qb.boost(boost); + return qb; + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningAdOnly.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningAdOnly.java index 3ff3a4a404f97..175a035a70f7e 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningAdOnly.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningAdOnly.java @@ -14,6 +14,6 @@ public class LocalStateMachineLearningAdOnly extends LocalStateMachineLearning { public LocalStateMachineLearningAdOnly(final Settings settings, final Path configPath) { - super(settings, configPath, new MlTestExtensionLoader(new MlTestExtension(true, true, true, false, false))); + super(settings, configPath, new MlTestExtensionLoader(new MlTestExtension(true, true, true, false, false, false))); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningDfaOnly.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningDfaOnly.java index 1a72f27865d8a..f054e52dc29ec 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningDfaOnly.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningDfaOnly.java @@ -14,6 +14,6 @@ public class LocalStateMachineLearningDfaOnly extends LocalStateMachineLearning { public LocalStateMachineLearningDfaOnly(final Settings settings, final Path configPath) { - super(settings, configPath, new MlTestExtensionLoader(new MlTestExtension(true, true, false, true, false))); + super(settings, configPath, new MlTestExtensionLoader(new MlTestExtension(true, true, false, true, false, false))); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningNlpOnly.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningNlpOnly.java index 0f11e8033b83d..a3d684011e932 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningNlpOnly.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningNlpOnly.java @@ -14,6 +14,6 @@ public class LocalStateMachineLearningNlpOnly extends LocalStateMachineLearning { public LocalStateMachineLearningNlpOnly(final Settings settings, final Path configPath) { - super(settings, configPath, new MlTestExtensionLoader(new MlTestExtension(true, true, false, false, true))); + super(settings, configPath, new MlTestExtensionLoader(new MlTestExtension(true, true, false, false, true, false))); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java index 08568bd3e0d1e..084a9d95939c5 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java @@ -160,7 +160,14 @@ private MachineLearningUsageTransportAction newUsageAction( licenseState, jobManagerHolder, new MachineLearningExtensionHolder( - new MachineLearningTests.MlTestExtension(true, true, isAnomalyDetectionEnabled, isDataFrameAnalyticsEnabled, isNlpEnabled) + new MachineLearningTests.MlTestExtension( + true, + true, + isAnomalyDetectionEnabled, + isDataFrameAnalyticsEnabled, + isNlpEnabled, + true + ) ) ); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java index 84cef907cd093..f5f81a5ca15f3 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java @@ -221,9 +221,8 @@ public void testNoAttributes_givenClash() throws IOException { public void testAnomalyDetectionOnly() throws IOException { Settings settings = Settings.builder().put("path.home", createTempDir()).build(); - try (MachineLearning machineLearning = createTrialLicensedMachineLearning(settings)) { - MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, true, false, false)); - machineLearning.loadExtensions(loader); + MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, true, false, false, false)); + try (MachineLearning machineLearning = createTrialLicensedMachineLearning(settings, loader)) { List restHandlers = machineLearning.getRestHandlers(settings, null, null, null, null, null, null); assertThat(restHandlers, hasItem(instanceOf(RestMlInfoAction.class))); assertThat(restHandlers, hasItem(instanceOf(RestGetJobsAction.class))); @@ -242,9 +241,8 @@ public void testAnomalyDetectionOnly() throws IOException { public void testDataFrameAnalyticsOnly() throws IOException { Settings settings = Settings.builder().put("path.home", createTempDir()).build(); - try (MachineLearning machineLearning = createTrialLicensedMachineLearning(settings)) { - MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, false, true, false)); - machineLearning.loadExtensions(loader); + MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, false, true, false, false)); + try (MachineLearning machineLearning = createTrialLicensedMachineLearning(settings, loader)) { List restHandlers = machineLearning.getRestHandlers(settings, null, null, null, null, null, null); assertThat(restHandlers, hasItem(instanceOf(RestMlInfoAction.class))); assertThat(restHandlers, not(hasItem(instanceOf(RestGetJobsAction.class)))); @@ -263,9 +261,8 @@ public void testDataFrameAnalyticsOnly() throws IOException { public void testNlpOnly() throws IOException { Settings settings = Settings.builder().put("path.home", createTempDir()).build(); - try (MachineLearning machineLearning = createTrialLicensedMachineLearning(settings)) { - MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, false, false, true)); - machineLearning.loadExtensions(loader); + MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, false, false, true, false)); + try (MachineLearning machineLearning = createTrialLicensedMachineLearning(settings, loader)) { List restHandlers = machineLearning.getRestHandlers(settings, null, null, null, null, null, null); assertThat(restHandlers, hasItem(instanceOf(RestMlInfoAction.class))); assertThat(restHandlers, not(hasItem(instanceOf(RestGetJobsAction.class)))); @@ -291,19 +288,22 @@ public static class MlTestExtension implements MachineLearningExtension { private final boolean isAnomalyDetectionEnabled; private final boolean isDataFrameAnalyticsEnabled; private final boolean isNlpEnabled; + private final boolean isLearningToRankEnabled; MlTestExtension( boolean useIlm, boolean includeNodeInfo, boolean isAnomalyDetectionEnabled, boolean isDataFrameAnalyticsEnabled, - boolean isNlpEnabled + boolean isNlpEnabled, + boolean isLearningToRankEnabled ) { this.useIlm = useIlm; this.includeNodeInfo = includeNodeInfo; this.isAnomalyDetectionEnabled = isAnomalyDetectionEnabled; this.isDataFrameAnalyticsEnabled = isDataFrameAnalyticsEnabled; this.isNlpEnabled = isNlpEnabled; + this.isLearningToRankEnabled = isLearningToRankEnabled; } @Override @@ -331,6 +331,11 @@ public boolean isNlpEnabled() { return isNlpEnabled; } + @Override + public boolean isLearningToRankEnabled() { + return isLearningToRankEnabled; + } + @Override public String[] getAnalyticsDestIndexAllowedSettings() { return ANALYTICS_DEST_INDEX_ALLOWED_SETTINGS; @@ -377,6 +382,12 @@ protected XPackLicenseState getLicenseState() { } public static MachineLearning createTrialLicensedMachineLearning(Settings settings) { - return new TrialLicensedMachineLearning(settings); + return createTrialLicensedMachineLearning(settings, null); + } + + public static MachineLearning createTrialLicensedMachineLearning(Settings settings, MlTestExtensionLoader loader) { + MachineLearning mlPlugin = new TrialLicensedMachineLearning(settings); + mlPlugin.loadExtensions(loader); + return mlPlugin; } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java index 196584e4a7ce2..cb5b98af29d57 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.search.aggregations.metrics.Min; import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import java.io.IOException; import java.util.Arrays; @@ -53,7 +54,7 @@ protected AnalysisModule createAnalysisModule() throws Exception { @Override protected List getSearchPlugins() { - return List.of(new MachineLearning(Settings.EMPTY)); + return List.of(MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY)); } private static final String TEXT_FIELD_NAME = "text"; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/InternalCategorizationAggregationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/InternalCategorizationAggregationTests.java index 1f9119463994d..be8b72d26fd71 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/InternalCategorizationAggregationTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/InternalCategorizationAggregationTests.java @@ -18,7 +18,7 @@ import org.elasticsearch.test.InternalMultiBucketAggregationTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import org.junit.After; import org.junit.Before; @@ -49,7 +49,7 @@ public void destroyHash() { @Override protected SearchPlugin registerPlugin() { - return new MachineLearning(Settings.EMPTY); + return MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregatorTests.java index 410fc474a503f..442641db8c4ed 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregatorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregatorTests.java @@ -23,7 +23,7 @@ import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import java.io.IOException; import java.util.Arrays; @@ -40,7 +40,7 @@ public class ChangePointAggregatorTests extends AggregatorTestCase { @Override protected List getSearchPlugins() { - return List.of(new MachineLearning(Settings.EMPTY)); + return List.of(MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY)); } private static final DateHistogramInterval INTERVAL = DateHistogramInterval.minutes(1); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregatorTests.java index 5a3c8fcc5f7c7..93ef45285b60e 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregatorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregatorTests.java @@ -32,7 +32,7 @@ import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.MultiValuesSourceFieldConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceType; -import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import org.elasticsearch.xpack.ml.aggs.frequentitemsets.EclatMapReducer.EclatResult; import org.elasticsearch.xpack.ml.aggs.frequentitemsets.FrequentItemSetCollector.FrequentItemSet; import org.elasticsearch.xpack.ml.aggs.frequentitemsets.mr.InternalItemSetMapReduceAggregation; @@ -66,7 +66,7 @@ public class FrequentItemSetsAggregatorTests extends AggregatorTestCase { @Override protected List getSearchPlugins() { - return List.of(new MachineLearning(Settings.EMPTY)); + return List.of(MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY)); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregationTests.java index 93ee8bec974a7..ccaa6b4f0f4ec 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregationTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregationTests.java @@ -26,7 +26,7 @@ import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import org.elasticsearch.xpack.ml.aggs.frequentitemsets.mr.InternalItemSetMapReduceAggregationTests.WordCountMapReducer.WordCounts; import org.elasticsearch.xpack.ml.aggs.frequentitemsets.mr.ItemSetMapReduceValueSource.Field; import org.elasticsearch.xpack.ml.aggs.frequentitemsets.mr.ItemSetMapReduceValueSource.ValueFormatter; @@ -247,7 +247,7 @@ protected void assertFromXContent( @Override protected SearchPlugin registerPlugin() { - return new MachineLearning(Settings.EMPTY); + return MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/heuristic/PValueScoreTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/heuristic/PValueScoreTests.java index 17898d7205b66..a6de69b684e12 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/heuristic/PValueScoreTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/heuristic/PValueScoreTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import java.util.List; import java.util.function.Function; @@ -56,16 +57,14 @@ public void testAssertions() { @Override protected NamedXContentRegistry xContentRegistry() { - return new NamedXContentRegistry( - new SearchModule(Settings.EMPTY, List.of(new MachineLearning(Settings.EMPTY))).getNamedXContents() - ); + MachineLearning mlPlugin = MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY); + return new NamedXContentRegistry(new SearchModule(Settings.EMPTY, List.of(mlPlugin)).getNamedXContents()); } @Override protected NamedWriteableRegistry writableRegistry() { - return new NamedWriteableRegistry( - new SearchModule(Settings.EMPTY, List.of(new MachineLearning(Settings.EMPTY))).getNamedWriteables() - ); + MachineLearning mlPlugin = MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY); + return new NamedWriteableRegistry(new SearchModule(Settings.EMPTY, List.of(mlPlugin)).getNamedWriteables()); } public void testPValueScore_WhenAllDocsContainTerm() { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InferencePipelineAggregationBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InferencePipelineAggregationBuilderTests.java index 4aed2261b89ef..8f1ef07b554da 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InferencePipelineAggregationBuilderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InferencePipelineAggregationBuilderTests.java @@ -23,10 +23,9 @@ import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfigUpdateTests; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ResultsFieldUpdate; -import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -43,7 +42,7 @@ public class InferencePipelineAggregationBuilderTests extends BasePipelineAggreg @Override protected List plugins() { - return Collections.singletonList(new MachineLearning(Settings.EMPTY)); + return List.of(MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY)); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InternalInferenceAggregationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InternalInferenceAggregationTests.java index 016a89fe4e4b7..fc1858167e7d7 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InternalInferenceAggregationTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InternalInferenceAggregationTests.java @@ -26,7 +26,7 @@ import org.elasticsearch.xpack.core.ml.inference.results.TopClassEntry; import org.elasticsearch.xpack.core.ml.inference.results.WarningInferenceResults; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ClassificationConfig; -import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import java.util.Arrays; import java.util.Collections; @@ -40,7 +40,7 @@ public class InternalInferenceAggregationTests extends InternalAggregationTestCa @Override protected SearchPlugin registerPlugin() { - return new MachineLearning(Settings.EMPTY); + return MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/kstest/InternalKSTestAggregationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/kstest/InternalKSTestAggregationTests.java index 2085e44925cc9..f064b37c1fdec 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/kstest/InternalKSTestAggregationTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/kstest/InternalKSTestAggregationTests.java @@ -14,7 +14,7 @@ import org.elasticsearch.search.aggregations.ParsedAggregation; import org.elasticsearch.test.InternalAggregationTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import java.util.Arrays; import java.util.List; @@ -28,7 +28,7 @@ public class InternalKSTestAggregationTests extends InternalAggregationTestCase< @Override protected SearchPlugin registerPlugin() { - return new MachineLearning(Settings.EMPTY); + return MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java index 0a54b97cf2f2a..f966ac85c7a65 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java @@ -114,7 +114,7 @@ public class TrainedModelAssignmentClusterServiceTests extends ESTestCase { private NodeAvailabilityZoneMapper nodeAvailabilityZoneMapper; private Client client; private static MockAppender appender; - private static Logger testLogger1 = LogManager.getLogger(TrainedModelAssignmentClusterService.class); + private static final Logger testLogger1 = LogManager.getLogger(TrainedModelAssignmentClusterService.class); @Before public void setupObjects() throws IllegalAccessException { @@ -546,6 +546,43 @@ public void testCreateAssignmentWhileResetModeIsTrue() throws InterruptedExcepti latch.await(); } + public void testHaveMlNodesChanged_ReturnsFalseWhenPreviouslyShuttingDownNode_IsMarkedAsReturning_ButIsNotAPresentNode() { + String model1 = "model-1"; + String shuttingDownNode = "ml-shutting-down-node"; + String mlNode1 = "ml-node-with-room"; + + ClusterState stateWithShuttingDownNodeAndMlNode1 = createClusterState( + List.of(shuttingDownNode, mlNode1), + Metadata.builder() + .putCustom( + TrainedModelAssignmentMetadata.NAME, + TrainedModelAssignmentMetadata.Builder.empty() + .addNewAssignment( + model1, + TrainedModelAssignment.Builder.empty(newParams(model1, 100)) + .addRoutingEntry(mlNode1, new RoutingInfo(1, 1, RoutingState.STARTING, "")) + ) + .build() + ) + .putCustom(NodesShutdownMetadata.TYPE, shutdownMetadata(shuttingDownNode)) + .build() + ); + + ClusterState stateWithMlNode1 = ClusterState.builder(stateWithShuttingDownNodeAndMlNode1) + .nodes(DiscoveryNodes.builder(stateWithShuttingDownNodeAndMlNode1.nodes()).remove(shuttingDownNode).build()) + .metadata( + Metadata.builder(stateWithShuttingDownNodeAndMlNode1.metadata()) + .putCustom(NodesShutdownMetadata.TYPE, NodesShutdownMetadata.EMPTY) + .build() + ) + .build(); + + var shutdownEvent = new ClusterChangedEvent("test", stateWithMlNode1, stateWithShuttingDownNodeAndMlNode1); + var metadata = TrainedModelAssignmentMetadata.fromState(shutdownEvent.state()); + + assertFalse(TrainedModelAssignmentClusterService.haveMlNodesChanged(shutdownEvent, metadata)); + } + public void testHaveMlNodesChanged_ReturnsTrueWhenNodeShutsDownAndWasRoutedTo() { String model1 = "model-1"; String mlNode1 = "ml-node-with-room"; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilderTests.java index 5e414a7f997d5..13f12f3cdc1e1 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilderTests.java @@ -7,10 +7,15 @@ package org.elasticsearch.xpack.ml.queries; +import org.apache.lucene.document.Document; import org.apache.lucene.document.FeatureField; +import org.apache.lucene.document.FloatDocValuesField; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionType; @@ -20,6 +25,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.plugins.Plugin; @@ -36,6 +42,7 @@ import java.util.Collection; import java.util.List; +import static org.elasticsearch.xpack.ml.queries.WeightedTokensQueryBuilder.TOKENS_FIELD; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.hasSize; @@ -43,11 +50,19 @@ public class TextExpansionQueryBuilderTests extends AbstractQueryTestCase { private static final String RANK_FEATURES_FIELD = "rank"; - private static int NUM_TOKENS = 10; + private static final int NUM_TOKENS = 10; @Override protected TextExpansionQueryBuilder doCreateTestQueryBuilder() { - var builder = new TextExpansionQueryBuilder(RANK_FEATURES_FIELD, randomAlphaOfLength(4), randomAlphaOfLength(4)); + TokenPruningConfig tokenPruningConfig = randomBoolean() + ? new TokenPruningConfig(randomIntBetween(1, 100), randomFloat(), randomBoolean()) + : null; + var builder = new TextExpansionQueryBuilder( + RANK_FEATURES_FIELD, + randomAlphaOfLength(4), + randomAlphaOfLength(4), + tokenPruningConfig + ); if (randomBoolean()) { builder.boost((float) randomDoubleBetween(0.1, 10.0, true)); } @@ -126,6 +141,44 @@ protected void doAssertLuceneQuery(TextExpansionQueryBuilder queryBuilder, Query } } + /** + * Overridden to ensure that {@link SearchExecutionContext} has a non-null {@link IndexReader} + */ + @Override + public void testCacheability() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + Document document = new Document(); + document.add(new FloatDocValuesField(RANK_FEATURES_FIELD, 1.0f)); + iw.addDocument(document); + try (IndexReader reader = iw.getReader()) { + SearchExecutionContext context = createSearchExecutionContext(newSearcher(reader)); + TextExpansionQueryBuilder queryBuilder = createTestQueryBuilder(); + QueryBuilder rewriteQuery = rewriteQuery(queryBuilder, new SearchExecutionContext(context)); + + assertNotNull(rewriteQuery.toQuery(context)); + assertTrue("query should be cacheable: " + queryBuilder.toString(), context.isCacheable()); + } + } + } + + /** + * Overridden to ensure that {@link SearchExecutionContext} has a non-null {@link IndexReader}; this query should always be rewritten + */ + @Override + public void testToQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + Document document = new Document(); + document.add(new FloatDocValuesField(RANK_FEATURES_FIELD, 1.0f)); + iw.addDocument(document); + try (IndexReader reader = iw.getReader()) { + SearchExecutionContext context = createSearchExecutionContext(newSearcher(reader)); + TextExpansionQueryBuilder queryBuilder = createTestQueryBuilder(); + IllegalStateException e = expectThrows(IllegalStateException.class, () -> queryBuilder.toQuery(context)); + assertEquals("text_expansion should have been rewritten to another query type", e.getMessage()); + } + } + } + public void testIllegalValues() { { IllegalArgumentException e = expectThrows( @@ -162,4 +215,55 @@ public void testToXContent() throws IOException { } }""", query); } + + public void testToXContentWithThresholds() throws IOException { + QueryBuilder query = new TextExpansionQueryBuilder("foo", "bar", "baz", new TokenPruningConfig(4, 0.3f, false)); + checkGeneratedJson(""" + { + "text_expansion": { + "foo": { + "model_text": "bar", + "model_id": "baz", + "pruning_config": { + "tokens_freq_ratio_threshold": 4.0, + "tokens_weight_threshold": 0.3 + } + } + } + }""", query); + } + + public void testToXContentWithThresholdsAndOnlyScorePrunedTokens() throws IOException { + QueryBuilder query = new TextExpansionQueryBuilder("foo", "bar", "baz", new TokenPruningConfig(4, 0.3f, true)); + checkGeneratedJson(""" + { + "text_expansion": { + "foo": { + "model_text": "bar", + "model_id": "baz", + "pruning_config": { + "tokens_freq_ratio_threshold": 4.0, + "tokens_weight_threshold": 0.3, + "only_score_pruned_tokens": true + } + } + } + }""", query); + } + + @Override + protected String[] shuffleProtectedFields() { + return new String[] { TOKENS_FIELD.getPreferredName() }; + } + + public void testThatTokensAreCorrectlyPruned() { + SearchExecutionContext searchExecutionContext = createSearchExecutionContext(); + TextExpansionQueryBuilder queryBuilder = createTestQueryBuilder(); + QueryBuilder rewrittenQueryBuilder = rewriteAndFetch(queryBuilder, searchExecutionContext); + if (queryBuilder.getTokenPruningConfig() == null) { + assertTrue(rewrittenQueryBuilder instanceof BoolQueryBuilder); + } else { + assertTrue(rewrittenQueryBuilder instanceof WeightedTokensQueryBuilder); + } + } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TokenPruningConfigTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TokenPruningConfigTests.java new file mode 100644 index 0000000000000..3f38a2ee891d5 --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TokenPruningConfigTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.queries; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractXContentSerializingTestCase; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; + +public class TokenPruningConfigTests extends AbstractXContentSerializingTestCase { + + public static TokenPruningConfig testInstance() { + return new TokenPruningConfig(randomIntBetween(1, 100), randomFloat(), randomBoolean()); + } + + @Override + protected Writeable.Reader instanceReader() { + return TokenPruningConfig::new; + } + + @Override + protected TokenPruningConfig createTestInstance() { + return testInstance(); + } + + @Override + protected TokenPruningConfig mutateInstance(TokenPruningConfig instance) throws IOException { + return null; + } + + @Override + protected TokenPruningConfig doParseInstance(XContentParser parser) throws IOException { + return TokenPruningConfig.fromXContent(parser); + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/WeightedTokensQueryBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/WeightedTokensQueryBuilderTests.java new file mode 100644 index 0000000000000..4d91c66de4b9e --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/WeightedTokensQueryBuilderTests.java @@ -0,0 +1,439 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.queries; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FeatureField; +import org.apache.lucene.document.FloatDocValuesField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.AbstractQueryTestCase; +import org.elasticsearch.xpack.core.ml.action.InferModelAction; +import org.elasticsearch.xpack.core.ml.inference.TrainedModelPrefixStrings; +import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; +import org.elasticsearch.xpack.ml.MachineLearning; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults.WeightedToken; +import static org.elasticsearch.xpack.ml.queries.WeightedTokensQueryBuilder.TOKENS_FIELD; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.Matchers.either; +import static org.hamcrest.Matchers.hasSize; + +public class WeightedTokensQueryBuilderTests extends AbstractQueryTestCase { + + private static final String RANK_FEATURES_FIELD = "rank"; + private static final List WEIGHTED_TOKENS = List.of(new TextExpansionResults.WeightedToken("foo", .42f)); + private static final int NUM_TOKENS = WEIGHTED_TOKENS.size(); + + @Override + protected WeightedTokensQueryBuilder doCreateTestQueryBuilder() { + return createTestQueryBuilder(randomBoolean()); + } + + private WeightedTokensQueryBuilder createTestQueryBuilder(boolean onlyScorePrunedTokens) { + TokenPruningConfig tokenPruningConfig = randomBoolean() + ? new TokenPruningConfig(randomIntBetween(1, 100), randomFloat(), onlyScorePrunedTokens) + : null; + + var builder = new WeightedTokensQueryBuilder(RANK_FEATURES_FIELD, WEIGHTED_TOKENS, tokenPruningConfig); + if (randomBoolean()) { + builder.boost((float) randomDoubleBetween(0.1, 10.0, true)); + } + if (randomBoolean()) { + builder.queryName(randomAlphaOfLength(4)); + } + return builder; + } + + @Override + protected Collection> getPlugins() { + return List.of(MachineLearning.class, MapperExtrasPlugin.class); + } + + @Override + protected boolean canSimulateMethod(Method method, Object[] args) throws NoSuchMethodException { + return method.equals(Client.class.getMethod("execute", ActionType.class, ActionRequest.class, ActionListener.class)) + && (args[0] instanceof InferModelAction); + } + + @Override + protected Object simulateMethod(Method method, Object[] args) { + InferModelAction.Request request = (InferModelAction.Request) args[1]; + assertEquals(InferModelAction.Request.DEFAULT_TIMEOUT_FOR_API, request.getInferenceTimeout()); + assertEquals(TrainedModelPrefixStrings.PrefixType.SEARCH, request.getPrefixType()); + + // Randomisation of tokens cannot be used here as {@code #doAssertLuceneQuery} + // asserts that 2 rewritten queries are the same + var response = InferModelAction.Response.builder() + .setId(request.getId()) + .addInferenceResults(List.of(new TextExpansionResults("foo", WEIGHTED_TOKENS.stream().toList(), randomBoolean()))) + .build(); + @SuppressWarnings("unchecked") // We matched the method above. + ActionListener listener = (ActionListener) args[2]; + listener.onResponse(response); + return null; + } + + @Override + protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { + mapperService.merge( + "_doc", + new CompressedXContent(Strings.toString(PutMappingRequest.simpleMapping(RANK_FEATURES_FIELD, "type=rank_features"))), + MapperService.MergeReason.MAPPING_UPDATE + ); + } + + /** + * Overridden to ensure that {@link SearchExecutionContext} has a non-null {@link IndexReader} + */ + @Override + public void testToQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + // Index at least one document so we have a freq > 0 + Document document = new Document(); + document.add(new FeatureField(RANK_FEATURES_FIELD, "foo", 1.0f)); + iw.addDocument(document); + try (IndexReader reader = iw.getReader()) { + SearchExecutionContext context = createSearchExecutionContext(newSearcher(reader)); + // We need to force token pruning config here, to get repeatable lucene queries for comparison + WeightedTokensQueryBuilder firstQuery = createTestQueryBuilder(false); + WeightedTokensQueryBuilder controlQuery = copyQuery(firstQuery); + QueryBuilder rewritten = rewriteQuery(firstQuery, context); + Query firstLuceneQuery = rewritten.toQuery(context); + assertNotNull("toQuery should not return null", firstLuceneQuery); + assertLuceneQuery(firstQuery, firstLuceneQuery, context); + assertEquals( + "query is not equal to its copy after calling toQuery, firstQuery: " + firstQuery + ", secondQuery: " + controlQuery, + firstQuery, + controlQuery + ); + assertEquals( + "equals is not symmetric after calling toQuery, firstQuery: " + firstQuery + ", secondQuery: " + controlQuery, + controlQuery, + firstQuery + ); + assertThat( + "query copy's hashcode is different from original hashcode after calling toQuery, firstQuery: " + + firstQuery + + ", secondQuery: " + + controlQuery, + controlQuery.hashCode(), + equalTo(firstQuery.hashCode()) + ); + WeightedTokensQueryBuilder secondQuery = copyQuery(firstQuery); + + // query _name never should affect the result of toQuery, we randomly set it to make sure + if (randomBoolean()) { + secondQuery.queryName( + secondQuery.queryName() == null + ? randomAlphaOfLengthBetween(1, 30) + : secondQuery.queryName() + randomAlphaOfLengthBetween(1, 10) + ); + } + context = new SearchExecutionContext(context); + Query secondLuceneQuery = rewriteQuery(secondQuery, context).toQuery(context); + assertNotNull("toQuery should not return null", secondLuceneQuery); + assertLuceneQuery(secondQuery, secondLuceneQuery, context); + + if (builderGeneratesCacheableQueries()) { + assertEquals( + "two equivalent query builders lead to different lucene queries hashcode", + secondLuceneQuery.hashCode(), + firstLuceneQuery.hashCode() + ); + assertEquals( + "two equivalent query builders lead to different lucene queries", + rewrite(secondLuceneQuery), + rewrite(firstLuceneQuery) + ); + } + + if (supportsBoost() && firstLuceneQuery instanceof MatchNoDocsQuery == false) { + secondQuery.boost(firstQuery.boost() + 1f + randomFloat()); + Query thirdLuceneQuery = rewriteQuery(secondQuery, context).toQuery(context); + assertNotEquals( + "modifying the boost doesn't affect the corresponding lucene query", + rewrite(firstLuceneQuery), + rewrite(thirdLuceneQuery) + ); + } + + } + } + } + + public void testPruningIsAppliedCorrectly() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + List documents = List.of( + createDocument( + List.of("the", "quick", "brown", "fox", "jumped", "over", "lazy", "dog", "me"), + List.of(.2f, 1.8f, 1.75f, 5.9f, 1.6f, 1.4f, .4f, 4.8f, 2.1f) + ), + createDocument( + List.of("the", "rains", "in", "spain", "fall", "mainly", "on", "plain", "me"), + List.of(.1f, 3.6f, .1f, 4.8f, .6f, .3f, .1f, 2.6f, 2.1f) + ), + createDocument( + List.of("betty", "bought", "butter", "but", "the", "was", "bitter", "me"), + List.of(6.8f, 1.4f, .5f, 3.2f, .1f, 3.2f, .6f, 2.1f) + ), + createDocument( + List.of("she", "sells", "seashells", "by", "the", "seashore", "me"), + List.of(.2f, 1.4f, 5.9f, .1f, .1f, 3.6f, 2.1f) + ) + ); + iw.addDocuments(documents); + + List inputTokens = List.of( + new WeightedToken("the", .1f), // Will be pruned - score too low, freq too high + new WeightedToken("black", 5.3f), // Will be pruned - does not exist in index + new WeightedToken("dog", 7.5f), // Will be kept - high score and low freq + new WeightedToken("jumped", 4.5f), // Will be kept - high score and low freq + new WeightedToken("on", .1f), // Will be kept - low score but also low freq + new WeightedToken("me", 3.8f) // Will be kept - high freq but also high score + ); + try (IndexReader reader = iw.getReader()) { + SearchExecutionContext context = createSearchExecutionContext(newSearcher(reader)); + + WeightedTokensQueryBuilder noPruningQuery = new WeightedTokensQueryBuilder(RANK_FEATURES_FIELD, inputTokens, null); + Query query = noPruningQuery.doToQuery(context); + assertCorrectLuceneQuery("noPruningQuery", query, List.of("the", "black", "dog", "jumped", "on", "me")); + + WeightedTokensQueryBuilder queryThatShouldBePruned = new WeightedTokensQueryBuilder( + RANK_FEATURES_FIELD, + inputTokens, + new TokenPruningConfig(1.5f, 0.5f, false) + ); + query = queryThatShouldBePruned.doToQuery(context); + assertCorrectLuceneQuery("queryThatShouldBePruned", query, List.of("dog", "jumped", "on", "me")); + + WeightedTokensQueryBuilder onlyScorePrunedTokensQuery = new WeightedTokensQueryBuilder( + RANK_FEATURES_FIELD, + inputTokens, + new TokenPruningConfig(1.5f, 0.5f, true) + ); + query = onlyScorePrunedTokensQuery.doToQuery(context); + assertCorrectLuceneQuery("onlyScorePrunedTokensQuery", query, List.of("the", "black")); + } + } + } + + private void assertCorrectLuceneQuery(String name, Query query, List expectedFeatureFields) { + assertTrue(query instanceof BooleanQuery); + List booleanClauses = ((BooleanQuery) query).clauses(); + assertEquals( + name + " had " + booleanClauses.size() + " clauses, expected " + expectedFeatureFields.size(), + expectedFeatureFields.size(), + booleanClauses.size() + ); + for (int i = 0; i < booleanClauses.size(); i++) { + Query clauseQuery = booleanClauses.get(i).getQuery(); + assertTrue(name + " query " + query + " expected to be a BoostQuery", clauseQuery instanceof BoostQuery); + // FeatureQuery is not visible so we check the String representation + assertTrue(name + " query " + query + " expected to be a FeatureQuery", clauseQuery.toString().contains("FeatureQuery")); + assertTrue( + name + " query " + query + " expected to have field " + expectedFeatureFields.get(i), + clauseQuery.toString().contains("feature=" + expectedFeatureFields.get(i)) + ); + } + } + + private Document createDocument(List tokens, List weights) { + if (tokens.size() != weights.size()) { + throw new IllegalArgumentException( + "tokens and weights must have the same size. Got " + tokens.size() + " and " + weights.size() + "." + ); + } + Document document = new Document(); + for (int i = 0; i < tokens.size(); i++) { + document.add(new FeatureField(RANK_FEATURES_FIELD, tokens.get(i), weights.get(i))); + } + return document; + } + + /** + * Overridden to ensure that {@link SearchExecutionContext} has a non-null {@link IndexReader} + */ + @Override + public void testCacheability() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + Document document = new Document(); + document.add(new FloatDocValuesField(RANK_FEATURES_FIELD, 1.0f)); + iw.addDocument(document); + try (IndexReader reader = iw.getReader()) { + SearchExecutionContext context = createSearchExecutionContext(newSearcher(reader)); + WeightedTokensQueryBuilder queryBuilder = createTestQueryBuilder(); + QueryBuilder rewriteQuery = rewriteQuery(queryBuilder, new SearchExecutionContext(context)); + + assertNotNull(rewriteQuery.toQuery(context)); + assertTrue("query should be cacheable: " + queryBuilder.toString(), context.isCacheable()); + } + } + } + + /** + * Overridden to ensure that {@link SearchExecutionContext} has a non-null {@link IndexReader} + */ + @Override + public void testMustRewrite() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + Document document = new Document(); + document.add(new FloatDocValuesField(RANK_FEATURES_FIELD, 1.0f)); + iw.addDocument(document); + try (IndexReader reader = iw.getReader()) { + SearchExecutionContext context = createSearchExecutionContext(newSearcher(reader)); + context.setAllowUnmappedFields(true); + WeightedTokensQueryBuilder queryBuilder = createTestQueryBuilder(); + queryBuilder.toQuery(context); + } + } + } + + @Override + protected void doAssertLuceneQuery(WeightedTokensQueryBuilder queryBuilder, Query query, SearchExecutionContext context) { + assertThat(query, instanceOf(BooleanQuery.class)); + BooleanQuery booleanQuery = (BooleanQuery) query; + assertEquals(booleanQuery.getMinimumNumberShouldMatch(), 1); + assertThat(booleanQuery.clauses(), hasSize(NUM_TOKENS)); + + Class featureQueryClass = FeatureField.newLinearQuery("", "", 0.5f).getClass(); + // if the weight is 1.0f a BoostQuery is returned + Class boostQueryClass = FeatureField.newLinearQuery("", "", 1.0f).getClass(); + + for (var clause : booleanQuery.clauses()) { + assertEquals(BooleanClause.Occur.SHOULD, clause.getOccur()); + assertThat(clause.getQuery(), either(instanceOf(featureQueryClass)).or(instanceOf(boostQueryClass))); + } + } + + public void testIllegalValues() { + List weightedTokens = List.of(new WeightedToken("foo", 1.0f)); + { + NullPointerException e = expectThrows( + NullPointerException.class, + () -> new WeightedTokensQueryBuilder(null, weightedTokens, null) + ); + assertEquals("[weighted_tokens] requires a fieldName", e.getMessage()); + } + { + NullPointerException e = expectThrows( + NullPointerException.class, + () -> new WeightedTokensQueryBuilder("field name", null, null) + ); + assertEquals("[weighted_tokens] requires tokens", e.getMessage()); + } + { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> new WeightedTokensQueryBuilder("field name", List.of(), null) + ); + assertEquals("[weighted_tokens] requires at least one token", e.getMessage()); + } + { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> new WeightedTokensQueryBuilder("field name", weightedTokens, new TokenPruningConfig(-1f, 0.0f, false)) + ); + assertEquals("[tokens_freq_ratio_threshold] must be between [1.0] and [100.0], got -1.0", e.getMessage()); + } + { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> new WeightedTokensQueryBuilder("field name", weightedTokens, new TokenPruningConfig(101f, 0.0f, false)) + ); + assertEquals("[tokens_freq_ratio_threshold] must be between [1.0] and [100.0], got 101.0", e.getMessage()); + } + { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> new WeightedTokensQueryBuilder("field name", weightedTokens, new TokenPruningConfig(5f, 5f, false)) + ); + assertEquals("[tokens_weight_threshold] must be between 0 and 1", e.getMessage()); + } + } + + public void testToXContent() throws Exception { + QueryBuilder query = new WeightedTokensQueryBuilder("foo", WEIGHTED_TOKENS, null); + checkGeneratedJson(""" + { + "weighted_tokens": { + "foo": { + "tokens": { + "foo": 0.42 + } + } + } + }""", query); + } + + public void testToXContentWithThresholds() throws Exception { + QueryBuilder query = new WeightedTokensQueryBuilder("foo", WEIGHTED_TOKENS, new TokenPruningConfig(4, 0.4f, false)); + checkGeneratedJson(""" + { + "weighted_tokens": { + "foo": { + "tokens": { + "foo": 0.42 + }, + "pruning_config": { + "tokens_freq_ratio_threshold": 4.0, + "tokens_weight_threshold": 0.4 + } + } + } + }""", query); + } + + public void testToXContentWithThresholdsAndOnlyScorePrunedTokens() throws Exception { + QueryBuilder query = new WeightedTokensQueryBuilder("foo", WEIGHTED_TOKENS, new TokenPruningConfig(4, 0.4f, true)); + checkGeneratedJson(""" + { + "weighted_tokens": { + "foo": { + "tokens": { + "foo": 0.42 + }, + "pruning_config": { + "tokens_freq_ratio_threshold": 4.0, + "tokens_weight_threshold": 0.4, + "only_score_pruned_tokens": true + } + } + } + }""", query); + } + + @Override + protected String[] shuffleProtectedFields() { + return new String[] { TOKENS_FIELD.getPreferredName() }; + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/vectors/TextEmbeddingQueryVectorBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/vectors/TextEmbeddingQueryVectorBuilderTests.java index a44aa9404f4f9..8575c7e1f4bf3 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/vectors/TextEmbeddingQueryVectorBuilderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/vectors/TextEmbeddingQueryVectorBuilderTests.java @@ -18,7 +18,7 @@ import org.elasticsearch.xpack.core.ml.action.InferModelAction; import org.elasticsearch.xpack.core.ml.inference.TrainedModelPrefixStrings; import org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingResults; -import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import java.io.IOException; import java.util.List; @@ -30,7 +30,7 @@ public class TextEmbeddingQueryVectorBuilderTests extends AbstractQueryVectorBui @Override protected List additionalPlugins() { - return List.of(new MachineLearning(Settings.EMPTY)); + return List.of(MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY)); } @Override diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java index 5d9736726b46f..f084b5cda4abe 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java @@ -8,6 +8,8 @@ import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.xpack.ql.expression.Alias; +import org.elasticsearch.xpack.ql.expression.Attribute; +import org.elasticsearch.xpack.ql.expression.AttributeMap; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Expressions; import org.elasticsearch.xpack.ql.expression.Literal; @@ -69,6 +71,7 @@ import static java.lang.Math.signum; import static java.util.Arrays.asList; +import static java.util.Collections.emptySet; import static org.elasticsearch.xpack.ql.expression.Literal.FALSE; import static org.elasticsearch.xpack.ql.expression.Literal.TRUE; import static org.elasticsearch.xpack.ql.expression.predicate.Predicates.combineAnd; @@ -1785,6 +1788,93 @@ protected Expression nonNullify(Expression exp, Expression nonNullExp) { } } + /** + * Simplify IsNotNull targets by resolving the underlying expression to its root fields with unknown + * nullability. + * e.g. + * (x + 1) / 2 IS NOT NULL --> x IS NOT NULL AND (x+1) / 2 IS NOT NULL + * SUBSTRING(x, 3) > 4 IS NOT NULL --> x IS NOT NULL AND SUBSTRING(x, 3) > 4 IS NOT NULL + * When dealing with multiple fields, a conjunction/disjunction based on the predicate: + * (x + y) / 4 IS NOT NULL --> x IS NOT NULL AND y IS NOT NULL AND (x + y) / 4 IS NOT NULL + * This handles the case of fields nested inside functions or expressions in order to avoid: + * - having to evaluate the whole expression + * - not pushing down the filter due to expression evaluation + * IS NULL cannot be simplified since it leads to a disjunction which prevents the filter to be + * pushed down: + * (x + 1) IS NULL --> x IS NULL OR x + 1 IS NULL + * and x IS NULL cannot be pushed down + *
+ * Implementation-wise this rule goes bottom-up, keeping an alias up to date to the current plan + * and then looks for replacing the target. + */ + public static class InferIsNotNull extends Rule { + + @Override + public LogicalPlan apply(LogicalPlan plan) { + // the alias map is shared across the whole plan + AttributeMap aliases = new AttributeMap<>(); + // traverse bottom-up to pick up the aliases as we go + plan = plan.transformUp(p -> inspectPlan(p, aliases)); + return plan; + } + + private LogicalPlan inspectPlan(LogicalPlan plan, AttributeMap aliases) { + // inspect just this plan properties + plan.forEachExpression(Alias.class, a -> aliases.put(a.toAttribute(), a.child())); + // now go about finding isNull/isNotNull + LogicalPlan newPlan = plan.transformExpressionsOnlyUp(IsNotNull.class, inn -> inferNotNullable(inn, aliases)); + return newPlan; + } + + private Expression inferNotNullable(IsNotNull inn, AttributeMap aliases) { + Expression result = inn; + Set refs = resolveExpressionAsRootAttributes(inn.field(), aliases); + // no refs found or could not detect - return the original function + if (refs.size() > 0) { + // add IsNull for the filters along with the initial inn + var innList = CollectionUtils.combine(refs.stream().map(r -> (Expression) new IsNotNull(inn.source(), r)).toList(), inn); + result = Predicates.combineAnd(innList); + } + return result; + } + + /** + * Unroll the expression to its references to get to the root fields + * that really matter for filtering. + */ + protected Set resolveExpressionAsRootAttributes(Expression exp, AttributeMap aliases) { + Set resolvedExpressions = new LinkedHashSet<>(); + boolean changed = doResolve(exp, aliases, resolvedExpressions); + return changed ? resolvedExpressions : emptySet(); + } + + private boolean doResolve(Expression exp, AttributeMap aliases, Set resolvedExpressions) { + boolean changed = false; + // check if the expression can be skipped or is not nullabe + if (skipExpression(exp) || exp.nullable() == Nullability.FALSE) { + resolvedExpressions.add(exp); + } else { + for (Expression e : exp.references()) { + Expression resolved = aliases.resolve(e, e); + // found a root attribute, bail out + if (resolved instanceof Attribute a && resolved == e) { + resolvedExpressions.add(a); + // don't mark things as change if the original expression hasn't been broken down + changed |= resolved != exp; + } else { + // go further + changed |= doResolve(resolved, aliases, resolvedExpressions); + } + } + } + return changed; + } + + protected boolean skipExpression(Expression e) { + return false; + } + } + public static final class SetAsOptimized extends Rule { @Override diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java index b14e46a96a9e6..1cab7dd87195b 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.expression.Nullability; import org.elasticsearch.xpack.ql.expression.function.aggregate.Count; +import org.elasticsearch.xpack.ql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.ql.expression.predicate.BinaryOperator; import org.elasticsearch.xpack.ql.expression.predicate.Predicates; import org.elasticsearch.xpack.ql.expression.predicate.Range; @@ -1768,7 +1769,90 @@ public void testPushDownFilterThroughAgg() throws Exception { // expected Filter expected = new Filter(EMPTY, new Aggregate(EMPTY, combinedFilter, emptyList(), emptyList()), aggregateCondition); assertEquals(expected, new PushDownAndCombineFilters().apply(fb)); + } + + public void testIsNotNullOnIsNullField() { + EsRelation relation = relation(); + var fieldA = getFieldAttribute("a"); + Expression inn = isNotNull(fieldA); + Filter f = new Filter(EMPTY, relation, inn); + + assertEquals(f, new OptimizerRules.InferIsNotNull().apply(f)); + } + + public void testIsNotNullOnOperatorWithOneField() { + EsRelation relation = relation(); + var fieldA = getFieldAttribute("a"); + Expression inn = isNotNull(new Add(EMPTY, fieldA, ONE)); + Filter f = new Filter(EMPTY, relation, inn); + Filter expected = new Filter(EMPTY, relation, new And(EMPTY, isNotNull(fieldA), inn)); + + assertEquals(expected, new OptimizerRules.InferIsNotNull().apply(f)); + } + + public void testIsNotNullOnOperatorWithTwoFields() { + EsRelation relation = relation(); + var fieldA = getFieldAttribute("a"); + var fieldB = getFieldAttribute("b"); + Expression inn = isNotNull(new Add(EMPTY, fieldA, fieldB)); + Filter f = new Filter(EMPTY, relation, inn); + Filter expected = new Filter(EMPTY, relation, new And(EMPTY, new And(EMPTY, isNotNull(fieldA), isNotNull(fieldB)), inn)); + + assertEquals(expected, new OptimizerRules.InferIsNotNull().apply(f)); + } + + public void testIsNotNullOnFunctionWithOneField() { + EsRelation relation = relation(); + var fieldA = getFieldAttribute("a"); + var pattern = L("abc"); + Expression inn = isNotNull( + new And(EMPTY, new TestStartsWith(EMPTY, fieldA, pattern, false), greaterThanOf(new Add(EMPTY, ONE, TWO), THREE)) + ); + + Filter f = new Filter(EMPTY, relation, inn); + Filter expected = new Filter(EMPTY, relation, new And(EMPTY, isNotNull(fieldA), inn)); + + assertEquals(expected, new OptimizerRules.InferIsNotNull().apply(f)); + } + + public void testIsNotNullOnFunctionWithTwoFields() { + EsRelation relation = relation(); + var fieldA = getFieldAttribute("a"); + var fieldB = getFieldAttribute("b"); + var pattern = L("abc"); + Expression inn = isNotNull(new TestStartsWith(EMPTY, fieldA, fieldB, false)); + + Filter f = new Filter(EMPTY, relation, inn); + Filter expected = new Filter(EMPTY, relation, new And(EMPTY, new And(EMPTY, isNotNull(fieldA), isNotNull(fieldB)), inn)); + + assertEquals(expected, new OptimizerRules.InferIsNotNull().apply(f)); + } + + public static class TestStartsWith extends StartsWith { + + public TestStartsWith(Source source, Expression input, Expression pattern, boolean caseInsensitive) { + super(source, input, pattern, caseInsensitive); + } + + @Override + public Expression replaceChildren(List newChildren) { + return new TestStartsWith(source(), newChildren.get(0), newChildren.get(1), isCaseInsensitive()); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, TestStartsWith::new, input(), pattern(), isCaseInsensitive()); + } + } + + public void testIsNotNullOnFunctionWithTwoField() {} + + private IsNotNull isNotNull(Expression field) { + return new IsNotNull(EMPTY, field); + } + private IsNull isNull(Expression field) { + return new IsNull(EMPTY, field); } private Literal nullOf(DataType dataType) { diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheMaintenanceService.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheMaintenanceService.java index 6dbe31e08dd0e..57af1787b91b7 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheMaintenanceService.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheMaintenanceService.java @@ -10,10 +10,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.bulk.BulkAction; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.search.ClosePointInTimeRequest; @@ -25,6 +25,7 @@ import org.elasticsearch.action.search.TransportClosePointInTimeAction; import org.elasticsearch.action.search.TransportOpenPointInTimeAction; import org.elasticsearch.action.search.TransportSearchAction; +import org.elasticsearch.action.support.RefCountingListener; import org.elasticsearch.action.support.TransportActions; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; @@ -37,16 +38,16 @@ import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.common.util.concurrent.ThrottledTaskRunner; +import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Releasable; -import org.elasticsearch.core.Releasables; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; @@ -73,9 +74,8 @@ import java.util.Objects; import java.util.Queue; import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -88,10 +88,10 @@ /** * A service that delete documents in the snapshot blob cache index when they are not required anymore. - * + *

* This service runs on the data node that contains the snapshot blob cache primary shard. It listens to cluster state updates to find * searchable snapshot indices that are deleted and checks if the index snapshot is still used by other searchable snapshot indices. If the - * index snapshot is not used anymore then i triggers the deletion of corresponding cached blobs in the snapshot blob cache index using a + * index snapshot is not used anymore then it triggers the deletion of corresponding cached blobs in the snapshot blob cache index using a * delete-by-query. */ public class BlobStoreCacheMaintenanceService implements ClusterStateListener { @@ -266,6 +266,10 @@ private static boolean hasSearchableSnapshotWith(final ClusterState state, final return false; } + private static Instant getExpirationTime(TimeValue retention, ThreadPool threadPool) { + return Instant.ofEpochMilli(threadPool.absoluteTimeInMillis()).minus(retention.duration(), retention.timeUnit().toChronoUnit()); + } + private static Map> listSearchableSnapshots(final ClusterState state) { Map> snapshots = null; for (IndexMetadata indexMetadata : state.metadata()) { @@ -396,327 +400,270 @@ public void onFailure(Exception e) { /** * A maintenance task that periodically cleans up unused cache entries from the blob store cache index. - * + *

* This task first opens a point-in-time context on the blob store cache system index and uses it to search all documents. For each * document found the task verifies if it belongs to an existing searchable snapshot index. If the doc does not belong to any * index then it is deleted as part of a bulk request. Once the bulk is executed the next batch of documents is searched for. Once * all documents from the PIT have been verified the task closes the PIT and completes itself. - * + *

* The task executes every step (PIT opening, searches, bulk deletes, PIT closing) using the generic thread pool. * The same task instance is used for all the steps and makes sure that a closed instance is not executed again. */ - private class PeriodicMaintenanceTask implements Runnable, Releasable { - + private class PeriodicMaintenanceTask implements Runnable { private final TimeValue keepAlive; private final int batchSize; - private final AtomicReference error = new AtomicReference<>(); - private final AtomicBoolean closed = new AtomicBoolean(); + private final ThrottledTaskRunner taskRunner; private final AtomicLong deletes = new AtomicLong(); private final AtomicLong total = new AtomicLong(); - private volatile Map> existingSnapshots; - private volatile Set existingRepositories; - private final AtomicReference searchResponse = new AtomicReference<>(); - private volatile Instant expirationTime; - private volatile String pointIntTimeId; - private volatile Object[] searchAfter; - PeriodicMaintenanceTask(TimeValue keepAlive, int batchSize) { this.keepAlive = keepAlive; this.batchSize = batchSize; + this.taskRunner = new ThrottledTaskRunner(this.getClass().getCanonicalName(), 2, threadPool.generic()); } @Override public void run() { - assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.GENERIC); - try { - ensureOpen(); - if (pointIntTimeId == null) { - final OpenPointInTimeRequest openRequest = new OpenPointInTimeRequest(SNAPSHOT_BLOB_CACHE_INDEX); - openRequest.keepAlive(keepAlive); - clientWithOrigin.execute(TransportOpenPointInTimeAction.TYPE, openRequest, new ActionListener<>() { - @Override - public void onResponse(OpenPointInTimeResponse response) { - logger.trace("periodic maintenance task initialized with point-in-time id [{}]", response.getPointInTimeId()); - PeriodicMaintenanceTask.this.pointIntTimeId = response.getPointInTimeId(); - executeNext(PeriodicMaintenanceTask.this); - } - - @Override - public void onFailure(Exception e) { - if (TransportActions.isShardNotAvailableException(e)) { - complete(null); - } else { - complete(e); - } - } - }); - return; + ActionListener.run(ActionListener.runAfter(new ActionListener() { + @Override + public void onResponse(Void unused) { + logger.info( + () -> format( + "periodic maintenance task completed (%s deleted documents out of a total of %s)", + deletes.get(), + total.get() + ) + ); } - final String pitId = pointIntTimeId; - assert Strings.hasLength(pitId); - - SearchResponse searchResponseRef; - do { - searchResponseRef = searchResponse.get(); - if (searchResponseRef == null) { - handleMissingSearchResponse(pitId); - return; + @Override + public void onFailure(Exception e) { + logger.warn( + () -> format( + "periodic maintenance task completed with failure (%s deleted documents out of a total of %s)", + deletes.get(), + total.get() + ), + e + ); + } + }, BlobStoreCacheMaintenanceService.this::startPeriodicTask), listener -> { + final OpenPointInTimeRequest openRequest = new OpenPointInTimeRequest(SNAPSHOT_BLOB_CACHE_INDEX); + openRequest.keepAlive(keepAlive); + clientWithOrigin.execute(TransportOpenPointInTimeAction.TYPE, openRequest, new ActionListener<>() { + @Override + public void onResponse(OpenPointInTimeResponse response) { + logger.trace("periodic maintenance task initialized with point-in-time id [{}]", response.getPointInTimeId()); + threadPool.generic().execute(ActionRunnable.wrap(listener, l -> { + final ClusterState state = clusterService.state(); + new RunningPeriodicMaintenanceTask( + response.getPointInTimeId(), + closingPitBefore(clientWithOrigin, response.getPointInTimeId(), l), + getExpirationTime(periodicTaskRetention, threadPool), + // compute the list of existing searchable snapshots and repositories up-front + listSearchableSnapshots(state), + RepositoriesMetadata.get(state) + .repositories() + .stream() + .map(RepositoryMetadata::name) + .collect(Collectors.toSet()) + ).run(); + })); } - } while (searchResponseRef.tryIncRef() == false); - try { - var searchHits = searchResponseRef.getHits().getHits(); - if (searchHits != null && searchHits.length > 0) { - updateWithSearchHits(searchHits); - return; + + @Override + public void onFailure(Exception e) { + if (TransportActions.isShardNotAvailableException(e)) { + listener.onResponse(null); + } else { + listener.onFailure(e); + } } - } finally { - searchResponseRef.decRef(); + }); + }); + } + + private static ActionListener closingPitBefore(Client client, String pointInTimeId, ActionListener listener) { + return new ActionListener<>() { + @Override + public void onResponse(Void unused) { + closePit(client, pointInTimeId, () -> listener.onResponse(null)); } - // we're done, complete the task - complete(null); - } catch (Exception e) { - complete(e); - } + + @Override + public void onFailure(Exception e) { + closePit(client, pointInTimeId, () -> listener.onFailure(e)); + } + }; } - private void handleMissingSearchResponse(String pitId) { - final SearchSourceBuilder searchSource = new SearchSourceBuilder(); - searchSource.fetchField(new FieldAndFormat(CachedBlob.CREATION_TIME_FIELD, "epoch_millis")); - searchSource.fetchSource(false); - searchSource.trackScores(false); - searchSource.sort(ShardDocSortField.NAME); - searchSource.size(batchSize); - if (searchAfter != null) { - searchSource.searchAfter(searchAfter); - searchSource.trackTotalHits(false); - } else { - searchSource.trackTotalHits(true); - } - final PointInTimeBuilder pointInTime = new PointInTimeBuilder(pitId); - pointInTime.setKeepAlive(keepAlive); - searchSource.pointInTimeBuilder(pointInTime); - final SearchRequest searchRequest = new SearchRequest(); - searchRequest.source(searchSource); - clientWithOrigin.execute(TransportSearchAction.TYPE, searchRequest, new ActionListener<>() { + private static void closePit(Client client, String pointInTimeId, Runnable onCompletion) { + client.execute(TransportClosePointInTimeAction.TYPE, new ClosePointInTimeRequest(pointInTimeId), new ActionListener<>() { @Override - public void onResponse(SearchResponse response) { - if (searchAfter == null) { - assert PeriodicMaintenanceTask.this.total.get() == 0L; - PeriodicMaintenanceTask.this.total.set(response.getHits().getTotalHits().value); + public void onResponse(ClosePointInTimeResponse response) { + if (response.isSucceeded()) { + logger.debug("periodic maintenance task successfully closed point-in-time id [{}]", pointInTimeId); + } else { + logger.debug("point-in-time id [{}] not found", pointInTimeId); } - PeriodicMaintenanceTask.this.setCurrentResponse(response); - PeriodicMaintenanceTask.this.searchAfter = null; - executeNext(PeriodicMaintenanceTask.this); + onCompletion.run(); } @Override public void onFailure(Exception e) { - complete(e); + logger.warn(() -> "failed to close point-in-time id [" + pointInTimeId + "]", e); + onCompletion.run(); } }); } - private void updateWithSearchHits(SearchHit[] searchHits) { - if (expirationTime == null) { - final TimeValue retention = periodicTaskRetention; - expirationTime = Instant.ofEpochMilli(threadPool.absoluteTimeInMillis()) - .minus(retention.duration(), retention.timeUnit().toChronoUnit()); - - final ClusterState state = clusterService.state(); - // compute the list of existing searchable snapshots and repositories once - existingSnapshots = listSearchableSnapshots(state); - existingRepositories = RepositoriesMetadata.get(state) - .repositories() - .stream() - .map(RepositoryMetadata::name) - .collect(Collectors.toSet()); + /** + * The maintenance task, once it has opened its PIT and started running so that it has all the state it needs to do its job. + */ + private class RunningPeriodicMaintenanceTask implements Runnable { + private final String pointInTimeId; + private final RefCountingListener listeners; + private final Instant expirationTime; + private final Map> existingSnapshots; + private final Set existingRepositories; + + RunningPeriodicMaintenanceTask( + String pointInTimeId, + ActionListener listener, + Instant expirationTime, + Map> existingSnapshots, + Set existingRepositories + ) { + this.pointInTimeId = pointInTimeId; + this.listeners = new RefCountingListener(listener); + this.expirationTime = expirationTime; + this.existingSnapshots = existingSnapshots; + this.existingRepositories = existingRepositories; } - Object[] lastSortValues = null; - final BulkRequest bulkRequest = new BulkRequest(); - try { - final Map> knownSnapshots = existingSnapshots; - assert knownSnapshots != null; - final Set knownRepositories = existingRepositories; - assert knownRepositories != null; - final Instant expirationTimeCopy = this.expirationTime; - assert expirationTimeCopy != null; - - for (SearchHit searchHit : searchHits) { - lastSortValues = searchHit.getSortValues(); - assert searchHit.getId() != null; - try { - boolean delete = false; - - // See {@link BlobStoreCacheService#generateId} - // doc id = {repository name}/{snapshot id}/{snapshot index id}/{shard id}/{file name}/@{file offset} - final String[] parts = Objects.requireNonNull(searchHit.getId()).split("/"); - assert parts.length == 6 : Arrays.toString(parts) + " vs " + searchHit.getId(); - - final String repositoryName = parts[0]; - if (knownRepositories.contains(repositoryName) == false) { - logger.trace("deleting blob store cache entry with id [{}]: repository does not exist", searchHit.getId()); - delete = true; - } else { - final Set knownIndexIds = knownSnapshots.get(parts[1]); - if (knownIndexIds == null || knownIndexIds.contains(parts[2]) == false) { - logger.trace("deleting blob store cache entry with id [{}]: not used", searchHit.getId()); - delete = true; - } - } - if (delete) { - final Instant creationTime = getCreationTime(searchHit); - if (creationTime.isAfter(expirationTimeCopy)) { - logger.trace( - "blob store cache entry with id [{}] was created recently, skipping deletion", - searchHit.getId() - ); - continue; - } - bulkRequest.add(new DeleteRequest().index(searchHit.getIndex()).id(searchHit.getId())); - } - } catch (Exception e) { - logger.warn( - () -> format("exception when parsing blob store cache entry with id [%s], skipping", searchHit.getId()), - e - ); - } - } - - assert lastSortValues != null; - if (bulkRequest.numberOfActions() == 0) { - setCurrentResponse(null); - this.searchAfter = lastSortValues; - bulkRequest.close(); - executeNext(this); - return; + @Override + public void run() { + assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.GENERIC); + try (listeners) { + executeSearch(new SearchRequest().source(getSearchSourceBuilder().trackTotalHits(true)), (searchResponse, refs) -> { + assert total.get() == 0L; + total.set(searchResponse.getHits().getTotalHits().value); + handleSearchResponse(searchResponse, refs); + }); } - } catch (Exception e) { - bulkRequest.close(); - throw e; } - final Object[] finalSearchAfter = lastSortValues; - clientWithOrigin.execute(BulkAction.INSTANCE, bulkRequest, ActionListener.releaseAfter(new ActionListener<>() { - @Override - public void onResponse(BulkResponse response) { - for (BulkItemResponse itemResponse : response.getItems()) { - if (itemResponse.isFailed() == false) { - assert itemResponse.getResponse() instanceof DeleteResponse; - deletes.incrementAndGet(); + private void executeSearch(SearchRequest searchRequest, BiConsumer responseConsumer) { + clientWithOrigin.execute(TransportSearchAction.TYPE, searchRequest, listeners.acquire(searchResponse -> { + searchResponse.mustIncRef(); + taskRunner.enqueueTask(ActionListener.runAfter(listeners.acquire(ref -> { + final var refs = AbstractRefCounted.of(ref::close); + try { + responseConsumer.accept(searchResponse, refs); + } finally { + refs.decRef(); } - } - PeriodicMaintenanceTask.this.setCurrentResponse(null); - PeriodicMaintenanceTask.this.searchAfter = finalSearchAfter; - executeNext(PeriodicMaintenanceTask.this); - } + }), searchResponse::decRef)); + })); + } - @Override - public void onFailure(Exception e) { - complete(e); - } - }, bulkRequest)); - } + private SearchSourceBuilder getSearchSourceBuilder() { + return new SearchSourceBuilder().fetchField(new FieldAndFormat(CachedBlob.CREATION_TIME_FIELD, "epoch_millis")) + .fetchSource(false) + .trackScores(false) + .sort(ShardDocSortField.NAME) + .size(batchSize) + .pointInTimeBuilder(new PointInTimeBuilder(pointInTimeId).setKeepAlive(keepAlive)); + } - public boolean isClosed() { - return closed.get(); - } + private void handleSearchResponse(SearchResponse searchResponse, RefCounted refs) { + assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.GENERIC); - private void ensureOpen() { - if (isClosed()) { - assert false : "should not use periodic task after close"; - throw new IllegalStateException("Periodic maintenance task is closed"); - } - } + if (listeners.isFailing()) { + return; + } - @Override - public void close() { - if (closed.compareAndSet(false, true)) { - setCurrentResponse(null); - final Exception e = error.get(); - if (e != null) { - logger.warn( - () -> format( - "periodic maintenance task completed with failure (%s deleted documents out of a total of %s)", - deletes.get(), - total.get() - ), - e - ); - } else { - logger.info( - () -> format( - "periodic maintenance task completed (%s deleted documents out of a total of %s)", - deletes.get(), - total.get() - ) - ); + final var searchHits = searchResponse.getHits().getHits(); + if (searchHits == null || searchHits.length == 0) { + return; } - } - } - private void complete(@Nullable Exception failure) { - assert isClosed() == false; - final Releasable releasable = () -> { + Object[] lastSortValues = null; + final BulkRequest bulkRequest = new BulkRequest(); try { - final Exception previous = error.getAndSet(failure); - assert previous == null : "periodic maintenance task already failed: " + previous; - close(); - } finally { - startPeriodicTask(); - } - }; - boolean waitForRelease = false; - try { - final String pitId = pointIntTimeId; - if (Strings.hasLength(pitId)) { - final ClosePointInTimeRequest closeRequest = new ClosePointInTimeRequest(pitId); - clientWithOrigin.execute( - TransportClosePointInTimeAction.TYPE, - closeRequest, - ActionListener.runAfter(new ActionListener<>() { - @Override - public void onResponse(ClosePointInTimeResponse response) { - if (response.isSucceeded()) { - logger.debug("periodic maintenance task successfully closed point-in-time id [{}]", pitId); - } else { - logger.debug("point-in-time id [{}] not found", pitId); + for (SearchHit searchHit : searchHits) { + lastSortValues = searchHit.getSortValues(); + assert searchHit.getId() != null; + try { + boolean delete = false; + + // See {@link BlobStoreCacheService#generateId} + // doc id = {repository name}/{snapshot id}/{snapshot index id}/{shard id}/{file name}/@{file offset} + final String[] parts = Objects.requireNonNull(searchHit.getId()).split("/"); + assert parts.length == 6 : Arrays.toString(parts) + " vs " + searchHit.getId(); + + final String repositoryName = parts[0]; + if (existingRepositories.contains(repositoryName) == false) { + logger.trace("deleting blob store cache entry with id [{}]: repository does not exist", searchHit.getId()); + delete = true; + } else { + final Set knownIndexIds = existingSnapshots.get(parts[1]); + if (knownIndexIds == null || knownIndexIds.contains(parts[2]) == false) { + logger.trace("deleting blob store cache entry with id [{}]: not used", searchHit.getId()); + delete = true; + } + } + if (delete) { + final Instant creationTime = getCreationTime(searchHit); + if (creationTime.isAfter(expirationTime)) { + logger.trace( + "blob store cache entry with id [{}] was created recently, skipping deletion", + searchHit.getId() + ); + continue; } + bulkRequest.add(new DeleteRequest().index(searchHit.getIndex()).id(searchHit.getId())); } + } catch (Exception e) { + logger.warn( + () -> format("exception when parsing blob store cache entry with id [%s], skipping", searchHit.getId()), + e + ); + } + } + } catch (Exception e) { + bulkRequest.close(); + throw e; + } - @Override - public void onFailure(Exception e) { - logger.warn(() -> "failed to close point-in-time id [" + pitId + "]", e); + if (bulkRequest.numberOfActions() > 0) { + refs.mustIncRef(); + clientWithOrigin.execute( + BulkAction.INSTANCE, + bulkRequest, + ActionListener.releaseAfter(ActionListener.releaseAfter(listeners.acquire(bulkResponse -> { + for (BulkItemResponse itemResponse : bulkResponse.getItems()) { + if (itemResponse.isFailed() == false) { + assert itemResponse.getResponse() instanceof DeleteResponse; + deletes.incrementAndGet(); + } } - }, () -> Releasables.close(releasable)) + }), refs::decRef), bulkRequest) ); - waitForRelease = true; - } - } finally { - if (waitForRelease == false) { - Releasables.close(releasable); + } else { + bulkRequest.close(); } - } - } - private void setCurrentResponse(SearchResponse response) { - if (response != null) { - response.mustIncRef(); - } - var previous = searchResponse.getAndSet(response); - if (previous != null) { - previous.decRef(); + assert lastSortValues != null; + executeSearch( + new SearchRequest().source(getSearchSourceBuilder().trackTotalHits(false).searchAfter(lastSortValues)), + this::handleSearchResponse + ); } } } - private void executeNext(PeriodicMaintenanceTask maintenanceTask) { - threadPool.generic().execute(maintenanceTask); - } - private static Instant getCreationTime(SearchHit searchHit) { final DocumentField creationTimeField = searchHit.field(CachedBlob.CREATION_TIME_FIELD); assert creationTimeField != null; diff --git a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java index 9c2fd118d59d9..52d87c2e32c87 100644 --- a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java +++ b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java @@ -89,32 +89,39 @@ public class JwtRestIT extends ESRestTestCase { put("xpack.security.authc.realms.jwt.jwt2.client_authentication.shared_secret", VALID_SHARED_SECRET); } }; + private static final String KEYSTORE_PASSWORD = "keystore-password"; @ClassRule public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .nodes(2) .distribution(DistributionType.DEFAULT) + .keystorePassword(KEYSTORE_PASSWORD) .configFile("http.key", Resource.fromClasspath("ssl/http.key")) .configFile("http.crt", Resource.fromClasspath("ssl/http.crt")) .configFile("ca.crt", Resource.fromClasspath("ssl/ca.crt")) + .configFile("ca-transport.crt", Resource.fromClasspath("ssl/ca-transport.crt")) + .configFile("transport.key", Resource.fromClasspath("ssl/transport.key")) + .configFile("transport.crt", Resource.fromClasspath("ssl/transport.crt")) .configFile("rsa.jwkset", Resource.fromClasspath("jwk/rsa-public-jwkset.json")) .setting("xpack.ml.enabled", "false") .setting("xpack.license.self_generated.type", "trial") .setting("xpack.security.enabled", "true") - .setting("xpack.security.http.ssl.enabled", "true") - .setting("xpack.security.transport.ssl.enabled", "false") + .setting("xpack.security.transport.ssl.enabled", "true") + .setting("xpack.security.transport.ssl.certificate", "transport.crt") + .setting("xpack.security.transport.ssl.key", "transport.key") + .setting("xpack.security.transport.ssl.certificate_authorities", "ca-transport.crt") .setting("xpack.security.authc.token.enabled", "true") .setting("xpack.security.authc.api_key.enabled", "true") - .setting("xpack.security.http.ssl.enabled", "true") .setting("xpack.security.http.ssl.certificate", "http.crt") .setting("xpack.security.http.ssl.key", "http.key") - .setting("xpack.security.http.ssl.key_passphrase", "http-password") .setting("xpack.security.http.ssl.certificate_authorities", "ca.crt") .setting("xpack.security.http.ssl.client_authentication", "optional") .settings(JwtRestIT::realmSettings) .keystore("xpack.security.authc.realms.jwt.jwt2.hmac_key", HMAC_PASSPHRASE) .keystore("xpack.security.authc.realms.jwt.jwt3.hmac_jwkset", HMAC_JWKSET) + .keystore("xpack.security.http.ssl.secure_key_passphrase", "http-password") + .keystore("xpack.security.transport.ssl.secure_key_passphrase", "transport-password") .keystore("xpack.security.authc.realms.jwt.jwt3.client_authentication.shared_secret", VALID_SHARED_SECRET) .keystore(keystoreSettings) .user("admin_user", "admin-password") @@ -508,7 +515,6 @@ public void testAuthenticationFailureIfDelegatedAuthorizationFails() throws Exce } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103308") public void testReloadClientSecret() throws Exception { final String principal = SERVICE_SUBJECT.get(); final String username = getUsernameFromPrincipal(principal); @@ -562,7 +568,9 @@ private void writeSettingToKeystoreThenReload(String setting, @Nullable String v keystoreSettings.put(setting, value); } cluster.updateStoredSecureSettings(); - assertOK(adminClient().performRequest(new Request("POST", "/_nodes/reload_secure_settings"))); + final var reloadRequest = new Request("POST", "/_nodes/reload_secure_settings"); + reloadRequest.setJsonEntity("{\"secure_settings_password\":\"" + KEYSTORE_PASSWORD + "\"}"); + assertOK(adminClient().performRequest(reloadRequest)); } public void testFailureOnInvalidClientAuthentication() throws Exception { diff --git a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/README.asciidoc b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/README.asciidoc index 37185a996fbba..cb1aadc94e4ac 100644 --- a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/README.asciidoc +++ b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/README.asciidoc @@ -28,6 +28,20 @@ rm http.zip rmdir http ----------------------------------------------------------------------------------------------------------- +[source,shell] +----------------------------------------------------------------------------------------------------------- +elasticsearch-certutil cert --pem --name=transport --out=transport.zip --pass="transport-password" --days=3500 \ + --ca-cert=ca-transport.crt --ca-key=ca-transport.key --ca-pass="ca-password" \ + --dns=localhost --dns=localhost.localdomain --dns=localhost4 --dns=localhost4.localdomain4 --dns=localhost6 --dns=localhost6.localdomain6 \ + --ip=127.0.0.1 --ip=0:0:0:0:0:0:0:1 + +unzip transport.zip +mv transport/transport.* ./ + +rm transport.zip +rmdir transport +----------------------------------------------------------------------------------------------------------- + [source,shell] ----------------------------------------------------------------------------------------------------------- elasticsearch-certutil cert --pem --name=pki --out=${PWD}/pki.zip --pass="pki-password" --days=3500 \ diff --git a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/ca-transport.crt b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/ca-transport.crt new file mode 100644 index 0000000000000..320f00ccde59b --- /dev/null +++ b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/ca-transport.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSTCCAjGgAwIBAgIUGuBmPtwyEv7WZ1H0Yy5vyEEYVR8wDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMjAwNDA3MTEzMDA1WhcNMjkxMTA2MTEzMDA1WjA0MTIwMAYD +VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALeTNx0a6X+Fhf6IQj4ggN9U +1HGIzJKEHGIpATDgbfdIv88e0O0I6HN7pmLf5LuUPDGc2oLGnxqATgnFek5eJ4QW +sKgflGB4C0EgQH4JAooIG0EI6aj3IcdzBwH8bdymAdsGj0Zcvm6wjhLixgiN3yIM +8KJAtJrSCITI88gfXhXyU0XCSzgruFkdvHjFBCWpCaK3hnjoiO65186PcGbrZHB8 +Izs2soa6H1AHVDMhmJjlwJWYtibjok+sgrjkDWG7cBh6Al7yXGUBOs9SgMXUpI3Y +0r/dDdeISdI5VzwKZpX6qYcNJI+jtgZUD0alMKBxjq3+v8GlDE/QVNyDwp/7SA8C +AwEAAaNTMFEwHQYDVR0OBBYEFMdSVLWtAhqfDXRQj+5o80nK1XaQMB8GA1UdIwQY +MBaAFMdSVLWtAhqfDXRQj+5o80nK1XaQMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBAEiaX+JtgDb6PuPqO9NtxuPEsg6Ghj0dVYLOYE9cmKS9xeyy +sIDJbPl8vy7gomVBDYe+3qM2UElKFdApKGjOctjNlH8npkQxg72sarBUzOfuFPx5 +Z6u5TuKMfqjx2dwpTr3Yp/iUFs9e9zcDMnQYt5B6SCjKpKKodKKh+CE6wOg4nbSF +43OYO9HFHfwIlEqvof7j0r5gnPf5fYSYybYFAqy4oAfpESPq9lJuEvA46TrGpmP6 +IpMYkJJ6O+98A7RHo5kstZJdnG5paAKobdPEYxbIZvRyMJ8IxW8kSAaTKsK7W34k +IYciDd/YY3R+nhnh8F5DjVcyc79Zkv9Cjig/OxQ= +-----END CERTIFICATE----- diff --git a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/ca-transport.key b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/ca-transport.key new file mode 100644 index 0000000000000..6bbdd44d274df --- /dev/null +++ b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/ca-transport.key @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-128-CBC,8209E02F62E3909502FECF5E5E9CF7A7 + +EdOFZ6/z/e4elfeAKs2B++/Px/IpiZdmiseZPjfwa6jgpY+8sehmze5+34vrxYJT +cMBH3QafmhdQZ4/Eo7DVFONrjJ3OmD5//ZiTIujTPwMsgGAdeq0yMC0cDkzg8SQ7 +KvTh0PY0feC6bVsY+YjDprDfpqIWf89F8ikgat9cmucV9YO3RbYnxgxRIztbHLP3 +GenAtdG+v7DzdefAdRQktBSNldkadsY6d/kVBknOHcA4pB/UtDpz77ZF40CNB95z +1Tr37nNnuRBUNHbKklXuozkvYLah66tFxA5v7Rf6F37d2QGBkgDphg/QMbJrrB+q +MsfiXeXqRaCzBN/ZuzTQAdQ/67XpQ+Ax89UOiT6SkKBKN1uNDk7Juzv5zHrq7aWS +aj1qtHDG2vMB+UM5A1MngD1LtXzs21Q0+9a2UT83x+VIP0hVq2uKmO8wAQ9gbBe9 +wkBPca4gLYlbIMWzaAe4DV0rcmux+i7Ezk8oVYW1JcoGjoZ3f9KewIQynBUlXXuO +EzSl4R3yiF/birrK9Lo6c9hOcQKCW2qAX73BKq8PjKgWT3rnqzg97q9PPK/vaien +fwSrTXDgEoy1RCwsPsxjyRf0LGFYLUFRVqrbFPhhjg4aEiuzawcpvRxjorC5UX0L +dpImNssdALDd0BbiqAbChUryFSSxFhQ2yo6hfUXZevD236b09V0jUpnZeyQjeTTk +fhhAUUpnd1YzWuYneD2JZQKvGdgWgYRyEKParFeHLjp95rXNWPSOgoAM+w0fFEjq +zkYQMaDGSnUWbc6LVv2exyRIRTrLAWamKnne7z8VxzetqXXmuX0WJb2lFiYMUw4/ +wf31RA8ZsVSgb9werSyPD9aRe/+YZM+kM3/3MC4jJGc6OJuDqEOhhB06L2Df2AWU +UQwZ7y+2yUC1kcFzc8+oT1TNgBHixouY+oqWkhbdCkbUFUe4FwXNXrMyrY9gZs1/ +PEkhVxxYgpLwifkbfQRJPeJvXxh7NxeolXyISaVENdLkMMYUhdsKTa+GOQbO2yfa +4BhOwAqJvyDFfsRxLiDlbxjzvY5qnMl0e/q8wZ60onHJOFCTCfm2BNx7sW+Sk5Kx +zm0Rxsz4rIIxA5S6zbbdsHxjTC9XiUelKaq+W0XTg76USYneORQNN/Mk9sCXvTud +HUqmSf1wREA1PdEcoJ3tMoAOZWGY43/IrdoG3bTNT96AdToD+D+Or8M2VcOZorVf +c3IRNfxGv2/SwhxW/z4tSLSToSJlt4QKxU9Xzm4UundDy1cHmS1faN6+bBnI5+/F +OKwzPCCUJ6H02CAjx2P/P6YEjoLl8B+7h4whlOfT/+IQbzOcGMpPyGu4jSf1KffA +asAQeBvYTx0QPdv2E7e216RLOlp/ERMzkUvF1G7UYKF7Ao6cUpSH6nvGABPLKNXV +fqjpWq8O4R1UEUXi6dqF1HfAHllI+vMw7LzRJK/5zVrWlJPm4c/Rng5OkK7aAGee +J0eTSlCdNpyaZzjyk2ZAQ54kZVqAS90zS1zo6lg2v9yfAfz6eYlfl2OGfFVG40Jt +oYxEVcG9LeD3XOkPOnTblHdKMor8cQt+TEJPu9eM31ay1QSilixx2yfOOFTgJZOi +-----END RSA PRIVATE KEY----- diff --git a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/transport.crt b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/transport.crt new file mode 100644 index 0000000000000..db93ca6ac4750 --- /dev/null +++ b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/transport.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDuDCCAqCgAwIBAgIVAIdpYPATbRn96E+eVTG/s0byNh/FMA0GCSqGSIb3DQEB +CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu +ZXJhdGVkIENBMB4XDTIwMDQwNzExMzA1OFoXDTI5MTEwNjExMzA1OFowFDESMBAG +A1UEAxMJdHJhbnNwb3J0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +pvfY9x3F8xR2+noX0PWxsc6XrKCxTQcbSHkkEr2Qr6BqGVXwEj/d6ocqdgcC7IZy +0HEwewBbO69Pg8INQ/21stcwZzW+4ZnKYsBmaZx1yCxYcplndFm9t5Fj/jTBsoJ3 +PemvJwMrewuVSwsE4WHVkXz4KVETfUd8DZiWoVnlRgaXfvvudib1DNxtuGEra+Zh +d3JcC1Wsn51qjpHsj/6s/usT6hmlm4Bu5tjAMxXFVX6J0skfRSVhLmNWgr86WBKB +9/qTJU34FBQGh2Ok/knkiO5rae+UCPpEpCNCCV3rFcMdxP613WfemRRqhUL0V6g3 +n4ExJa0853SsfvPEyHOADQIDAQABo4HgMIHdMB0GA1UdDgQWBBSraIvkNPX2TQQg +h8Ee3mWCALYr/zAfBgNVHSMEGDAWgBTHUlS1rQIanw10UI/uaPNJytV2kDCBjwYD +VR0RBIGHMIGEgglsb2NhbGhvc3SCF2xvY2FsaG9zdDYubG9jYWxkb21haW42hwR/ +AAABhxAAAAAAAAAAAAAAAAAAAAABggpsb2NhbGhvc3Q0ggpsb2NhbGhvc3Q2ghVs +b2NhbGhvc3QubG9jYWxkb21haW6CF2xvY2FsaG9zdDQubG9jYWxkb21haW40MAkG +A1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEBABRKISeKdMRRQAvZBan2ONUFUD3P +7Aq8JP+GttC9Y+1uI0CTIZsnih+tuZ9l2Gq9nyfBVMcDSutlnBoQZLg5lCYQPaK7 +SuFhPRHGauEGYw1TjhrMEPokxzoT/X0/Sz5Ter6/qWzPKQs9EuaVJ27XfZd+kySn +S+cXd95woi+S8JQzQbcpA0rXCnWJ3l2frwG/3Hg8f82x2c6vgOzTG0Hklp0sFkUt +UqaBHGXPLiitaB01jUX60HZbxt5HIEseLctUmQlDtAEWwA3X6cRUEjulwRx8s52T +1FT2ORbVJ7ybKARGBSs932Fv2rWGmg8pOBA4ulJTJNvT0T+ob/H/i40Qd04= +-----END CERTIFICATE----- diff --git a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/transport.key b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/transport.key new file mode 100644 index 0000000000000..601c665a48ce6 --- /dev/null +++ b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/transport.key @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-128-CBC,DAC0DDB93011ABD08161118074F353A7 + +hPjzr8y4t3omv6jItFSxF/UeirrdlMhFoxxsw+E5fl4hRjD2J6LuUpOl0XBuvrCO +2NN9Simlkfo57l2O8tZ3xwKU037x9qP2O3wo0FZ4OuRcLbXZtp5kIV30/wdo0kbp +GV+18PtGfReo75rszs/VAm9Hg1URqVw0La2r7DomYQB9FJY8N8mwSdSvF194kjGO +pBxiuzzECUwXEGuMRzmc1Cddbw7NsIdg43FRd1uoC4dqj9yBonYEYe5P8WgopL4N +obTi6PzH+kqDSCaJo7Fdr9CYo37f2YsSbtHmuEZP58J/aSB9nl5wdAmas3/dohrI +5GSM9zp+UocFuV6Uf+X9TTJMt97BlRgFdPODh88pTKGLVQyKeBPQbVjgwl9mttxO +i+c/dej/jHt0gwlt8cvZw0Ss50YdNnWtck91yYpXE7iz59CTY+QI24DEvsaP4bkR +QYdIhJHOYamGW0ttCSU8bw1h9RubIvSa+BoiuB+1TaCYU+azuaAYnFlyuR31z4rD +yniPMnb0+5uOkU/srwb4MxVVw/0iYkKAGTEwdLPKhyheuDU9ixkNOQ/k12zV0R7d +gzMFQOlrB4v8Y4LrsNPnAz/uCTvKgBrOS8p076qeGkSX+JIZVNHYyzLnSy7p6hjO +eD3tDx/SA1HaiLzD1VqujnYb6wshYjQGkSPSY3COq8dQgpCqMAlkOycUQO1SbuNt +HZFv9X0w2z5HjPJXtKLLXMLeluNNRQD+IVhvbZjIM1cAUQNqL3OQPGa7W5RYoFYK +rDffzQAzukD5dt6jH+uu3cwnEeJiW8hxZ0+DHJR1X5EJWpN544yTl8jgSPT8MPAU +kxq7OyE0F/JY+UWP1hPILimDrf3Ov8KRtTDGsSvV3IcX+92QKMcvnK21QBZqZjSs +zcmjp2jN1MLwieJyZ3un0MUT9kOyG/5vGoAJe9O/KDtv6rrhKQN5JHi5yKw0Uwi9 +CwrwwkxbRLSBbWugZGXyBHkR/RGIuEEysLKRFej2q4WBZrPOzZlgyvgBbd6/4Eg5 +twngo6JTmYALwVJNW/drok1H0JelanZ6jM/JjNRFWMZnS5o+4cwQURB5O7TIKHdV +7zEkaw82Ng0Nq8dPrU8N9G3LTmIC1E4t++L+D18C2lV0ZDd0Svh3NIA65FXSRvX9 +2g9GQGfGGbgydmjv9j5lx6VdhuTgYKRL4b5bS7VnH+F9m864g/MtSQpXQPR5B54g +YHFGiKCAzruZt1MmJ5m8Jvpg84i2lIZkGImwAstV7xVkmQoC3i77awmcQP6s7rJd +Lo7RKEysVPDbzdnZnWGK0PWJwtgsqrAvVcK7ghygi+vSQkDF0L7qunchOKa00oZR +LZa08b5BWuXeqw4lXZDQDT7hk3NUyW3H7Z1uxUlt1kvcGb6zrInW6Bin0hqsODvj +0drMOZp/5NTDSwcEzkW+LgjfKZw8Szmhlt3v+luNFr3KzbnFtEvewD1OVikNGzm9 +sfZ899zNkWfvNJaXL3bvzbTn9d8T15YKCwO9RqPpYKDqXBaC4+OjbNsy4AW/JHPr +H/i3D3rhMXR/CALhp4+Knq4o3vMA+3TsUeZ3lOTogobVloWfixIIiRXfaqT4LmEC +-----END RSA PRIVATE KEY----- diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/GroupsResolverTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/GroupsResolverTestCase.java index 57bda2ad9cc1d..09a7e33e51901 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/GroupsResolverTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/GroupsResolverTestCase.java @@ -54,6 +54,10 @@ protected static RealmConfig config(RealmConfig.RealmIdentifier realmId, Setting @Before public void setUpLdapConnection() throws Exception { + doSetupLdapConnection(); + } + + protected void doSetupLdapConnection() throws Exception { Path trustPath = getDataPath(trustPath()); this.ldapConnection = LdapTestUtils.openConnection(ldapUrl(), bindDN(), bindPassword(), trustPath); } diff --git a/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java b/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java index 0efe2797c7f76..556a417fb5e79 100644 --- a/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java +++ b/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java @@ -43,7 +43,6 @@ public class XPackRestIT extends AbstractXPackRestTest { .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB") .user("x_pack_rest_user", "x-pack-test-password") .feature(FeatureFlag.TIME_SERIES_MODE) - .feature(FeatureFlag.LEARNING_TO_RANK) .configFile("testnode.pem", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem")) .configFile("testnode.crt", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt")) .configFile("service_tokens", Resource.fromClasspath("service_tokens")) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/text_expansion_search.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/text_expansion_search.yml index a099e327c32f0..5e29d3cdf2ae6 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/text_expansion_search.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/text_expansion_search.yml @@ -107,3 +107,183 @@ setup: model_text: "octopus comforter smells" - match: { hits.total.value: 4 } - match: { hits.hits.0._source.source_text: "the octopus comforter smells" } + +--- +"Test text expansion search with pruning config": + - skip: + version: " - 8.12.99" + reason: "pruning introduced in 8.13.0" + + - do: + search: + index: index-with-rank-features + body: + query: + text_expansion: + ml.tokens: + model_id: text_expansion_model + model_text: "octopus comforter smells" + pruning_config: + tokens_freq_ratio_threshold: 4 + tokens_weight_threshold: 0.4 + - match: { hits.total.value: 4 } + - match: { hits.hits.0._source.source_text: "the octopus comforter smells" } + +--- +"Test named, boosted text expansion search with pruning config": + - skip: + version: " - 8.12.99" + reason: "pruning introduced in 8.13.0" + - do: + search: + index: index-with-rank-features + body: + query: + text_expansion: + ml.tokens: + model_id: text_expansion_model + model_text: "octopus comforter smells" + pruning_config: + tokens_freq_ratio_threshold: 4 + tokens_weight_threshold: 0.4 + - match: { hits.total.value: 4 } + - match: { hits.hits.0._source.source_text: "the octopus comforter smells" } + - match: { hits.hits.0._score: 3.0 } + + - do: + search: + index: index-with-rank-features + body: + query: + text_expansion: + ml.tokens: + model_id: text_expansion_model + model_text: "octopus comforter smells" + pruning_config: + tokens_freq_ratio_threshold: 4 + tokens_weight_threshold: 0.4 + _name: i-like-naming-my-queries + boost: 100.0 + - match: { hits.total.value: 4 } + - match: { hits.hits.0._source.source_text: "the octopus comforter smells" } + - match: { hits.hits.0.matched_queries: ["i-like-naming-my-queries"] } + - match: { hits.hits.0._score: 300.0 } + +--- +"Test text expansion search with default pruning config": + - skip: + version: " - 8.12.99" + reason: "pruning introduced in 8.13.0" + + - do: + search: + index: index-with-rank-features + body: + query: + text_expansion: + ml.tokens: + model_id: text_expansion_model + model_text: "octopus comforter smells" + pruning_config: {} + - match: { hits.total.value: 4 } + - match: { hits.hits.0._source.source_text: "the octopus comforter smells" } + +--- +"Test text expansion search with weighted tokens rescoring only pruned tokens": + - skip: + version: " - 8.12.99" + reason: "pruning introduced in 8.13.0" + + - do: + search: + index: index-with-rank-features + body: + query: + text_expansion: + ml.tokens: + model_id: text_expansion_model + model_text: "octopus comforter smells" + pruning_config: + tokens_freq_ratio_threshold: 4 + tokens_weight_threshold: 0.4 + only_score_pruned_tokens: true + - match: { hits.total.value: 0 } + +--- +"Test weighted tokens search": + - skip: + version: " - 8.12.99" + reason: "weighted token search introduced in 8.13.0" + + - do: + search: + index: index-with-rank-features + body: + query: + weighted_tokens: + ml.tokens: + tokens: [{"the": 1.0}, {"comforter":1.0}, {"smells":1.0}, {"bad": 1.0}] + pruning_config: + tokens_freq_ratio_threshold: 1 + tokens_weight_threshold: 0.4 + only_score_pruned_tokens: false + - match: { hits.total.value: 5 } + - match: { hits.hits.0._source.source_text: "the octopus comforter smells" } + +--- +"Test weighted tokens search with default pruning config": + - skip: + version: " - 8.12.99" + reason: "weighted token search introduced in 8.13.0" + + - do: + search: + index: index-with-rank-features + body: + query: + weighted_tokens: + ml.tokens: + tokens: [{"the": 1.0}, {"comforter":1.0}, {"smells":1.0}, {"bad": 1.0}] + pruning_config: {} + - match: { hits.total.value: 5 } + - match: { hits.hits.0._source.source_text: "the octopus comforter smells" } + +--- +"Test weighted tokens search only scoring pruned tokens": + - skip: + version: " - 8.12.99" + reason: "weighted token search introduced in 8.13.0" + + - do: + search: + index: index-with-rank-features + body: + query: + weighted_tokens: + ml.tokens: + tokens: [{"the": 1.0}, {"comforter":1.0}, {"smells":1.0}, {"bad": 1.0}] + pruning_config: + tokens_freq_ratio_threshold: 4 + tokens_weight_threshold: 0.4 + only_score_pruned_tokens: true + - match: { hits.total.value: 0 } + +--- +"Test weighted tokens search that prunes tokens based on frequency": + - skip: + version: " - 8.12.99" + reason: "weighted token search introduced in 8.13.0" + + - do: + search: + index: index-with-rank-features + body: + query: + weighted_tokens: + ml.tokens: + tokens: [{"the": 1.0}, {"octopus":1.0}, {"comforter":1.0}, {"is": 1.0}, {"the": 1.0}, {"best": 1.0}, {"of": 1.0}, {"the": 1.0}, {"bunch": 1.0}] + pruning_config: + tokens_freq_ratio_threshold: 3 + tokens_weight_threshold: 0.4 + only_score_pruned_tokens: true + - match: { hits.total.value: 0 } diff --git a/x-pack/qa/oidc-op-tests/build.gradle b/x-pack/qa/oidc-op-tests/build.gradle index 3987073b1e6ad..8f46613d5d9f0 100644 --- a/x-pack/qa/oidc-op-tests/build.gradle +++ b/x-pack/qa/oidc-op-tests/build.gradle @@ -1,16 +1,13 @@ import org.elasticsearch.gradle.internal.info.BuildParams apply plugin: 'elasticsearch.internal-java-rest-test' -apply plugin: 'elasticsearch.test.fixtures' dependencies { javaRestTestImplementation(testArtifact(project(xpackModule('core')))) javaRestTestImplementation(testArtifact(project(xpackModule('security')))) + javaRestTestImplementation project(":x-pack:test:idp-fixture") } -testFixtures.useFixture ":x-pack:test:idp-fixture", "http-proxy" -testFixtures.useFixture ":x-pack:test:idp-fixture", "oidc-provider" - tasks.named('javaRestTest') { usesDefaultDistribution() // test suite uses jks which is not supported in fips mode diff --git a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtWithOidcAuthIT.java b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtWithOidcAuthIT.java index 2d3fc611758b0..18224c887c663 100644 --- a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtWithOidcAuthIT.java +++ b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtWithOidcAuthIT.java @@ -106,7 +106,7 @@ public void testAuthenticateWithOidcIssuedJwt() throws Exception { new Scope(OIDCScopeValue.OPENID), new ClientID(clientId), new URI(redirectUri) - ).endpointURI(new URI(C2ID_AUTH_ENDPOINT)).state(new State(state)).nonce(new Nonce(nonce)).build(); + ).endpointURI(new URI(c2id.getC2OPUrl() + "/c2id-login")).state(new State(state)).nonce(new Nonce(nonce)).build(); final String implicitFlowURI = authenticateAtOP(oidcAuthRequest.toURI()); diff --git a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/C2IdOpTestCase.java b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/C2IdOpTestCase.java index 106e1d27910f2..56d3bbe77c78a 100644 --- a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/C2IdOpTestCase.java +++ b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/C2IdOpTestCase.java @@ -10,6 +10,8 @@ import net.minidev.json.JSONObject; import net.minidev.json.parser.JSONParser; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.StatusLine; @@ -38,11 +40,17 @@ import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.test.fixtures.idp.HttpProxyTestContainer; +import org.elasticsearch.test.fixtures.idp.OidcProviderTestContainer; +import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.core.common.socket.SocketAccess; import org.hamcrest.Matchers; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; +import org.testcontainers.containers.Network; import java.io.FileNotFoundException; import java.io.IOException; @@ -56,25 +64,21 @@ import static org.hamcrest.Matchers.equalTo; +@ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) public abstract class C2IdOpTestCase extends ESRestTestCase { protected static final String TEST_SUBJECT_ID = "alice"; - // URLs for accessing the C2id OP - private static final String C2OP_PORT = getEphemeralTcpPortFromProperty("oidc-provider", "8080"); - private static final String C2ID_HOST = "http://127.0.0.1:" + C2OP_PORT; - protected static final String C2ID_ISSUER = C2ID_HOST + "/c2id"; - private static final String PROXY_PORT = getEphemeralTcpPortFromProperty("http-proxy", "8888"); - private static final String C2ID_LOGIN_API = C2ID_HOST + "/c2id-login/api/"; - private static final String C2ID_REGISTRATION_URL = C2ID_HOST + "/c2id/clients"; - protected static final String C2ID_AUTH_ENDPOINT = C2ID_HOST + "/c2id-login"; - // SHA256 of this is defined in x-pack/test/idp-fixture/oidc/override.properties private static final String OP_API_BEARER_TOKEN = "811fa888f3e0fdc9e01d4201bfeee46a"; private static Path HTTP_TRUSTED_CERT; private static final String CLIENT_SECRET = "b07efb7a1cf6ec9462afe7b6d3ab55c6c7880262aa61ac28dded292aca47c9a2"; - @ClassRule + + private static Network network = Network.newNetwork(); + protected static OidcProviderTestContainer c2id = new OidcProviderTestContainer(network); + protected static HttpProxyTestContainer proxy = new HttpProxyTestContainer(network); + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) .nodes(1) @@ -86,10 +90,10 @@ public abstract class C2IdOpTestCase extends ESRestTestCase { .setting("xpack.security.authc.realms.file.file.order", "0") .setting("xpack.security.authc.realms.native.native.order", "1") .setting("xpack.security.authc.realms.oidc.c2id.order", "2") - .setting("xpack.security.authc.realms.oidc.c2id.op.issuer", C2ID_ISSUER) - .setting("xpack.security.authc.realms.oidc.c2id.op.authorization_endpoint", C2ID_HOST + "/c2id-login") - .setting("xpack.security.authc.realms.oidc.c2id.op.token_endpoint", C2ID_HOST + "/c2id/token") - .setting("xpack.security.authc.realms.oidc.c2id.op.userinfo_endpoint", C2ID_HOST + "/c2id/userinfo") + .setting("xpack.security.authc.realms.oidc.c2id.op.issuer", () -> c2id.getC2IssuerUrl()) + .setting("xpack.security.authc.realms.oidc.c2id.op.authorization_endpoint", () -> c2id.getC2OPUrl() + "/c2id-login") + .setting("xpack.security.authc.realms.oidc.c2id.op.token_endpoint", () -> c2id.getC2OPUrl() + "/c2id/token") + .setting("xpack.security.authc.realms.oidc.c2id.op.userinfo_endpoint", () -> c2id.getC2OPUrl() + "/c2id/userinfo") .setting("xpack.security.authc.realms.oidc.c2id.op.jwkset_path", "op-jwks.json") .setting("xpack.security.authc.realms.oidc.c2id.rp.redirect_uri", "https://my.fantastic.rp/cb") .setting("xpack.security.authc.realms.oidc.c2id.rp.client_id", "https://my.elasticsearch.org/rp") @@ -99,10 +103,10 @@ public abstract class C2IdOpTestCase extends ESRestTestCase { .setting("xpack.security.authc.realms.oidc.c2id.claims.mail", "email") .setting("xpack.security.authc.realms.oidc.c2id.claims.groups", "groups") .setting("xpack.security.authc.realms.oidc.c2id-implicit.order", "3") - .setting("xpack.security.authc.realms.oidc.c2id-implicit.op.issuer", C2ID_ISSUER) - .setting("xpack.security.authc.realms.oidc.c2id-implicit.op.authorization_endpoint", C2ID_HOST + "/c2id-login") - .setting("xpack.security.authc.realms.oidc.c2id-implicit.op.token_endpoint", C2ID_HOST + "/c2id/token") - .setting("xpack.security.authc.realms.oidc.c2id-implicit.op.userinfo_endpoint", C2ID_HOST + "/c2id/userinfo") + .setting("xpack.security.authc.realms.oidc.c2id-implicit.op.issuer", () -> c2id.getC2IssuerUrl()) + .setting("xpack.security.authc.realms.oidc.c2id-implicit.op.authorization_endpoint", () -> c2id.getC2OPUrl() + "/c2id-login") + .setting("xpack.security.authc.realms.oidc.c2id-implicit.op.token_endpoint", () -> c2id.getC2OPUrl() + "/c2id/token") + .setting("xpack.security.authc.realms.oidc.c2id-implicit.op.userinfo_endpoint", () -> c2id.getC2OPUrl() + "/c2id/userinfo") .setting("xpack.security.authc.realms.oidc.c2id-implicit.op.jwkset_path", "op-jwks.json") .setting("xpack.security.authc.realms.oidc.c2id-implicit.rp.redirect_uri", "https://my.fantastic.rp/cb") .setting("xpack.security.authc.realms.oidc.c2id-implicit.rp.client_id", "elasticsearch-rp") @@ -112,10 +116,10 @@ public abstract class C2IdOpTestCase extends ESRestTestCase { .setting("xpack.security.authc.realms.oidc.c2id-implicit.claims.mail", "email") .setting("xpack.security.authc.realms.oidc.c2id-implicit.claims.groups", "groups") .setting("xpack.security.authc.realms.oidc.c2id-proxy.order", "4") - .setting("xpack.security.authc.realms.oidc.c2id-proxy.op.issuer", C2ID_ISSUER) - .setting("xpack.security.authc.realms.oidc.c2id-proxy.op.authorization_endpoint", C2ID_HOST + "/c2id-login") - .setting("xpack.security.authc.realms.oidc.c2id-proxy.op.token_endpoint", C2ID_HOST + "/c2id/token") - .setting("xpack.security.authc.realms.oidc.c2id-proxy.op.userinfo_endpoint", C2ID_HOST + "/c2id/userinfo") + .setting("xpack.security.authc.realms.oidc.c2id-proxy.op.issuer", () -> c2id.getC2IssuerUrl()) + .setting("xpack.security.authc.realms.oidc.c2id-proxy.op.authorization_endpoint", () -> c2id.getC2OPUrl() + "/c2id-login") + .setting("xpack.security.authc.realms.oidc.c2id-proxy.op.token_endpoint", () -> c2id.getC2OPUrl() + "/c2id/token") + .setting("xpack.security.authc.realms.oidc.c2id-proxy.op.userinfo_endpoint", () -> c2id.getC2OPUrl() + "/c2id/userinfo") .setting("xpack.security.authc.realms.oidc.c2id-proxy.op.jwkset_path", "op-jwks.json") .setting("xpack.security.authc.realms.oidc.c2id-proxy.rp.redirect_uri", "https://my.fantastic.rp/cb") .setting("xpack.security.authc.realms.oidc.c2id-proxy.rp.client_id", "https://my.elasticsearch.org/rp") @@ -125,12 +129,12 @@ public abstract class C2IdOpTestCase extends ESRestTestCase { .setting("xpack.security.authc.realms.oidc.c2id-proxy.claims.mail", "email") .setting("xpack.security.authc.realms.oidc.c2id-proxy.claims.groups", "groups") .setting("xpack.security.authc.realms.oidc.c2id-proxy.http.proxy.host", "127.0.0.1") - .setting("xpack.security.authc.realms.oidc.c2id-proxy.http.proxy.port", PROXY_PORT) + .setting("xpack.security.authc.realms.oidc.c2id-proxy.http.proxy.port", () -> proxy.getProxyPort().toString()) .setting("xpack.security.authc.realms.oidc.c2id-post.order", "5") - .setting("xpack.security.authc.realms.oidc.c2id-post.op.issuer", C2ID_ISSUER) - .setting("xpack.security.authc.realms.oidc.c2id-post.op.authorization_endpoint", C2ID_HOST + "/c2id-login") - .setting("xpack.security.authc.realms.oidc.c2id-post.op.token_endpoint", C2ID_HOST + "/c2id/token") - .setting("xpack.security.authc.realms.oidc.c2id-post.op.userinfo_endpoint", C2ID_HOST + "/c2id/userinfo") + .setting("xpack.security.authc.realms.oidc.c2id-post.op.issuer", () -> c2id.getC2IssuerUrl()) + .setting("xpack.security.authc.realms.oidc.c2id-post.op.authorization_endpoint", () -> c2id.getC2OPUrl() + "/c2id-login") + .setting("xpack.security.authc.realms.oidc.c2id-post.op.token_endpoint", () -> c2id.getC2OPUrl() + "/c2id/token") + .setting("xpack.security.authc.realms.oidc.c2id-post.op.userinfo_endpoint", () -> c2id.getC2OPUrl() + "/c2id/userinfo") .setting("xpack.security.authc.realms.oidc.c2id-post.op.jwkset_path", "op-jwks.json") .setting("xpack.security.authc.realms.oidc.c2id-post.rp.redirect_uri", "https://my.fantastic.rp/cb") .setting("xpack.security.authc.realms.oidc.c2id-post.rp.client_id", "elasticsearch-post") @@ -141,10 +145,10 @@ public abstract class C2IdOpTestCase extends ESRestTestCase { .setting("xpack.security.authc.realms.oidc.c2id-post.claims.mail", "email") .setting("xpack.security.authc.realms.oidc.c2id-post.claims.groups", "groups") .setting("xpack.security.authc.realms.oidc.c2id-jwt.order", "6") - .setting("xpack.security.authc.realms.oidc.c2id-jwt.op.issuer", C2ID_ISSUER) - .setting("xpack.security.authc.realms.oidc.c2id-jwt.op.authorization_endpoint", C2ID_HOST + "/c2id-login") - .setting("xpack.security.authc.realms.oidc.c2id-jwt.op.token_endpoint", C2ID_HOST + "/c2id/token") - .setting("xpack.security.authc.realms.oidc.c2id-jwt.op.userinfo_endpoint", C2ID_HOST + "/c2id/userinfo") + .setting("xpack.security.authc.realms.oidc.c2id-jwt.op.issuer", () -> c2id.getC2IssuerUrl()) + .setting("xpack.security.authc.realms.oidc.c2id-jwt.op.authorization_endpoint", () -> c2id.getC2OPUrl() + "/c2id-login") + .setting("xpack.security.authc.realms.oidc.c2id-jwt.op.token_endpoint", () -> c2id.getC2OPUrl() + "/c2id/token") + .setting("xpack.security.authc.realms.oidc.c2id-jwt.op.userinfo_endpoint", () -> c2id.getC2OPUrl() + "/c2id/userinfo") .setting("xpack.security.authc.realms.oidc.c2id-jwt.op.jwkset_path", "op-jwks.json") .setting("xpack.security.authc.realms.oidc.c2id-jwt.rp.redirect_uri", "https://my.fantastic.rp/cb") .setting("xpack.security.authc.realms.oidc.c2id-jwt.rp.client_id", "elasticsearch-post-jwt") @@ -155,7 +159,7 @@ public abstract class C2IdOpTestCase extends ESRestTestCase { .setting("xpack.security.authc.realms.oidc.c2id-jwt.claims.mail", "email") .setting("xpack.security.authc.realms.oidc.c2id-jwt.claims.groups", "groups") .setting("xpack.security.authc.realms.jwt.op-jwt.order", "7") - .setting("xpack.security.authc.realms.jwt.op-jwt.allowed_issuer", C2ID_ISSUER) + .setting("xpack.security.authc.realms.jwt.op-jwt.allowed_issuer", () -> c2id.getC2IssuerUrl()) .setting("xpack.security.authc.realms.jwt.op-jwt.allowed_audiences", "elasticsearch-jwt1,elasticsearch-jwt2") .setting("xpack.security.authc.realms.jwt.op-jwt.pkc_jwkset_path", "op-jwks.json") .setting("xpack.security.authc.realms.jwt.op-jwt.claims.principal", "sub") @@ -174,6 +178,9 @@ public abstract class C2IdOpTestCase extends ESRestTestCase { .user("x_pack_rest_user", "x-pack-test-password", "superuser", false) .build(); + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(network).around(c2id).around(proxy).around(cluster); + @Override protected String getTestRestCluster() { return cluster.getHttpAddresses(); @@ -193,24 +200,17 @@ public static void readTrustedCert() throws Exception { HTTP_TRUSTED_CERT = PathUtils.get(resource.toURI()); } - protected static String getEphemeralTcpPortFromProperty(String service, String port) { - String key = "test.fixtures." + service + ".tcp." + port; - final String value = System.getProperty(key); - assertNotNull("Expected the actual value for port " + port + " to be in system property " + key, value); - return value; - } - /** * Register one or more OIDC clients on the C2id server. This should be done once (per client) only. * C2id server only supports dynamic registration, so we can't pre-seed its config with our client data. */ protected static void registerClients(String... jsonBody) throws IOException { try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + String ci2dRegistrationUrl = c2id.getC2OPUrl() + "/c2id/clients"; final BasicHttpContext context = new BasicHttpContext(); - final List requests = new ArrayList<>(jsonBody.length); for (String body : jsonBody) { - HttpPost httpPost = new HttpPost(C2ID_REGISTRATION_URL); + HttpPost httpPost = new HttpPost(ci2dRegistrationUrl); httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); httpPost.setHeader("Accept", "application/json"); httpPost.setHeader("Content-type", "application/json"); @@ -240,12 +240,13 @@ protected Settings restAdminSettings() { } protected String authenticateAtOP(URI opAuthUri) throws Exception { + String c2LoginApi = c2id.getC2OPUrl() + "/c2id-login/api/"; // C2ID doesn't have a non JS login page :/, so use their API directly // see https://connect2id.com/products/server/docs/guides/login-page try (CloseableHttpClient httpClient = HttpClients.createDefault()) { final BasicHttpContext context = new BasicHttpContext(); // Initiate the authentication process - HttpPost httpPost = new HttpPost(C2ID_LOGIN_API + "initAuthRequest"); + HttpPost httpPost = new HttpPost(c2LoginApi + "initAuthRequest"); String initJson = Strings.format(""" {"qs":"%s"} """, opAuthUri.getRawQuery()); @@ -258,7 +259,7 @@ protected String authenticateAtOP(URI opAuthUri) throws Exception { final String sid = initResponse.getAsString("sid"); // Actually authenticate the user with ldapAuth HttpPost loginHttpPost = new HttpPost( - C2ID_LOGIN_API + "authenticateSubject?cacheBuster=" + randomAlphaOfLength(8) + "&authSessionId=" + sid + c2LoginApi + "authenticateSubject?cacheBuster=" + randomAlphaOfLength(8) + "&authSessionId=" + sid ); String loginJson = """ {"username":"alice","password":"secret"}"""; @@ -268,9 +269,7 @@ protected String authenticateAtOP(URI opAuthUri) throws Exception { return parseJsonResponse(response); }); - HttpPut consentHttpPut = new HttpPut( - C2ID_LOGIN_API + "updateAuthRequest" + "/" + sid + "?cacheBuster=" + randomAlphaOfLength(8) - ); + HttpPut consentHttpPut = new HttpPut(c2LoginApi + "updateAuthRequest" + "/" + sid + "?cacheBuster=" + randomAlphaOfLength(8)); String consentJson = """ {"claims":["name", "email"],"scope":["openid"]}"""; configureJsonRequest(consentHttpPut, consentJson); diff --git a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java index 9302f731ff285..cd37d86626333 100644 --- a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java +++ b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java @@ -182,7 +182,7 @@ private void verifyElasticsearchAccessTokenForCodeFlow(String accessToken) throw assertThat(map.get("metadata"), instanceOf(Map.class)); final Map metadata = (Map) map.get("metadata"); assertThat(metadata.get("oidc(sub)"), equalTo("alice")); - assertThat(metadata.get("oidc(iss)"), equalTo(C2ID_ISSUER)); + assertThat(metadata.get("oidc(iss)"), equalTo(c2id.getC2IssuerUrl())); } private void verifyElasticsearchAccessTokenForImplicitFlow(String accessToken) throws Exception { @@ -194,7 +194,7 @@ private void verifyElasticsearchAccessTokenForImplicitFlow(String accessToken) t assertThat(map.get("metadata"), instanceOf(Map.class)); final Map metadata = (Map) map.get("metadata"); assertThat(metadata.get("oidc(sub)"), equalTo("alice")); - assertThat(metadata.get("oidc(iss)"), equalTo(C2ID_ISSUER)); + assertThat(metadata.get("oidc(iss)"), equalTo(c2id.getC2IssuerUrl())); } private PrepareAuthResponse getRedirectedFromFacilitator(String realmName) throws Exception { diff --git a/x-pack/qa/openldap-tests/build.gradle b/x-pack/qa/openldap-tests/build.gradle index 844f47c9a53f5..78a03c556bc11 100644 --- a/x-pack/qa/openldap-tests/build.gradle +++ b/x-pack/qa/openldap-tests/build.gradle @@ -1,18 +1,16 @@ apply plugin: 'elasticsearch.standalone-test' -apply plugin: 'elasticsearch.test.fixtures' dependencies { testImplementation(testArtifact(project(xpackModule('security')))) testImplementation(testArtifact(project(xpackModule('core')))) + testImplementation project(":x-pack:test:idp-fixture") + testImplementation "junit:junit:${versions.junit}" + testImplementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + testImplementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" } -testFixtures.useFixture ":x-pack:test:idp-fixture", "openldap" -Project idpFixtureProject = project(":x-pack:test:idp-fixture") -String outputDir = "${project.buildDir}/generated-resources/${project.name}" -def copyIdpTrust = tasks.register("copyIdpTrust", Copy) { - from idpFixtureProject.file('openldap/certs/ca.jks'); - from idpFixtureProject.file('openldap/certs/ca_server.pem'); - into outputDir +tasks.named('test') { + // test suite uses jks which is not supported in fips mode + systemProperty 'tests.security.manager', 'false' } -project.sourceSets.test.output.dir(outputDir, builtBy: copyIdpTrust) diff --git a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java index c1375823548df..7d1610d2ccc0f 100644 --- a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java +++ b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.test; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import com.unboundid.ldap.sdk.LDAPConnection; import com.unboundid.ldap.sdk.LDAPException; @@ -19,6 +20,8 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.test.fixtures.idp.OpenLdapTestContainer; +import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; import org.elasticsearch.test.junit.annotations.Network; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -37,8 +40,8 @@ import org.elasticsearch.xpack.security.authc.ldap.support.SessionFactory; import org.junit.After; import org.junit.Before; +import org.junit.ClassRule; -import java.nio.file.Path; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -53,10 +56,9 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.startsWith; +@ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) public class OpenLdapTests extends ESTestCase { - public static final String OPEN_LDAP_DNS_URL = "ldaps://localhost:" + getFromProperty("636"); - /** * * ip.es.io is magic that will resolve any IP-like DNS name into the embedded IP @@ -67,11 +69,12 @@ public class OpenLdapTests extends ESTestCase { * so in order to have a "not-valid-hostname" failure, we need a second * hostname that isn't in the certificate's Subj Alt Name list */ - private static final String OPEN_LDAP_ES_IO_URL = "ldaps://127.0.0.1.ip.es.io:" + getFromProperty("636"); + + @ClassRule + public static final OpenLdapTestContainer openLdap = new OpenLdapTestContainer(); public static final String PASSWORD = "NickFuryHeartsES"; private static final String HAWKEYE_DN = "uid=hawkeye,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; - public static final String LDAPTRUST_PATH = "/ca.jks"; private static final SecureString PASSWORD_SECURE_STRING = new SecureString(PASSWORD.toCharArray()); public static final String REALM_NAME = "oldap-test"; @@ -96,7 +99,6 @@ public boolean enableWarningsCheck() { @Before public void initializeSslSocketFactory() throws Exception { - Path truststore = getDataPath(LDAPTRUST_PATH); /* * Prior to each test we reinitialize the socket factory with a new SSLService so that we get a new SSLContext. * If we re-use an SSLContext, previously connected sessions can get re-established which breaks hostname @@ -105,14 +107,14 @@ public void initializeSslSocketFactory() throws Exception { MockSecureSettings mockSecureSettings = new MockSecureSettings(); Settings.Builder builder = Settings.builder().put("path.home", createTempDir()); // fake realms so ssl will get loaded - builder.put("xpack.security.authc.realms.ldap.foo.ssl.truststore.path", truststore); + builder.put("xpack.security.authc.realms.ldap.foo.ssl.truststore.path", openLdap.getJavaKeyStorePath()); mockSecureSettings.setString("xpack.security.authc.realms.ldap.foo.ssl.truststore.secure_password", "changeit"); builder.put("xpack.security.authc.realms.ldap.foo.ssl.verification_mode", SslVerificationMode.FULL); - builder.put("xpack.security.authc.realms.ldap." + REALM_NAME + ".ssl.truststore.path", truststore); + builder.put("xpack.security.authc.realms.ldap." + REALM_NAME + ".ssl.truststore.path", openLdap.getJavaKeyStorePath()); mockSecureSettings.setString("xpack.security.authc.realms.ldap." + REALM_NAME + ".ssl.truststore.secure_password", "changeit"); builder.put("xpack.security.authc.realms.ldap." + REALM_NAME + ".ssl.verification_mode", SslVerificationMode.CERTIFICATE); - builder.put("xpack.security.authc.realms.ldap.vmode_full.ssl.truststore.path", truststore); + builder.put("xpack.security.authc.realms.ldap.vmode_full.ssl.truststore.path", openLdap.getJavaKeyStorePath()); mockSecureSettings.setString("xpack.security.authc.realms.ldap.vmode_full.ssl.truststore.secure_password", "changeit"); builder.put("xpack.security.authc.realms.ldap.vmode_full.ssl.verification_mode", SslVerificationMode.FULL); globalSettings = builder.setSecureSettings(mockSecureSettings).build(); @@ -127,7 +129,7 @@ public void testConnect() throws Exception { final RealmConfig.RealmIdentifier realmId = new RealmConfig.RealmIdentifier("ldap", "oldap-test"); RealmConfig config = new RealmConfig( realmId, - buildLdapSettings(realmId, OPEN_LDAP_DNS_URL, userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL), + buildLdapSettings(realmId, openLdap.getLdapUrl(), userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL), TestEnvironment.newEnvironment(globalSettings), new ThreadContext(Settings.EMPTY) ); @@ -150,7 +152,7 @@ public void testGroupSearchScopeBase() throws Exception { final RealmConfig.RealmIdentifier realmId = new RealmConfig.RealmIdentifier("ldap", REALM_NAME); RealmConfig config = new RealmConfig( realmId, - buildLdapSettings(realmId, OPEN_LDAP_DNS_URL, userTemplate, groupSearchBase, LdapSearchScope.BASE), + buildLdapSettings(realmId, openLdap.getLdapUrl(), userTemplate, groupSearchBase, LdapSearchScope.BASE), TestEnvironment.newEnvironment(globalSettings), new ThreadContext(Settings.EMPTY) ); @@ -169,7 +171,7 @@ public void testCustomFilter() throws Exception { String userTemplate = "uid={0},ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; final RealmConfig.RealmIdentifier realmId = new RealmConfig.RealmIdentifier("ldap", "oldap-test"); Settings settings = Settings.builder() - .put(buildLdapSettings(realmId, OPEN_LDAP_DNS_URL, userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL)) + .put(buildLdapSettings(realmId, openLdap.getLdapUrl(), userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL)) .put(getFullSettingKey(realmId.getName(), SearchGroupsResolverSettings.FILTER), "(&(objectclass=posixGroup)(memberUid={0}))") .put(getFullSettingKey(realmId.getName(), SearchGroupsResolverSettings.USER_ATTRIBUTE), "uid") .build(); @@ -192,9 +194,10 @@ public void testStandardLdapConnectionHostnameVerificationFailure() throws Excep String groupSearchBase = "ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; String userTemplate = "uid={0},ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; final RealmConfig.RealmIdentifier realmId = new RealmConfig.RealmIdentifier("ldap", "vmode_full"); + String openLdapEsIoURL = "ldaps://127.0.0.1.ip.es.io:" + openLdap.getDefaultPort(); Settings settings = Settings.builder() // The certificate used in the vagrant box is valid for "localhost", but not for "*.ip.es.io" - .put(buildLdapSettings(realmId, OPEN_LDAP_ES_IO_URL, userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL)) + .put(buildLdapSettings(realmId, openLdapEsIoURL, userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL)) .build(); final Environment env = TestEnvironment.newEnvironment(globalSettings); RealmConfig config = new RealmConfig(realmId, settings, env, new ThreadContext(Settings.EMPTY)); @@ -220,7 +223,7 @@ public void testStandardLdapConnectionHostnameVerificationSuccess() throws Excep final RealmConfig.RealmIdentifier realmId = new RealmConfig.RealmIdentifier("ldap", "vmode_full"); Settings settings = Settings.builder() // The certificate used in the vagrant box is valid for "localhost" (but not for "*.ip.es.io") - .put(buildLdapSettings(realmId, OPEN_LDAP_DNS_URL, userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL)) + .put(buildLdapSettings(realmId, openLdap.getLdapUrl(), userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL)) .build(); RealmConfig config = new RealmConfig( @@ -320,7 +323,7 @@ private Settings buildLdapSettings( Settings.Builder builder = Settings.builder() .put(LdapTestCase.buildLdapSettings(realmId, urls, templates, groupSearchBase, scope, null, false)); builder.put(getFullSettingKey(realmId.getName(), SearchGroupsResolverSettings.USER_ATTRIBUTE), "uid"); - return builder.put(SSLConfigurationSettings.TRUSTSTORE_PATH.realm(realmId).getKey(), getDataPath(LDAPTRUST_PATH)) + return builder.put(SSLConfigurationSettings.TRUSTSTORE_PATH.realm(realmId).getKey(), openLdap.getJavaKeyStorePath()) .put(SSLConfigurationSettings.LEGACY_TRUSTSTORE_PASSWORD.realm(realmId).getKey(), "changeit") .put(globalSettings) .put(getFullSettingKey(realmId, RealmSettings.ORDER_SETTING), 0) @@ -340,8 +343,7 @@ private List groups(LdapSession ldapSession) { } private LDAPConnection setupOpenLdapConnection() throws Exception { - Path truststore = getDataPath(LDAPTRUST_PATH); - return LdapTestUtils.openConnection(OpenLdapTests.OPEN_LDAP_DNS_URL, HAWKEYE_DN, OpenLdapTests.PASSWORD, truststore); + return LdapTestUtils.openConnection(openLdap.getLdapUrl(), HAWKEYE_DN, OpenLdapTests.PASSWORD, openLdap.getJavaKeyStorePath()); } private Map resolve(LDAPConnection connection, LdapMetadataResolver resolver) throws Exception { diff --git a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java index eb3365010b550..60bff29bfb58d 100644 --- a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java +++ b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.security.authc.ldap; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.MockSecureSettings; @@ -16,6 +18,8 @@ import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.OpenLdapTests; +import org.elasticsearch.test.fixtures.idp.OpenLdapTestContainer; +import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.security.authc.RealmConfig; @@ -31,6 +35,7 @@ import org.elasticsearch.xpack.security.authc.ldap.support.SessionFactory; import org.junit.After; import org.junit.Before; +import org.junit.ClassRule; import java.nio.file.Path; import java.text.MessageFormat; @@ -44,15 +49,18 @@ import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.is; +@ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) public class OpenLdapUserSearchSessionFactoryTests extends ESTestCase { private Settings globalSettings; private ThreadPool threadPool; - private static final String LDAPCACERT_PATH = "/ca_server.pem"; + + @ClassRule + public static final OpenLdapTestContainer openLdapContainer = new OpenLdapTestContainer(); @Before public void init() { - Path caPath = getDataPath(LDAPCACERT_PATH); + Path caPath = openLdapContainer.getCaCertPath(); /* * Prior to each test we reinitialize the socket factory with a new SSLService so that we get a new SSLContext. * If we re-use an SSLContext, previously connected sessions can get re-established which breaks hostname @@ -79,7 +87,7 @@ public void testUserSearchWithBindUserOpenLDAP() throws Exception { .put( LdapTestCase.buildLdapSettings( realmId, - new String[] { OpenLdapTests.OPEN_LDAP_DNS_URL }, + new String[] { openLdapContainer.getLdapUrl() }, Strings.EMPTY_ARRAY, groupSearchBase, LdapSearchScope.ONE_LEVEL, diff --git a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java index feec06f4b3b6d..4cc12e5af448e 100644 --- a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java +++ b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java @@ -6,15 +6,21 @@ */ package org.elasticsearch.xpack.security.authc.ldap; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.OpenLdapTests; +import org.elasticsearch.test.fixtures.idp.OpenLdapTestContainer; +import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.ldap.SearchGroupsResolverSettings; import org.elasticsearch.xpack.core.security.authc.ldap.support.LdapSearchScope; import org.elasticsearch.xpack.core.security.support.NoOpLogger; +import org.junit.ClassRule; +import java.nio.file.Path; import java.util.List; import static org.elasticsearch.xpack.core.security.authc.RealmSettings.getFullSettingKey; @@ -24,11 +30,15 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +@ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) public class SearchGroupsResolverTests extends GroupsResolverTestCase { private static final String BRUCE_BANNER_DN = "uid=hulk,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; private static final RealmConfig.RealmIdentifier REALM_ID = new RealmConfig.RealmIdentifier("ldap", "my-ldap-realm"); + @ClassRule + public static final OpenLdapTestContainer openLdapContainer = new OpenLdapTestContainer(); + public void testResolveSubTree() throws Exception { Settings settings = Settings.builder() .put(getFullSettingKey(REALM_ID, SearchGroupsResolverSettings.BASE_DN), "dc=oldap,dc=test,dc=elasticsearch,dc=com") @@ -202,7 +212,7 @@ public void testReadBinaryUserAttribute() throws Exception { @Override protected String ldapUrl() { - return OpenLdapTests.OPEN_LDAP_DNS_URL; + return openLdapContainer.getLdapUrl(); } @Override @@ -217,6 +227,12 @@ protected String bindPassword() { @Override protected String trustPath() { - return "/ca.jks"; + return "/openldap/certs/ca.jks"; + } + + @Override + protected void doSetupLdapConnection() throws Exception { + Path trustPath = openLdapContainer.getJavaKeyStorePath(); + this.ldapConnection = LdapTestUtils.openConnection(ldapUrl(), bindDN(), bindPassword(), trustPath); } } diff --git a/x-pack/qa/password-protected-keystore/src/javaRestTest/java/org/elasticsearch/password_protected_keystore/ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT.java b/x-pack/qa/password-protected-keystore/src/javaRestTest/java/org/elasticsearch/password_protected_keystore/ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT.java index 49950b553bb20..0625ec166e32c 100644 --- a/x-pack/qa/password-protected-keystore/src/javaRestTest/java/org/elasticsearch/password_protected_keystore/ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT.java +++ b/x-pack/qa/password-protected-keystore/src/javaRestTest/java/org/elasticsearch/password_protected_keystore/ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT.java @@ -37,12 +37,12 @@ public class ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT extends ESR .nodes(NUM_NODES) .keystorePassword(KEYSTORE_PASSWORD) .name("javaRestTest") + .keystore(nodeSpec -> Map.of("xpack.security.transport.ssl.secure_key_passphrase", "transport-password")) .setting("xpack.security.enabled", "true") .setting("xpack.security.authc.anonymous.roles", "anonymous") .setting("xpack.security.transport.ssl.enabled", "true") .setting("xpack.security.transport.ssl.certificate", "transport.crt") .setting("xpack.security.transport.ssl.key", "transport.key") - .setting("xpack.security.transport.ssl.key_passphrase", "transport-password") .setting("xpack.security.transport.ssl.certificate_authorities", "ca.crt") .rolesFile(Resource.fromClasspath("roles.yml")) .configFile("transport.key", Resource.fromClasspath("ssl/transport.key")) diff --git a/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java b/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java index 703b9e608db17..27250dd4e3367 100644 --- a/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java @@ -206,7 +206,6 @@ public void testAutoFollowing() throws Exception { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103094") public void testCannotFollowLeaderInUpgradedCluster() throws Exception { if (upgradeState != UpgradeState.ALL) { return; diff --git a/x-pack/qa/saml-idp-tests/build.gradle b/x-pack/qa/saml-idp-tests/build.gradle index 6027a421b62f2..6a7d60f88a1d7 100644 --- a/x-pack/qa/saml-idp-tests/build.gradle +++ b/x-pack/qa/saml-idp-tests/build.gradle @@ -1,46 +1,13 @@ -import org.elasticsearch.gradle.LazyPropertyMap - -Project idpFixtureProject = project(':x-pack:test:idp-fixture') - apply plugin: 'elasticsearch.internal-java-rest-test' -apply plugin: 'elasticsearch.test.fixtures' dependencies { javaRestTestImplementation testArtifact(project(xpackModule('core'))) javaRestTestImplementation "com.google.jimfs:jimfs:${versions.jimfs}" javaRestTestImplementation "com.google.guava:guava:${versions.jimfs_guava}" + javaRestTestImplementation project(":x-pack:test:idp-fixture") } -testFixtures.useFixture ":x-pack:test:idp-fixture" - -String outputDir = "${project.buildDir}/generated-resources/${project.name}" -tasks.register("copyIdpFiles", Sync) { - dependsOn idpFixtureProject.postProcessFixture - // Don't attempt to get ephemeral ports when Docker is not available - onlyIf(idpFixtureProject.postProcessFixture.path + " not skipped") { - idpFixtureProject.postProcessFixture.state.skipped == false - } - from idpFixtureProject.files('idp/shibboleth-idp/credentials/idp-browser.pem', 'idp/shibboleth-idp/metadata/idp-metadata.xml', - 'idp/shibboleth-idp/credentials/sp-signing.key', 'idp/shibboleth-idp/credentials/sp-signing.crt'); - into outputDir - def expandProps = new LazyPropertyMap<>("lazy port config") - expandProps.put("port", () -> idpFixtureProject.postProcessFixture.ext."test.fixtures.shibboleth-idp.tcp.4443") - inputs.properties(expandProps) - filesMatching("idp-metadata.xml") { - expand(expandProps) - } -} - -normalization { - runtimeClasspath { - ignore 'idp-metadata.xml' - } -} tasks.named("javaRestTest").configure { usesDefaultDistribution() - classpath += files(tasks.named("copyIdpFiles")) - onlyIf(idpFixtureProject.postProcessFixture.path + " not skipped") { - idpFixtureProject.postProcessFixture.state.skipped == false - } } diff --git a/x-pack/qa/saml-idp-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticationIT.java b/x-pack/qa/saml-idp-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticationIT.java index 625c0ffae167e..c8b3b3fc3aed2 100644 --- a/x-pack/qa/saml-idp-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticationIT.java +++ b/x-pack/qa/saml-idp-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticationIT.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.security.authc.saml; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; @@ -42,6 +44,9 @@ import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.test.fixtures.idp.IdpTestContainer; +import org.elasticsearch.test.fixtures.idp.OpenLdapTestContainer; +import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -52,6 +57,10 @@ import org.hamcrest.Matchers; import org.junit.Before; import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; +import org.testcontainers.containers.Network; +import org.testcontainers.shaded.org.apache.commons.io.IOUtils; import java.io.IOException; import java.io.InputStream; @@ -82,13 +91,17 @@ /** * An integration test for validating SAML authentication against a real Identity Provider (Shibboleth) */ +@ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) public class SamlAuthenticationIT extends ESRestTestCase { private static final String SAML_RESPONSE_FIELD = "SAMLResponse"; private static final String KIBANA_PASSWORD = "K1b@na K1b@na K1b@na"; - @ClassRule - public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + private static Network network = Network.newNetwork(); + private static OpenLdapTestContainer openLdapTestContainer = new OpenLdapTestContainer(network); + private static IdpTestContainer idpFixture = new IdpTestContainer(network); + + private static ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) .setting("xpack.license.self_generated.type", "trial") .setting("xpack.security.enabled", "true") @@ -131,12 +144,25 @@ public class SamlAuthenticationIT extends ESRestTestCase { .setting("xpack.security.authc.realms.native.native.order", "4") .setting("xpack.ml.enabled", "false") .setting("logger.org.elasticsearch.xpack.security", "TRACE") - .configFile("sp-signing.key", Resource.fromClasspath("sp-signing.key")) - .configFile("idp-metadata.xml", Resource.fromClasspath("idp-metadata.xml")) - .configFile("sp-signing.crt", Resource.fromClasspath("sp-signing.crt")) + .configFile("sp-signing.key", Resource.fromClasspath("/idp/shibboleth-idp/credentials/sp-signing.key")) + .configFile("idp-metadata.xml", Resource.fromString(SamlAuthenticationIT::calculateIdpMetaData)) + .configFile("sp-signing.crt", Resource.fromClasspath("/idp/shibboleth-idp/credentials/sp-signing.crt")) .user("test_admin", "x-pack-test-password") .build(); + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(network).around(openLdapTestContainer).around(idpFixture).around(cluster); + + private static String calculateIdpMetaData() { + Resource resource = Resource.fromClasspath("/idp/shibboleth-idp/metadata/idp-metadata.xml"); + try (InputStream stream = resource.asStream()) { + String metadata = IOUtils.toString(stream, "UTF-8"); + return metadata.replace("${port}", String.valueOf(idpFixture.getDefaultPort())); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + @Override protected String getTestRestCluster() { return cluster.getHttpAddresses(); @@ -526,7 +552,7 @@ private CloseableHttpClient getHttpClient() throws Exception { } private SSLContext getClientSslContext() throws Exception { - final Path pem = getDataPath("/idp-browser.pem"); + final Path pem = idpFixture.getBrowserPem(); final X509ExtendedTrustManager trustManager = CertParsingUtils.getTrustManagerFromPEM(List.of(pem)); SSLContext context = SSLContext.getInstance("TLS"); context.init(new KeyManager[0], new TrustManager[] { trustManager }, new SecureRandom()); diff --git a/x-pack/test/idp-fixture/README.txt b/x-pack/test/idp-fixture/README.txt index 8e42bb142e4ee..c05f53772ed65 100644 --- a/x-pack/test/idp-fixture/README.txt +++ b/x-pack/test/idp-fixture/README.txt @@ -1 +1 @@ -Provisions OpenLDAP + shibboleth IDP 3.4.2 using docker compose +Provisions OpenLDAP + shibboleth IDP 3.4.2 testcontainer fixtures diff --git a/x-pack/test/idp-fixture/build.gradle b/x-pack/test/idp-fixture/build.gradle index 0f5363a278f60..407fb520fcae1 100644 --- a/x-pack/test/idp-fixture/build.gradle +++ b/x-pack/test/idp-fixture/build.gradle @@ -1,40 +1,27 @@ -import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.Architecture -import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER; +apply plugin: 'elasticsearch.java' +apply plugin: 'elasticsearch.cache-test-fixtures' -apply plugin: 'elasticsearch.test.fixtures' - -dockerCompose { - composeAdditionalArgs = ['--compatibility'] +configurations.all { + transitive = false } -tasks.named("preProcessFixture").configure { - file("${testFixturesDir}/shared/oidc").mkdirs() -} +dependencies { + testImplementation project(':test:framework') -tasks.register("copyFiles", Sync) { - from file("oidc/override.properties.template") - into "${buildDir}/config" - doLast { - file("${buildDir}/config").setReadable(true, false) - file("${buildDir}/config/override.properties.template").setReadable(true, false) - } -} + api project(':test:fixtures:testcontainer-utils') + api "junit:junit:${versions.junit}" + api "org.testcontainers:testcontainers:${versions.testcontainer}" + implementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" + implementation "org.slf4j:slf4j-api:${versions.slf4j}" + implementation "com.github.docker-java:docker-java-api:${versions.dockerJava}" -tasks.named("postProcessFixture").configure { - dependsOn "copyFiles" - inputs.dir("${testFixturesDir}/shared/oidc") - File confTemplate = file("${buildDir}/config/override.properties.template") - File confFile = file("${testFixturesDir}/shared/oidc/override.properties") - outputs.file(confFile) - doLast { - assert confTemplate.exists() - String confContents = confTemplate.text - .replace("\${MAPPED_PORT}", "${ext."test.fixtures.oidc-provider.tcp.8080"}") - confFile.text = confContents - } -} + runtimeOnly "com.github.docker-java:docker-java-transport-zerodep:${versions.dockerJava}" + runtimeOnly "com.github.docker-java:docker-java-transport:${versions.dockerJava}" + runtimeOnly "com.github.docker-java:docker-java-core:${versions.dockerJava}" + runtimeOnly "org.apache.commons:commons-compress:${versions.commonsCompress}" + runtimeOnly "org.rnorth.duct-tape:duct-tape:${versions.ductTape}" -tasks.named('composePull').configure { - enabled = false // this task fails due to docker-compose oddities + // ensure we have proper logging during when used in tests + runtimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" + runtimeOnly "org.hamcrest:hamcrest:${versions.hamcrest}" } diff --git a/x-pack/test/idp-fixture/docker-compose.yml b/x-pack/test/idp-fixture/docker-compose.yml deleted file mode 100644 index e431fa4ede611..0000000000000 --- a/x-pack/test/idp-fixture/docker-compose.yml +++ /dev/null @@ -1,67 +0,0 @@ -version: "3.7" -services: - openldap: - command: --copy-service --loglevel debug - image: "osixia/openldap:1.4.0" - ports: - - "389" - - "636" - environment: - LDAP_ADMIN_PASSWORD: "NickFuryHeartsES" - LDAP_DOMAIN: "oldap.test.elasticsearch.com" - LDAP_BASE_DN: "DC=oldap,DC=test,DC=elasticsearch,DC=com" - LDAP_TLS: "true" - LDAP_TLS_CRT_FILENAME: "ldap_server.pem" - LDAP_TLS_CA_CRT_FILENAME: "ca_server.pem" - LDAP_TLS_KEY_FILENAME: "ldap_server.key" - LDAP_TLS_VERIFY_CLIENT: "never" - LDAP_TLS_CIPHER_SUITE: "NORMAL" - LDAP_LOG_LEVEL: 256 - volumes: - - ./openldap/ldif/users.ldif:/container/service/slapd/assets/config/bootstrap/ldif/custom/20-bootstrap-users.ldif - - ./openldap/ldif/config.ldif:/container/service/slapd/assets/config/bootstrap/ldif/custom/10-bootstrap-config.ldif - - ./openldap/certs:/container/service/slapd/assets/certs - - shibboleth-idp: - build: - context: . - dockerfile: ./idp/Dockerfile - depends_on: - - openldap - environment: - - JETTY_MAX_HEAP=64m - ports: - - 4443 - expose: - - 4443 - links: - - openldap:openldap - restart: always #ensure ephemeral port mappings are properly updated - healthcheck: - test: curl -f -s --http0.9 http://localhost:4443 --connect-timeout 10 --max-time 10 --output - > /dev/null - interval: 5s - timeout: 20s - retries: 60 - start_period: 10s - - oidc-provider: - build: - context: . - dockerfile: ./oidc/Dockerfile - depends_on: - - http-proxy - ports: - - "8080" - expose: - - "8080" - volumes: - - ./testfixtures_shared/shared/oidc/:/config/c2id/ - - http-proxy: - image: "nginx:latest" - volumes: - - ./oidc/nginx.conf:/etc/nginx/nginx.conf - ports: - - "8888" - expose: - - "8888" diff --git a/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/HttpProxyTestContainer.java b/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/HttpProxyTestContainer.java new file mode 100644 index 0000000000000..e517c2a9fe2c3 --- /dev/null +++ b/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/HttpProxyTestContainer.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.test.fixtures.idp; + +import org.elasticsearch.test.fixtures.testcontainers.DockerEnvironmentAwareTestContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.images.builder.ImageFromDockerfile; + +public final class HttpProxyTestContainer extends DockerEnvironmentAwareTestContainer { + + public static final String DOCKER_BASE_IMAGE = "nginx:latest"; + private static final Integer PORT = 8888; + + /** + * for packer caching only + * */ + public HttpProxyTestContainer() { + this(Network.newNetwork()); + } + + public HttpProxyTestContainer(Network network) { + super( + new ImageFromDockerfile("es-http-proxy-fixture").withDockerfileFromBuilder( + builder -> builder.from(DOCKER_BASE_IMAGE).copy("oidc/nginx.conf", "/etc/nginx/nginx.conf").build() + ).withFileFromClasspath("oidc/nginx.conf", "/oidc/nginx.conf") + ); + addExposedPort(PORT); + withNetwork(network); + + } + + public Integer getProxyPort() { + return getMappedPort(PORT); + } +} diff --git a/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/IdpTestContainer.java b/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/IdpTestContainer.java new file mode 100644 index 0000000000000..ed19dc997fd8e --- /dev/null +++ b/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/IdpTestContainer.java @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.test.fixtures.idp; + +import org.elasticsearch.test.fixtures.testcontainers.DockerEnvironmentAwareTestContainer; +import org.junit.rules.TemporaryFolder; +import org.testcontainers.containers.Network; +import org.testcontainers.images.builder.ImageFromDockerfile; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.elasticsearch.test.fixtures.ResourceUtils.copyResourceToFile; + +public final class IdpTestContainer extends DockerEnvironmentAwareTestContainer { + + public static final String DOCKER_BASE_IMAGE = "openjdk:11.0.16-jre"; + + private final TemporaryFolder temporaryFolder = new TemporaryFolder(); + private Path certsPath; + + /** + * for packer caching only + * */ + protected IdpTestContainer() { + this(Network.newNetwork()); + } + + public IdpTestContainer(Network network) { + super( + new ImageFromDockerfile("es-idp-testfixture").withDockerfileFromBuilder( + builder -> builder.from(DOCKER_BASE_IMAGE) + .env("jetty_version", "9.3.27.v20190418") + .env("jetty_hash", "7c7c80dd1c9f921771e2b1a05deeeec652d5fcaa") + .env("idp_version", "3.4.3") + .env("idp_hash", "eb86bc7b6366ce2a44f97cae1b014d307b84257e3149469b22b2d091007309db") + .env("dta_hash", "2f547074b06952b94c35631398f36746820a7697") + .env("slf4j_version", "1.7.25") + .env("slf4j_hash", "da76ca59f6a57ee3102f8f9bd9cee742973efa8a") + .env("logback_version", "1.2.3") + .env("logback_classic_hash", "7c4f3c474fb2c041d8028740440937705ebb473a") + .env("logback_core_hash", "864344400c3d4d92dfeb0a305dc87d953677c03c") + .env("logback_access_hash", "e8a841cb796f6423c7afd8738df6e0e4052bf24a") + + .env("JETTY_HOME", "/opt/jetty-home") + .env("JETTY_BASE", "/opt/shib-jetty-base") + .env("PATH", "$PATH:$JAVA_HOME/bin") + .env("JETTY_BROWSER_SSL_KEYSTORE_PASSWORD", "secret") + .env("JETTY_BACKCHANNEL_SSL_KEYSTORE_PASSWORD", "secret") + .env("JETTY_MAX_HEAP", "64m") + // Manually override the jetty keystore otherwise it will attempt to download and fail + .run("mkdir -p /opt/shib-jetty-base/modules") + .copy("idp/jetty-custom/ssl.mod", "/opt/shib-jetty-base/modules/ssl.mod") + .copy("idp/jetty-custom/keystore", "/opt/shib-jetty-base/etc/keystore") + // Download Jetty, verify the hash, and install, initialize a new base + .run( + "wget -q https://repo.maven.apache.org/maven2/org/eclipse/jetty/jetty-distribution/$jetty_version/jetty-distribution-$jetty_version.tar.gz" + + " && echo \"$jetty_hash jetty-distribution-$jetty_version.tar.gz\" | sha1sum -c -" + + " && tar -zxvf jetty-distribution-$jetty_version.tar.gz -C /opt" + + " && ln -s /opt/jetty-distribution-$jetty_version/ /opt/jetty-home" + ) + // Config Jetty + .run( + "mkdir -p /opt/shib-jetty-base/modules /opt/shib-jetty-base/lib/ext /opt/shib-jetty-base/lib/logging /opt/shib-jetty-base/resources" + + " && cd /opt/shib-jetty-base" + + " && touch start.ini" + + " && java -jar ../jetty-home/start.jar --add-to-startd=http,https,deploy,ext,annotations,jstl,rewrite" + ) + // Download Shibboleth IdP, verify the hash, and install + .run( + "wget -q https://shibboleth.net/downloads/identity-provider/archive/$idp_version/shibboleth-identity-provider-$idp_version.tar.gz" + + " && echo \"$idp_hash shibboleth-identity-provider-$idp_version.tar.gz\" | sha256sum -c -" + + " && tar -zxvf shibboleth-identity-provider-$idp_version.tar.gz -C /opt" + + " && ln -s /opt/shibboleth-identity-provider-$idp_version/ /opt/shibboleth-idp" + ) + // Download the library to allow SOAP Endpoints, verify the hash, and place + .run( + "wget -q https://build.shibboleth.net/nexus/content/repositories/releases/net/shibboleth/utilities/jetty9/jetty9-dta-ssl/1.0.0/jetty9-dta-ssl-1.0.0.jar" + + " && echo \"$dta_hash jetty9-dta-ssl-1.0.0.jar\" | sha1sum -c -" + + " && mv jetty9-dta-ssl-1.0.0.jar /opt/shib-jetty-base/lib/ext/" + ) + // Download the slf4j library for Jetty logging, verify the hash, and place + .run( + "wget -q https://repo.maven.apache.org/maven2/org/slf4j/slf4j-api/$slf4j_version/slf4j-api-$slf4j_version.jar" + + " && echo \"$slf4j_hash slf4j-api-$slf4j_version.jar\" | sha1sum -c -" + + " && mv slf4j-api-$slf4j_version.jar /opt/shib-jetty-base/lib/logging/" + ) + // Download the logback_classic library for Jetty logging, verify the hash, and place + .run( + "wget -q https://repo.maven.apache.org/maven2/ch/qos/logback/logback-classic/$logback_version/logback-classic-$logback_version.jar" + + " && echo \"$logback_classic_hash logback-classic-$logback_version.jar\" | sha1sum -c -" + + " && mv logback-classic-$logback_version.jar /opt/shib-jetty-base/lib/logging/" + ) + // Download the logback-core library for Jetty logging, verify the hash, and place + .run( + "wget -q https://repo.maven.apache.org/maven2/ch/qos/logback/logback-core/$logback_version/logback-core-$logback_version.jar" + + " && echo \"$logback_core_hash logback-core-$logback_version.jar\" | sha1sum -c -" + + " && mv logback-core-$logback_version.jar /opt/shib-jetty-base/lib/logging/" + ) + // Download the logback-access library for Jetty logging, verify the hash, and place + .run( + "wget -q https://repo.maven.apache.org/maven2/ch/qos/logback/logback-access/$logback_version/logback-access-$logback_version.jar" + + " && echo \"$logback_access_hash logback-access-$logback_version.jar\" | sha1sum -c -" + + " && mv logback-access-$logback_version.jar /opt/shib-jetty-base/lib/logging/" + ) + // ## Copy local files + .copy("idp/shib-jetty-base/", "/opt/shib-jetty-base/") + .copy("idp/shibboleth-idp/", "/opt/shibboleth-idp/") + .copy("idp/bin/", "/usr/local/bin/") + // Setting owner ownership and permissions + .run( + "useradd jetty -U -s /bin/false" + + " && chown -R root:jetty /opt" + + " && chmod -R 640 /opt" + + " && chown -R root:jetty /opt/shib-jetty-base" + + " && chmod -R 640 /opt/shib-jetty-base" + + " && chmod -R 750 /opt/shibboleth-idp/bin" + ) + .run("chmod 750 /usr/local/bin/run-jetty.sh /usr/local/bin/init-idp.sh") + .run("chmod +x /opt/jetty-home/bin/jetty.sh") + // Opening 4443 (browser TLS), 8443 (mutual auth TLS) + .cmd("run-jetty.sh") + // .expose(4443) + .build() + + ) + .withFileFromClasspath("idp/jetty-custom/ssl.mod", "/idp/jetty-custom/ssl.mod") + .withFileFromClasspath("idp/jetty-custom/keystore", "/idp/jetty-custom/keystore") + .withFileFromClasspath("idp/shib-jetty-base/", "/idp/shib-jetty-base/") + .withFileFromClasspath("idp/shibboleth-idp/", "/idp/shibboleth-idp/") + .withFileFromClasspath("idp/bin/", "/idp/bin/") + + ); + withNetworkAliases("idp"); + withNetwork(network); + addExposedPorts(4443, 8443); + } + + @Override + public void stop() { + super.stop(); + temporaryFolder.delete(); + } + + public Path getBrowserPem() { + try { + temporaryFolder.create(); + certsPath = temporaryFolder.newFolder("certs").toPath(); + return copyResourceToFile(getClass(), certsPath, "idp/shibboleth-idp/credentials/idp-browser.pem"); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public Integer getDefaultPort() { + return getMappedPort(4443); + } +} diff --git a/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/OidcProviderTestContainer.java b/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/OidcProviderTestContainer.java new file mode 100644 index 0000000000000..89090fa6e11bc --- /dev/null +++ b/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/OidcProviderTestContainer.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.test.fixtures.idp; + +import org.elasticsearch.test.fixtures.testcontainers.DockerEnvironmentAwareTestContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.images.builder.ImageFromDockerfile; +import org.testcontainers.images.builder.Transferable; + +public final class OidcProviderTestContainer extends DockerEnvironmentAwareTestContainer { + + private static final int PORT = 8080; + + /** + * for packer caching only + * */ + protected OidcProviderTestContainer() { + this(Network.newNetwork()); + } + + public OidcProviderTestContainer(Network network) { + super( + new ImageFromDockerfile("es-oidc-provider-fixture").withFileFromClasspath("oidc/setup.sh", "/oidc/setup.sh") + // we cannot make use of docker file builder + // as it does not support multi-stage builds + .withFileFromClasspath("Dockerfile", "oidc/Dockerfile") + ); + withNetworkAliases("oidc-provider"); + withNetwork(network); + addExposedPort(PORT); + } + + @Override + public void start() { + super.start(); + copyFileToContainer( + Transferable.of( + "op.issuer=http://127.0.0.1:" + + getMappedPort(PORT) + + "/c2id\n" + + "op.authz.endpoint=http://127.0.0.1:" + + getMappedPort(PORT) + + "/c2id-login/\n" + + "op.reg.apiAccessTokenSHA256=d1c4fa70d9ee708d13cfa01daa0e060a05a2075a53c5cc1ad79e460e96ab5363\n" + + "jose.jwkSer=RnVsbCBrZXk6CnsKICAia2V5cyI6IFsKICAgIHsKICAgICAgInAiOiAiLXhhN2d2aW5tY3N3QXU3Vm1mV2loZ2o3U3gzUzhmd2dFSTdMZEVveW5FU1RzcElaeUY5aHc0NVhQZmI5VHlpbzZsOHZTS0F5RmU4T2lOalpkNE1Ra0ttYlJzTmxxR1Y5VlBoWF84UG1JSm5mcGVhb3E5YnZfU0k1blZHUl9zYUUzZE9sTEE2VWpaS0lsRVBNb0ZuRlZCMUFaUU9qQlhRRzZPTDg2eDZ2NHMwIiwKICAgICAgImt0eSI6ICJSU0EiLAogICAgICAicSI6ICJ2Q3pDQUlpdHV0MGx1V0djQloyLUFabURLc1RxNkkxcUp0RmlEYkIyZFBNQVlBNldOWTdaWEZoVWxsSjJrT2ZELWdlYjlkYkN2ODBxNEwyajVZSjZoOTBUc1NRWWVHRlljN1lZMGdCMU5VR3l5cXctb29QN0EtYlJmMGI3b3I4ajZJb0hzQTZKa2JranN6c3otbkJ2U2RmUURlZkRNSVc3Ni1ZWjN0c2hsY2MiLAogICAgICAiZCI6ICJtbFBOcm1zVVM5UmJtX1I5SElyeHdmeFYzZnJ2QzlaQktFZzRzc1ZZaThfY09lSjV2U1hyQV9laEtwa2g4QVhYaUdWUGpQbVlyd29xQzFVUksxUkZmLVg0dG10emV2OUVHaU12Z0JCaEF5RkdTSUd0VUNla2x4Q2dhb3BpMXdZSU1Bd0M0STZwMUtaZURxTVNCWVZGeHA5ZWlJZ2pwb05JbV9lR3hXUUs5VHNnYmk5T3lyc1VqaE9KLVczN2JVMEJWUU56UXpxODhCcGxmNzM3VmV1dy1FeDZaMk1iWXR3SWdfZ0JVb0JEZ0NrZkhoOVE4MElYcEZRV0x1RzgwenFrdkVwTHZ0RWxLbDRvQ3BHVnBjcmFUOFNsOGpYc3FDT1k0dnVRT19LRVUzS2VPNUNJbHd4eEhJYXZjQTE5cHFpSWJ5cm1LbThxS0ZEWHluUFJMSGFNZ1EiLAogICAgICAiZSI6ICJBUUFCIiwKICAgICAgImtpZCI6ICJyc2EzODRfMjA0OCIsCiAgICAgICJxaSI6ICJzMldTamVrVDl3S2JPbk9neGNoaDJPY3VubzE2Y20wS281Z3hoUWJTdVMyMldfUjJBR2ZVdkRieGF0cTRLakQ3THo3X1k2TjdTUkwzUVpudVhoZ1djeXgyNGhrUGppQUZLNmlkYVZKQzJqQmgycEZTUDVTNXZxZ0lsME12eWY4NjlwdkN4S0NzaGRKMGdlRWhveE93VkRPYXJqdTl2Zm9IQV90LWJoRlZrUnciLAogICAgICAiZHAiOiAiQlJhQTFqYVRydG9mTHZBSUJBYW1OSEVhSm51RU9zTVJJMFRCZXFuR1BNUm0tY2RjSG1OUVo5WUtqb2JpdXlmbnhGZ0piVDlSeElBRG0ySkpoZEp5RTN4Y1dTSzhmSjBSM1Jick1aT1dwako0QmJTVzFtU1VtRnlKTGxib3puRFhZR2RaZ1hzS0o1UkFrRUNQZFBCY3YwZVlkbk9NYWhfZndfaFZoNjRuZ2tFIiwKICAgICAgImFsZyI6ICJSU0EzODQiLAogICAgICAiZHEiOiAiUFJoVERKVlR3cDNXaDZfWFZrTjIwMUlpTWhxcElrUDN1UTYyUlRlTDNrQ2ZXSkNqMkZPLTRxcVRIQk0tQjZJWUVPLXpoVWZyQnhiMzJ1djNjS2JDWGFZN3BJSFJxQlFEQWQ2WGhHYzlwc0xqNThXd3VGY2RncERJYUFpRjNyc3NUMjJ4UFVvYkJFTVdBalV3bFJrNEtNTjItMnpLQk5FR3lIcDIzOUpKdnpVIiwKICAgICAgIm4iOiAidUpDWDVDbEZpM0JnTXBvOWhRSVZ2SDh0Vi1jLTVFdG5OeUZxVm91R3NlNWwyUG92MWJGb0tsRllsU25YTzNWUE9KRWR3azNDdl9VT0UtQzlqZERYRHpvS3Z4RURaTVM1TDZWMFpIVEJoNndIOV9iN3JHSlBxLV9RdlNkejczSzZxbHpGaUtQamRvdTF6VlFYTmZfblBZbnRnQkdNRUtBc1pRNGp0cWJCdE5lV0h0MF9UM001cEktTV9KNGVlRWpCTW95TkZuU2ExTEZDVmZRNl9YVnpjelp1TlRGMlh6UmdRWkFmcmJGRXZ6eXR1TzVMZTNTTXFrUUFJeDhFQmkwYXVlRUNqNEQ4cDNVNXFVRG92NEF2VnRJbUZlbFJvb1pBMHJtVW1KRHJ4WExrVkhuVUpzaUF6ZW9TLTNBSnV1bHJkMGpuNjJ5VjZHV2dFWklZMVNlZVd3IgogICAgfQogIF0KfQo\n" + + "op.authz.alwaysPromptForConsent=true\n" + + "op.authz.alwaysPromptForAuth=true" + ), + "config/c2id/override.properties" + ); + } + + public String getC2OPUrl() { + return "http://127.0.0.1:" + getMappedPort(PORT); + } + + public String getC2IssuerUrl() { + return getC2OPUrl() + "/c2id"; + } + +} diff --git a/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/OpenLdapTestContainer.java b/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/OpenLdapTestContainer.java new file mode 100644 index 0000000000000..2f65134f2ec72 --- /dev/null +++ b/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/OpenLdapTestContainer.java @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.test.fixtures.idp; + +import org.elasticsearch.test.fixtures.testcontainers.DockerEnvironmentAwareTestContainer; +import org.junit.rules.TemporaryFolder; +import org.testcontainers.containers.Network; +import org.testcontainers.images.builder.ImageFromDockerfile; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.elasticsearch.test.fixtures.ResourceUtils.copyResourceToFile; + +public final class OpenLdapTestContainer extends DockerEnvironmentAwareTestContainer { + + public static final String DOCKER_BASE_IMAGE = "osixia/openldap:1.4.0"; + + private final TemporaryFolder temporaryFolder = new TemporaryFolder(); + private Path certsPath; + + public OpenLdapTestContainer() { + this(Network.newNetwork()); + } + + public OpenLdapTestContainer(Network network) { + super( + new ImageFromDockerfile("es-openldap-testfixture").withDockerfileFromBuilder( + builder -> builder.from(DOCKER_BASE_IMAGE) + .env("LDAP_ADMIN_PASSWORD", "NickFuryHeartsES") + .env("LDAP_DOMAIN", "oldap.test.elasticsearch.com") + .env("LDAP_BASE_DN", "DC=oldap,DC=test,DC=elasticsearch,DC=com") + .env("LDAP_TLS", "true") + .env("LDAP_TLS_CRT_FILENAME", "ldap_server.pem") + .env("LDAP_TLS_CA_CRT_FILENAME", "ca_server.pem") + .env("LDAP_TLS_KEY_FILENAME", "ldap_server.key") + .env("LDAP_TLS_VERIFY_CLIENT", "never") + .env("LDAP_TLS_CIPHER_SUITE", "NORMAL") + .env("LDAP_LOG_LEVEL", "256") + .copy( + "openldap/ldif/users.ldif", + "/container/service/slapd/assets/config/bootstrap/ldif/custom/20-bootstrap-users.ldif" + ) + .copy( + "openldap/ldif/config.ldif", + "/container/service/slapd/assets/config/bootstrap/ldif/custom/10-bootstrap-config.ldif" + ) + .copy("openldap/certs", "/container/service/slapd/assets/certs") + + .build() + ) + .withFileFromClasspath("openldap/certs", "/openldap/certs/") + .withFileFromClasspath("openldap/ldif/users.ldif", "/openldap/ldif/users.ldif") + .withFileFromClasspath("openldap/ldif/config.ldif", "/openldap/ldif/config.ldif") + ); + // withLogConsumer(new Slf4jLogConsumer(logger())); + withNetworkAliases("openldap"); + withNetwork(network); + withExposedPorts(389, 636); + } + + public String getLdapUrl() { + return "ldaps://localhost:" + getMappedPort(636); + } + + @Override + public void start() { + super.start(); + setupCerts(); + } + + @Override + public void stop() { + super.stop(); + temporaryFolder.delete(); + } + + private void setupCerts() { + try { + temporaryFolder.create(); + certsPath = temporaryFolder.newFolder("certs").toPath(); + copyResourceToFile(getClass(), certsPath, "openldap/certs/ca.jks"); + copyResourceToFile(getClass(), certsPath, "openldap/certs/ca_server.key"); + copyResourceToFile(getClass(), certsPath, "openldap/certs/ca_server.pem"); + copyResourceToFile(getClass(), certsPath, "openldap/certs/dhparam.pem"); + copyResourceToFile(getClass(), certsPath, "openldap/certs/ldap_server.key"); + copyResourceToFile(getClass(), certsPath, "openldap/certs/ldap_server.pem"); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public Path getJavaKeyStorePath() { + return certsPath.resolve("ca.jks"); + } + + public Path getCaCertPath() { + return certsPath.resolve("ca_server.pem"); + } + + public Integer getDefaultPort() { + return getMappedPort(636); + } +} diff --git a/x-pack/test/idp-fixture/idp/Dockerfile b/x-pack/test/idp-fixture/src/main/resources/idp/Dockerfile similarity index 100% rename from x-pack/test/idp-fixture/idp/Dockerfile rename to x-pack/test/idp-fixture/src/main/resources/idp/Dockerfile diff --git a/x-pack/test/idp-fixture/idp/bin/init-idp.sh b/x-pack/test/idp-fixture/src/main/resources/idp/bin/init-idp.sh similarity index 100% rename from x-pack/test/idp-fixture/idp/bin/init-idp.sh rename to x-pack/test/idp-fixture/src/main/resources/idp/bin/init-idp.sh diff --git a/x-pack/test/idp-fixture/idp/bin/run-jetty.sh b/x-pack/test/idp-fixture/src/main/resources/idp/bin/run-jetty.sh similarity index 100% rename from x-pack/test/idp-fixture/idp/bin/run-jetty.sh rename to x-pack/test/idp-fixture/src/main/resources/idp/bin/run-jetty.sh diff --git a/x-pack/test/idp-fixture/idp/jetty-custom/keystore b/x-pack/test/idp-fixture/src/main/resources/idp/jetty-custom/keystore similarity index 100% rename from x-pack/test/idp-fixture/idp/jetty-custom/keystore rename to x-pack/test/idp-fixture/src/main/resources/idp/jetty-custom/keystore diff --git a/x-pack/test/idp-fixture/idp/jetty-custom/ssl.mod b/x-pack/test/idp-fixture/src/main/resources/idp/jetty-custom/ssl.mod similarity index 100% rename from x-pack/test/idp-fixture/idp/jetty-custom/ssl.mod rename to x-pack/test/idp-fixture/src/main/resources/idp/jetty-custom/ssl.mod diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-backchannel.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-backchannel.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-backchannel.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-backchannel.xml diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-logging.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-logging.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-logging.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-logging.xml diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-requestlog.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-requestlog.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-requestlog.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-requestlog.xml diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-rewrite.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-rewrite.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-rewrite.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-rewrite.xml diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-ssl-context.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-ssl-context.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-ssl-context.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-ssl-context.xml diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/modules/backchannel.mod b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/modules/backchannel.mod similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/modules/backchannel.mod rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/modules/backchannel.mod diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/resources/logback-access.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/resources/logback-access.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/resources/logback-access.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/resources/logback-access.xml diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/resources/logback.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/resources/logback.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/resources/logback.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/resources/logback.xml diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/start.d/backchannel.ini b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/start.d/backchannel.ini similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/start.d/backchannel.ini rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/start.d/backchannel.ini diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/start.d/ssl.ini b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/start.d/ssl.ini similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/start.d/ssl.ini rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/start.d/ssl.ini diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/start.ini b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/start.ini similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/start.ini rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/start.ini diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/webapps/idp.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/webapps/idp.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/webapps/idp.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/webapps/idp.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/access-control.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/access-control.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/access-control.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/access-control.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/admin/general-admin.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/admin/general-admin.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/admin/general-admin.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/admin/general-admin.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/admin/metrics.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/admin/metrics.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/admin/metrics.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/admin/metrics.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/attribute-filter.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/attribute-filter.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/attribute-filter.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/attribute-filter.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/attribute-resolver.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/attribute-resolver.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/attribute-resolver.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/attribute-resolver.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/audit.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/audit.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/audit.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/audit.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/authn-comparison.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/authn-comparison.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/authn-comparison.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/authn-comparison.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/authn-events-flow.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/authn-events-flow.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/authn-events-flow.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/authn-events-flow.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/duo-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/duo-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/duo-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/duo-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/duo.properties b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/duo.properties similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/duo.properties rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/duo.properties diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/external-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/external-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/external-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/external-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/function-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/function-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/function-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/function-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/general-authn.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/general-authn.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/general-authn.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/general-authn.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/ipaddress-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/ipaddress-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/ipaddress-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/ipaddress-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/jaas-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/jaas-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/jaas-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/jaas-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/jaas.config b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/jaas.config similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/jaas.config rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/jaas.config diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/krb5-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/krb5-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/krb5-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/krb5-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/ldap-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/ldap-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/ldap-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/ldap-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/mfa-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/mfa-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/mfa-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/mfa-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/password-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/password-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/password-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/password-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/remoteuser-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/remoteuser-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/remoteuser-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/remoteuser-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/remoteuser-internal-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/remoteuser-internal-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/remoteuser-internal-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/remoteuser-internal-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/spnego-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/spnego-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/spnego-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/spnego-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/x509-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/x509-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/x509-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/x509-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/x509-internal-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/x509-internal-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/x509-internal-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/x509-internal-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/attribute-sourced-subject-c14n-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/attribute-sourced-subject-c14n-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/attribute-sourced-subject-c14n-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/attribute-sourced-subject-c14n-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/simple-subject-c14n-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/simple-subject-c14n-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/simple-subject-c14n-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/simple-subject-c14n-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/subject-c14n-events-flow.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/subject-c14n-events-flow.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/subject-c14n-events-flow.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/subject-c14n-events-flow.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/subject-c14n.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/subject-c14n.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/subject-c14n.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/subject-c14n.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/x500-subject-c14n-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/x500-subject-c14n-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/x500-subject-c14n-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/x500-subject-c14n-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/cas-protocol.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/cas-protocol.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/cas-protocol.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/cas-protocol.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/credentials.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/credentials.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/credentials.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/credentials.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/errors.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/errors.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/errors.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/errors.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/global.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/global.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/global.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/global.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/idp.properties b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/idp.properties similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/idp.properties rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/idp.properties diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/consent-intercept-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/consent-intercept-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/consent-intercept-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/consent-intercept-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/context-check-intercept-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/context-check-intercept-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/context-check-intercept-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/context-check-intercept-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/expiring-password-intercept-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/expiring-password-intercept-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/expiring-password-intercept-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/expiring-password-intercept-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/impersonate-intercept-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/impersonate-intercept-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/impersonate-intercept-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/impersonate-intercept-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/intercept-events-flow.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/intercept-events-flow.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/intercept-events-flow.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/intercept-events-flow.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/profile-intercept.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/profile-intercept.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/profile-intercept.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/profile-intercept.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/ldap.properties b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/ldap.properties similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/ldap.properties rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/ldap.properties diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/logback.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/logback.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/logback.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/logback.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/metadata-providers.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/metadata-providers.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/metadata-providers.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/metadata-providers.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/relying-party.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/relying-party.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/relying-party.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/relying-party.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/saml-nameid.properties b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/saml-nameid.properties similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/saml-nameid.properties rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/saml-nameid.properties diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/saml-nameid.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/saml-nameid.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/saml-nameid.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/saml-nameid.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/services.properties b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/services.properties similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/services.properties rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/services.properties diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/services.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/services.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/services.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/services.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/session-manager.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/session-manager.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/session-manager.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/session-manager.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/README b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/README similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/README rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/README diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/ca_server.pem b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/ca_server.pem similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/ca_server.pem rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/ca_server.pem diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-backchannel.crt b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-backchannel.crt similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-backchannel.crt rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-backchannel.crt diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-backchannel.p12 b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-backchannel.p12 similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-backchannel.p12 rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-backchannel.p12 diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-browser.key b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-browser.key similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-browser.key rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-browser.key diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-browser.p12 b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-browser.p12 similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-browser.p12 rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-browser.p12 diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-browser.pem b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-browser.pem similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-browser.pem rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-browser.pem diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-encryption.crt b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-encryption.crt similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-encryption.crt rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-encryption.crt diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-encryption.key b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-encryption.key similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-encryption.key rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-encryption.key diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-signing.crt b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-signing.crt similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-signing.crt rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-signing.crt diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-signing.key b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-signing.key similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-signing.key rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-signing.key diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/sealer.jks b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/sealer.jks similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/sealer.jks rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/sealer.jks diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/sealer.kver b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/sealer.kver similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/sealer.kver rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/sealer.kver diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/sp-signing.crt b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/sp-signing.crt similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/sp-signing.crt rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/sp-signing.crt diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/sp-signing.key b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/sp-signing.key similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/sp-signing.key rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/sp-signing.key diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/README.asciidoc b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/README.asciidoc similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/README.asciidoc rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/README.asciidoc diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/idp-docs-metadata.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/idp-docs-metadata.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/idp-docs-metadata.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/idp-docs-metadata.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/idp-metadata.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/idp-metadata.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/idp-metadata.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/idp-metadata.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/sp-metadata.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/sp-metadata.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/sp-metadata.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/sp-metadata.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/sp-metadata2.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/sp-metadata2.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/sp-metadata2.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/sp-metadata2.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/sp-metadata3.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/sp-metadata3.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/sp-metadata3.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/sp-metadata3.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/admin/unlock-keys.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/admin/unlock-keys.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/admin/unlock-keys.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/admin/unlock-keys.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/client-storage/client-storage-read.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/client-storage/client-storage-read.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/client-storage/client-storage-read.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/client-storage/client-storage-read.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/client-storage/client-storage-write.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/client-storage/client-storage-write.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/client-storage/client-storage-write.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/client-storage/client-storage-write.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/duo.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/duo.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/duo.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/duo.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/error.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/error.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/error.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/error.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/intercept/attribute-release.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/intercept/attribute-release.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/intercept/attribute-release.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/intercept/attribute-release.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/intercept/expiring-password.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/intercept/expiring-password.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/intercept/expiring-password.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/intercept/expiring-password.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/intercept/impersonate.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/intercept/impersonate.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/intercept/impersonate.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/intercept/impersonate.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/intercept/terms-of-use.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/intercept/terms-of-use.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/intercept/terms-of-use.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/intercept/terms-of-use.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/login-error.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/login-error.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/login-error.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/login-error.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/login.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/login.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/login.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/login.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/logout-complete.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/logout-complete.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/logout-complete.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/logout-complete.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/logout-propagate.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/logout-propagate.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/logout-propagate.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/logout-propagate.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/logout.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/logout.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/logout.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/logout.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/spnego-unavailable.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/spnego-unavailable.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/spnego-unavailable.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/spnego-unavailable.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/user-prefs.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/user-prefs.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/user-prefs.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/user-prefs.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/css/consent.css b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/css/consent.css similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/css/consent.css rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/css/consent.css diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/css/logout.css b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/css/logout.css similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/css/logout.css rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/css/logout.css diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/css/main.css b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/css/main.css similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/css/main.css rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/css/main.css diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/images/dummylogo-mobile.png b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/images/dummylogo-mobile.png similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/images/dummylogo-mobile.png rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/images/dummylogo-mobile.png diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/images/dummylogo.png b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/images/dummylogo.png similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/images/dummylogo.png rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/images/dummylogo.png diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/images/failure-32x32.png b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/images/failure-32x32.png similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/images/failure-32x32.png rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/images/failure-32x32.png diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/images/success-32x32.png b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/images/success-32x32.png similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/images/success-32x32.png rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/images/success-32x32.png diff --git a/x-pack/test/idp-fixture/oidc/Dockerfile b/x-pack/test/idp-fixture/src/main/resources/oidc/Dockerfile similarity index 100% rename from x-pack/test/idp-fixture/oidc/Dockerfile rename to x-pack/test/idp-fixture/src/main/resources/oidc/Dockerfile diff --git a/x-pack/test/idp-fixture/oidc/nginx.conf b/x-pack/test/idp-fixture/src/main/resources/oidc/nginx.conf similarity index 100% rename from x-pack/test/idp-fixture/oidc/nginx.conf rename to x-pack/test/idp-fixture/src/main/resources/oidc/nginx.conf diff --git a/x-pack/test/idp-fixture/oidc/op-jwks.json b/x-pack/test/idp-fixture/src/main/resources/oidc/op-jwks.json similarity index 100% rename from x-pack/test/idp-fixture/oidc/op-jwks.json rename to x-pack/test/idp-fixture/src/main/resources/oidc/op-jwks.json diff --git a/x-pack/test/idp-fixture/oidc/override.properties.template b/x-pack/test/idp-fixture/src/main/resources/oidc/override.properties.template similarity index 100% rename from x-pack/test/idp-fixture/oidc/override.properties.template rename to x-pack/test/idp-fixture/src/main/resources/oidc/override.properties.template diff --git a/x-pack/test/idp-fixture/oidc/setup.sh b/x-pack/test/idp-fixture/src/main/resources/oidc/setup.sh similarity index 100% rename from x-pack/test/idp-fixture/oidc/setup.sh rename to x-pack/test/idp-fixture/src/main/resources/oidc/setup.sh diff --git a/x-pack/test/idp-fixture/openldap/certs/README b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/README similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/README rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/README diff --git a/x-pack/test/idp-fixture/openldap/certs/ca.jks b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/ca.jks similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/ca.jks rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/ca.jks diff --git a/x-pack/test/idp-fixture/openldap/certs/ca_server.key b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/ca_server.key similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/ca_server.key rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/ca_server.key diff --git a/x-pack/test/idp-fixture/openldap/certs/ca_server.pem b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/ca_server.pem similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/ca_server.pem rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/ca_server.pem diff --git a/x-pack/test/idp-fixture/openldap/certs/dhparam.pem b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/dhparam.pem similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/dhparam.pem rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/dhparam.pem diff --git a/x-pack/test/idp-fixture/openldap/certs/ldap_server.csr b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/ldap_server.csr similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/ldap_server.csr rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/ldap_server.csr diff --git a/x-pack/test/idp-fixture/openldap/certs/ldap_server.key b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/ldap_server.key similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/ldap_server.key rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/ldap_server.key diff --git a/x-pack/test/idp-fixture/openldap/certs/ldap_server.pem b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/ldap_server.pem similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/ldap_server.pem rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/ldap_server.pem diff --git a/x-pack/test/idp-fixture/openldap/certs/templates/ca_server.conf b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/templates/ca_server.conf similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/templates/ca_server.conf rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/templates/ca_server.conf diff --git a/x-pack/test/idp-fixture/openldap/certs/templates/ldap_server.conf b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/templates/ldap_server.conf similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/templates/ldap_server.conf rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/templates/ldap_server.conf diff --git a/x-pack/test/idp-fixture/openldap/ldif/config.ldif b/x-pack/test/idp-fixture/src/main/resources/openldap/ldif/config.ldif similarity index 100% rename from x-pack/test/idp-fixture/openldap/ldif/config.ldif rename to x-pack/test/idp-fixture/src/main/resources/openldap/ldif/config.ldif diff --git a/x-pack/test/idp-fixture/openldap/ldif/users.ldif b/x-pack/test/idp-fixture/src/main/resources/openldap/ldif/users.ldif similarity index 100% rename from x-pack/test/idp-fixture/openldap/ldif/users.ldif rename to x-pack/test/idp-fixture/src/main/resources/openldap/ldif/users.ldif