Merge remote-tracking branch 'origin/main' into lucene_snapshot

.ci/init.gradle

@@ -1,95 +1,3 @@
-import com.bettercloud.vault.VaultConfig
-import com.bettercloud.vault.Vault
-
-initscript {
-  repositories {
-    mavenCentral()
-  }
-  dependencies {
-    classpath 'com.bettercloud:vault-java-driver:4.1.0'
-  }
-}
-
-boolean USE_ARTIFACTORY = false
-
-if (System.getenv('VAULT_ADDR') == null) {
-  // When trying to reproduce errors outside of CI, it can be useful to allow this to just return rather than blowing up
-  if (System.getenv('CI') == null) {
-    return
-  }
-
-  throw new GradleException("You must set the VAULT_ADDR environment variable to use this init script.")
-}
-
-if (System.getenv('VAULT_ROLE_ID') == null && System.getenv('VAULT_SECRET_ID') == null && System.getenv('VAULT_TOKEN') == null) {
-    // When trying to reproduce errors outside of CI, it can be useful to allow this to just return rather than blowing up
-  if (System.getenv('CI') == null) {
-    return
-  }
-
-  throw new GradleException("You must set either the VAULT_ROLE_ID and VAULT_SECRET_ID environment variables, " +
-    "or the VAULT_TOKEN environment variable to use this init script.")
-}
-
-final String vaultPathPrefix = System.getenv('VAULT_ADDR') ==~ /.+vault-ci.+\.dev.*/ ? "secret/ci/elastic-elasticsearch/migrated" : "secret/elasticsearch-ci"
-
-final String vaultToken = System.getenv('VAULT_TOKEN') ?: new Vault(
-  new VaultConfig()
-    .address(System.env.VAULT_ADDR)
-    .engineVersion(1)
-    .build()
-)
-  .withRetries(5, 1000)
-  .auth()
-  .loginByAppRole("approle", System.env.VAULT_ROLE_ID, System.env.VAULT_SECRET_ID)
-  .getAuthClientToken()
-
-final Vault vault = new Vault(
-  new VaultConfig()
-    .address(System.env.VAULT_ADDR)
-    .engineVersion(1)
-    .token(vaultToken)
-    .build()
-)
-  .withRetries(5, 1000)
-
-
-if (USE_ARTIFACTORY) {
-  final Map<String, String> artifactoryCredentials = vault.logical()
-    .read("${vaultPathPrefix}/artifactory.elstc.co")
-    .getData()
-  logger.info("Using elastic artifactory repos")
-  Closure configCache = {
-    return {
-      name "artifactory-gradle-release"
-      url "https://artifactory.elstc.co/artifactory/gradle-release"
-      credentials {
-        username artifactoryCredentials.get("username")
-        password artifactoryCredentials.get("token")
-      }
-    }
-  }
-  settingsEvaluated { settings ->
-    settings.pluginManagement {
-      repositories {
-        maven configCache()
-      }
-    }
-  }
-  projectsLoaded {
-    allprojects {
-      buildscript {
-        repositories {
-          maven configCache()
-        }
-      }
-      repositories {
-        maven configCache()
-      }
-    }
-  }
-}
-
 gradle.settingsEvaluated { settings ->
   settings.pluginManager.withPlugin("com.gradle.develocity") {
     settings.develocity {
@@ -98,14 +6,10 @@ gradle.settingsEvaluated { settings ->
   }
 }
 
-
 final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
 final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false'))
 
 if (buildCacheUrl) {
-  final Map<String, String> buildCacheCredentials = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ? [:] : vault.logical()
-    .read("${vaultPathPrefix}/gradle-build-cache")
-    .getData()
   gradle.settingsEvaluated { settings ->
     settings.buildCache {
       local {
@@ -116,11 +20,10 @@ if (buildCacheUrl) {
         url = buildCacheUrl
         push = buildCachePush
         credentials {
-          username = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ?: buildCacheCredentials.get("username")
-          password = System.getenv("GRADLE_BUILD_CACHE_PASSWORD") ?: buildCacheCredentials.get("password")
+          username = System.getenv("GRADLE_BUILD_CACHE_USERNAME")
+          password = System.getenv("GRADLE_BUILD_CACHE_PASSWORD")
         }
       }
     }
   }
 }
-

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -24,12 +24,17 @@ allprojects {
   }
 }
 
+interface Injected {
+  @Inject FileSystemOperations getFs()
+}
+
 // Applying this stuff, particularly the idea-ext plugin, has a cost so avoid it unless we're running in the IDE
 if (providers.systemProperty('idea.active').getOrNull() == 'true') {
   project.apply(plugin: org.jetbrains.gradle.ext.IdeaExtPlugin)
 
   def elasticsearchProject = locateElasticsearchWorkspace(gradle)
 
+  def rootFolder = project.rootDir
   tasks.register('configureIdeCheckstyle') {
     group = 'ide'
     description = 'Generated a suitable checkstyle config for IDEs'
@@ -39,10 +44,10 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     String checkstyleConfig = "${resources}/checkstyle.xml"
     String checkstyleSuppressions = "${resources}/checkstyle_suppressions.xml"
     String checkstyleIdeFragment = "${resources}/checkstyle_ide_fragment.xml"
-    String checkstyleIdeConfig = "${rootDir}/checkstyle_ide.xml"
+    String checkstyleIdeConfig = "${rootFolder}/checkstyle_ide.xml"
 
     String checkstylePluginConfigTemplate = "${resources}/checkstyle-idea.xml"
-    String checkstylePluginConfig = "${rootDir}/.idea/checkstyle-idea.xml"
+    String checkstylePluginConfig = "${rootFolder}/.idea/checkstyle-idea.xml"
 
     inputs.files(
       file(checkstyleConfig),
@@ -53,31 +58,33 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
       file(checkstyleIdeConfig),
       file(checkstylePluginConfig)
     )
+    def injected = project.objects.newInstance(Injected)
 
+    def projectFolder = project.layout.projectDirectory.asFile
     doLast {
       // Configure the IntelliJ Checkstyle plugin by copying a standard file. We don't simply commit
       // the result to version control, because the plugin has a habit of modifying the file and
       // replacing the `$PROJECT_DIR$` placeholders, which developers must then revert.
-      project.copy {
+      injected.fs.copy {
         from(checkstylePluginConfigTemplate)
-        into("${rootDir}/.idea")
+        into("${rootFolder}/.idea")
         expand(jarLocation: buildConventionsJar, configLocation: checkstyleIdeConfig)
       }
 
       // Create an IDE-specific checkstyle config by first copying the standard config
       Files.copy(
-        Paths.get(file(checkstyleConfig).getPath()),
-        Paths.get(file(checkstyleIdeConfig).getPath()),
+        Paths.get(new File(checkstyleConfig).getPath()),
+        Paths.get(new File(checkstyleIdeConfig).getPath()),
         StandardCopyOption.REPLACE_EXISTING
       )
 
       // There are some rules that we only want to enable in an IDE. These
       // are extracted to a separate file, and merged into the IDE-specific
       // Checkstyle config.
-      Node xmlFragment = parseXml(checkstyleIdeFragment)
+      Node xmlFragment = IdeaXmlUtil.parseXml(checkstyleIdeFragment)
 
       // Edit the copy so that IntelliJ can copy with it
-      modifyXml(checkstyleIdeConfig, { xml ->
+      IdeaXmlUtil.modifyXml(checkstyleIdeConfig, { xml ->
         // Add all the nodes from the fragment file
         Node treeWalker = xml.module.find { it.'@name' == 'TreeWalker' }
         xmlFragment.module.each { treeWalker.append(it) }
@@ -103,7 +110,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     description = 'Configures the appropriate JVM for Gradle'
 
     doLast {
-      modifyXml('.idea/gradle.xml') { xml ->
+      IdeaXmlUtil.modifyXml('.idea/gradle.xml') { xml ->
         def gradleSettings = xml.component.find { it.'@name' == 'GradleSettings' }.option[0].GradleProjectSettings
         // Remove configured JVM option to force IntelliJ to use the project JDK for Gradle
         gradleSettings.option.findAll { it.'@name' == 'gradleJvm' }.each { it.parent().remove(it) }
@@ -127,7 +134,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     description = 'Enable per-module *.iml files'
 
     doLast {
-      modifyXml('.idea/misc.xml') {xml ->
+      IdeaXmlUtil.modifyXml('.idea/misc.xml') {xml ->
         def externalStorageConfig = xml.component.find { it.'@name' == 'ExternalStorageConfigurationManager' }
         if (externalStorageConfig) {
           xml.remove(externalStorageConfig)
@@ -142,13 +149,13 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     description = 'Enables preview features on native library module'
     dependsOn tasks.named("enableExternalConfiguration")
 
-    ext {
-      enablePreview = { moduleFile, languageLevel ->
-        modifyXml(moduleFile) { xml ->
+//    ext {
+      def enablePreview = { moduleFile, languageLevel ->
+        IdeaXmlUtil.modifyXml(moduleFile) { xml ->
           xml.component.find { it.'@name' == 'NewModuleRootManager' }?.'@LANGUAGE_LEVEL' = languageLevel
         }
       }
-    }
+//    }
 
     doLast {
       enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.main.iml', 'JDK_21_PREVIEW')
@@ -278,33 +285,37 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
  * @param preface optional front matter to add after the XML declaration
  * but before the XML document, e.g. a doctype or comment
  */
-void modifyXml(Object path, Action<? super Node> action, String preface = null) {
-  if (project.file(path).exists()) {
-    Node xml = parseXml(path)
-    action.execute(xml)
-
-    File xmlFile = project.file(path)
-    xmlFile.withPrintWriter { writer ->
-      def printer = new XmlNodePrinter(writer)
-      printer.namespaceAware = true
-      printer.preserveWhitespace = true
-      writer.write("<?xml version=\"1.0\"?>\n")
-
-      if (preface != null) {
-        writer.write(preface)
+
+class IdeaXmlUtil {
+  static Node parseXml(Object xmlPath) {
+    File xmlFile = new File(xmlPath)
+    XmlParser xmlParser = new XmlParser(false, true, true)
+    xmlParser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false)
+    Node xml = xmlParser.parse(xmlFile)
+    return xml
+  }
+
+  static void modifyXml(Object xmlPath, Action<? super Node> action, String preface = null) {
+    File xmlFile = new File(xmlPath)
+    if (xmlFile.exists()) {
+      Node xml = parseXml(xmlPath)
+      action.execute(xml)
+
+      xmlFile.withPrintWriter { writer ->
+        def printer = new XmlNodePrinter(writer)
+        printer.namespaceAware = true
+        printer.preserveWhitespace = true
+        writer.write("<?xml version=\"1.0\"?>\n")
+
+        if (preface != null) {
+          writer.write(preface)
+        }
+        printer.print(xml)
       }
-      printer.print(xml)
     }
   }
 }
 
-Node parseXml(Object path) {
-  File xmlFile = project.file(path)
-  XmlParser xmlParser = new XmlParser(false, true, true)
-  xmlParser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false)
-  Node xml = xmlParser.parse(xmlFile)
-  return xml
-}
 
 Pair<File, IncludedBuild> locateElasticsearchWorkspace(Gradle gradle) {
   if (gradle.parent == null) {

distribution/docker/src/docker/iron_bank/hardening_manifest.yaml

@@ -1,21 +1,17 @@
 ---
 apiVersion: v1
-
 # The repository name in registry1, excluding /ironbank/
 name: "elastic/elasticsearch/elasticsearch"
-
 # List of tags to push for the repository in registry1
 # The most specific version should be the first tag and will be shown
 # on ironbank.dsop.io
 tags:
 - "${version}"
 - "latest"
-
 # Build args passed to Dockerfile ARGs
 args:
   BASE_IMAGE: "redhat/ubi/ubi9"
-  BASE_TAG: "9.4"
-
+  BASE_TAG: "9.5"
 # Docker image labels
 labels:
   org.opencontainers.image.title: "elasticsearch"
@@ -34,7 +30,6 @@ labels:
   mil.dso.ironbank.image.type: "commercial"
   # Product the image belongs to for grouping multiple images
   mil.dso.ironbank.product.name: "elasticsearch"
-
 # List of resources to make available to the offline build context
 resources:
   - filename: "elasticsearch-${version}-linux-x86_64.tar.gz"
@@ -47,7 +42,6 @@ resources:
     validation:
       type: "sha256"
       value: "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c"
-
 # List of project maintainers
 maintainers:
   - name: "Mark Vieira"

docs/changelog/116687.yaml

@@ -0,0 +1,5 @@
+pr: 116687
+summary: Add LogsDB option to route on sort fields
+area: Logs
+type: enhancement
+issues: []

docs/changelog/118353.yaml

@@ -0,0 +1,5 @@
+pr: 118353
+summary: Epoch Millis Rounding Down and Not Up 2
+area: Infra/Core
+type: bug
+issues: []

docs/changelog/118562.yaml

@@ -0,0 +1,6 @@
+pr: 118562
+summary: Update data stream deprecations warnings to new format and filter searchable
+  snapshots from response
+area: Data streams
+type: enhancement
+issues: []

docs/changelog/118603.yaml

@@ -0,0 +1,6 @@
+pr: 118603
+summary: Allow DATE_PARSE to read the timezones
+area: ES|QL
+type: bug
+issues:
+ - 117680

docs/changelog/118931.yaml

@@ -0,0 +1,6 @@
+pr: 118931
+summary: Add a `LicenseAware` interface for licensed Nodes
+area: ES|QL
+type: enhancement
+issues:
+ - 117405

docs/changelog/118941.yaml

@@ -0,0 +1,5 @@
+pr: 118941
+summary: Allow archive and searchable snapshots indices in N-2 version
+area: Recovery
+type: enhancement
+issues: []

docs/reference/esql/esql-limitations.asciidoc

@@ -112,7 +112,7 @@ it is necessary to use the search function, like <<esql-match>>, in a <<esql-whe
 directly after the <<esql-from>> source command, or close enough to it.
 Otherwise, the query will fail with a validation error.
 Another limitation is that any <<esql-where>> command containing a full-text search function
-cannot also use disjunctions (`OR`).
+cannot also use disjunctions (`OR`) unless all functions used in the OR clauses are full-text functions themselves.
 
 For example, this query is valid:
 
@@ -139,6 +139,15 @@ FROM books
 | WHERE MATCH(author, "Faulkner") OR author LIKE "Hemingway"
 ----
 
+However this query will succeed because it uses full text functions on both `OR` clauses:
+
+[source,esql]
+----
+FROM books
+| WHERE MATCH(author, "Faulkner") OR QSTR("author: Hemingway")
+----
+
+
 Note that, because of <<esql-limitations-text-fields,the way {esql} treats `text` values>>,
 any queries on `text` fields that do not explicitly use the full-text functions,
 <<esql-match>> or <<esql-qstr>>, will behave as if the fields are actually `keyword` fields: