Merge branch 'main' into make-thirdpartycheckinputseffective

.buildkite/pipelines/periodic-packaging.template.yml

@@ -14,6 +14,7 @@ steps:
               - sles-15
               - ubuntu-2004
               - ubuntu-2204
+              - ubuntu-2404
               - rocky-8
               - rocky-9
               - rhel-8

.buildkite/pipelines/periodic-packaging.yml

@@ -15,6 +15,7 @@ steps:
               - sles-15
               - ubuntu-2004
               - ubuntu-2204
+              - ubuntu-2404
               - rocky-8
               - rocky-9
               - rhel-8

.buildkite/pipelines/periodic-platform-support.yml

@@ -14,6 +14,7 @@ steps:
               - sles-15
               - ubuntu-2004
               - ubuntu-2204
+              - ubuntu-2404
               - rocky-8
               - rocky-9
               - rhel-8

.buildkite/pipelines/pull-request/packaging-tests-unix.yml

@@ -17,6 +17,7 @@ steps:
               - sles-15
               - ubuntu-2004
               - ubuntu-2204
+              - ubuntu-2404
               - rocky-8
               - rocky-9
               - rhel-8

.buildkite/scripts/gradle-configuration-cache-validation.sh

@@ -3,16 +3,16 @@
 set -euo pipefail
 
 # This is a workaround for https://github.com/gradle/gradle/issues/28159
-.ci/scripts/run-gradle.sh --no-daemon precommit
+.ci/scripts/run-gradle.sh --no-daemon $@
 
-.ci/scripts/run-gradle.sh --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2
+.ci/scripts/run-gradle.sh --configuration-cache -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 $@
 
 # Create a temporary file
 tmpOutputFile=$(mktemp)
 trap "rm $tmpOutputFile" EXIT
 
 echo "2nd run"
-.ci/scripts/run-gradle.sh --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 | tee $tmpOutputFile
+.ci/scripts/run-gradle.sh --configuration-cache -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 $@ | tee $tmpOutputFile
 
 # Check if the command was successful
 if grep -q "Configuration cache entry reused." $tmpOutputFile; then

.ci/scripts/run-gradle.sh

@@ -5,35 +5,39 @@ mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d
 
 MAX_WORKERS=4
 
-# Don't run this stuff on Windows
-if ! uname -a | grep -q MING; then
-  # drop page cache and kernel slab objects on linux
-  [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
+if [ -z "$MAX_WORKER_ENV" ]; then
+    # Don't run this stuff on Windows
+    if ! uname -a | grep -q MING; then
+      # drop page cache and kernel slab objects on linux
+      [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
 
-  if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
-    MAX_WORKERS=16
-  elif [ -f /proc/cpuinfo ]; then
-    MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo  | uniq | sed 's/\s\+//g' |  cut -d':' -f 2`
-  else
-    if [[ "$OSTYPE" == "darwin"* ]]; then
-        MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'`
-    else
-        echo "Unsupported OS Type: $OSTYPE"
-        exit 1
-    fi
-  fi
-  if pwd | grep -v -q ^/dev/shm ; then
-    echo "Not running on a ramdisk, reducing number of workers"
-    MAX_WORKERS=$(($MAX_WORKERS*2/3))
-  fi
+      if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
+        MAX_WORKERS=16
+      elif [ -f /proc/cpuinfo ]; then
+        MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo  | uniq | sed 's/\s\+//g' |  cut -d':' -f 2`
+      else
+        if [[ "$OSTYPE" == "darwin"* ]]; then
+            MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'`
+        else
+            echo "Unsupported OS Type: $OSTYPE"
+            exit 1
+        fi
+      fi
+      if pwd | grep -v -q ^/dev/shm ; then
+        echo "Not running on a ramdisk, reducing number of workers"
+        MAX_WORKERS=$(($MAX_WORKERS*2/3))
+      fi
 
-  # Export glibc version as environment variable since some BWC tests are incompatible with later versions
-  export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/')
-fi
+      # Export glibc version as environment variable since some BWC tests are incompatible with later versions
+      export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/')
+    fi
 
-# Running on 2-core machines without ramdisk can make this value be 0
-if [[ "$MAX_WORKERS" == "0" ]]; then
-  MAX_WORKERS=1
+    # Running on 2-core machines without ramdisk can make this value be 0
+    if [[ "$MAX_WORKERS" == "0" ]]; then
+      MAX_WORKERS=1
+    fi
+else
+    MAX_WORKERS=$MAX_WORKER_ENV
 fi
 
 set -e

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java

@@ -16,6 +16,7 @@
 import org.gradle.api.Action;
 import org.gradle.api.GradleException;
 import org.gradle.api.Project;
+import org.gradle.api.Task;
 import org.gradle.api.logging.LogLevel;
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.Property;
@@ -103,10 +104,16 @@ private static TaskProvider<LoggedExec> createRunBwcGradleTask(
             loggedExec.dependsOn("checkoutBwcBranch");
             loggedExec.getWorkingDir().set(checkoutDir.get());
 
-            loggedExec.getNonTrackedEnvironment().put("JAVA_HOME", providerFactory.of(JavaHomeValueSource.class, spec -> {
-                spec.getParameters().getVersion().set(unreleasedVersionInfo.map(it -> it.version()));
-                spec.getParameters().getCheckoutDir().set(checkoutDir);
-            }).flatMap(s -> getJavaHome(objectFactory, toolChainService, Integer.parseInt(s))));
+            loggedExec.doFirst(new Action<Task>() {
+                @Override
+                public void execute(Task task) {
+                    Provider<String> minimumCompilerVersionValueSource = providerFactory.of(JavaHomeValueSource.class, spec -> {
+                        spec.getParameters().getVersion().set(unreleasedVersionInfo.map(it -> it.version()));
+                        spec.getParameters().getCheckoutDir().set(checkoutDir);
+                    }).flatMap(s -> getJavaHome(objectFactory, toolChainService, Integer.parseInt(s)));
+                    loggedExec.getNonTrackedEnvironment().put("JAVA_HOME", minimumCompilerVersionValueSource.get());
+                }
+            });
 
             if (isCi && OS.current() != OS.WINDOWS) {
                 // TODO: Disabled for now until we can figure out why files are getting corrupted
@@ -169,14 +176,6 @@ private static Provider<String> getJavaHome(ObjectFactory objectFactory, JavaToo
             .map(launcher -> launcher.getMetadata().getInstallationPath().getAsFile().getAbsolutePath());
     }
 
-    private static String readFromFile(File file) {
-        try {
-            return FileUtils.readFileToString(file).trim();
-        } catch (IOException ioException) {
-            throw new GradleException("Cannot read java properties file.", ioException);
-        }
-    }
-
     public abstract static class JavaHomeValueSource implements ValueSource<String, JavaHomeValueSource.Params> {
 
         private String minimumCompilerVersionPath(Version bwcVersion) {
@@ -185,6 +184,14 @@ private String minimumCompilerVersionPath(Version bwcVersion) {
                 : "buildSrc/" + MINIMUM_COMPILER_VERSION_PATH;
         }
 
+        private static String readFromFile(File file) {
+            try {
+                return FileUtils.readFileToString(file).trim();
+            } catch (IOException ioException) {
+                throw new GradleException("Cannot read java properties file.", ioException);
+            }
+        }
+
         @Override
         public String obtain() {
             return readFromFile(

distribution/docker/src/docker/iron_bank/hardening_manifest.yaml

@@ -50,9 +50,12 @@ resources:
 
 # List of project maintainers
 maintainers:
-  - name: "Rory Hunter"
-    email: "rory.hunter@elastic.co"
-    username: "rory"
+  - name: "Mark Vieira"
+    email: "mark.vieira@elastic.co"
+    username: "mark-vieira"
+  - name: "Rene Gröschke"
+    email: "rene.groschke@elastic.co"
+    username: "breskeby"
   - email: "klepal_alexander@bah.com"
     name: "Alexander Klepal"
     username: "alexander.klepal"

docs/changelog/117581.yaml

@@ -0,0 +1,5 @@
+pr: 117581
+summary: Make reserved built-in roles queryable
+area: Authorization
+type: enhancement
+issues: []

docs/changelog/117643.yaml

@@ -0,0 +1,6 @@
+pr: 117643
+summary: Drop null columns in text formats
+area: ES|QL
+type: bug
+issues:
+  - 116848

docs/changelog/117851.yaml

@@ -0,0 +1,5 @@
+pr: 117851
+summary: Addition of `tier_preference`, `creation_date` and `version` fields in Elasticsearch monitoring template
+area: Monitoring
+type: enhancement
+issues: []

docs/changelog/118454.yaml

@@ -0,0 +1,5 @@
+pr: 118454
+summary: Fix RLIKE folding with (unsupported) case insensitive pattern
+area: ES|QL
+type: bug
+issues: []

muted-tests.yml

@@ -40,9 +40,6 @@ tests:
 - class: org.elasticsearch.packaging.test.WindowsServiceTests
   method: test33JavaChanged
   issue: https://github.com/elastic/elasticsearch/issues/113177
-- class: org.elasticsearch.smoketest.MlWithSecurityIT
-  method: test {yaml=ml/sparse_vector_search/Test sparse_vector search with query vector and pruning config}
-  issue: https://github.com/elastic/elasticsearch/issues/108997
 - class: org.elasticsearch.packaging.test.WindowsServiceTests
   method: test80JavaOptsInEnvVar
   issue: https://github.com/elastic/elasticsearch/issues/113219
@@ -79,9 +76,6 @@ tests:
 - class: org.elasticsearch.xpack.restart.MLModelDeploymentFullClusterRestartIT
   method: testDeploymentSurvivesRestart {cluster=UPGRADED}
   issue: https://github.com/elastic/elasticsearch/issues/115528
-- class: org.elasticsearch.action.update.UpdateResponseTests
-  method: testToAndFromXContent
-  issue: https://github.com/elastic/elasticsearch/issues/115689
 - class: org.elasticsearch.xpack.shutdown.NodeShutdownIT
   method: testStalledShardMigrationProperlyDetected
   issue: https://github.com/elastic/elasticsearch/issues/115697
@@ -305,8 +299,12 @@ tests:
 - class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
   method: testInvalidJSON
   issue: https://github.com/elastic/elasticsearch/issues/116521
-- class: org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizerTests
-  issue: https://github.com/elastic/elasticsearch/issues/118721
+- class: org.elasticsearch.xpack.ccr.rest.ShardChangesRestIT
+  method: testShardChangesNoOperation
+  issue: https://github.com/elastic/elasticsearch/issues/118800
+- class: org.elasticsearch.xpack.security.QueryableReservedRolesIT
+  method: testDeletingAndCreatingSecurityIndexTriggersSynchronization
+  issue: https://github.com/elastic/elasticsearch/issues/118806
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -55,7 +55,6 @@ static TransportVersion def(int id) {
     public static final TransportVersion V_7_3_0 = def(7_03_00_99);
     public static final TransportVersion V_7_4_0 = def(7_04_00_99);
     public static final TransportVersion V_7_6_0 = def(7_06_00_99);
-    public static final TransportVersion V_7_7_0 = def(7_07_00_99);
     public static final TransportVersion V_7_8_0 = def(7_08_00_99);
     public static final TransportVersion V_7_8_1 = def(7_08_01_99);
     public static final TransportVersion V_7_9_0 = def(7_09_00_99);

server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java

@@ -89,7 +89,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.V_7_7_0;
+        return TransportVersions.ZERO;
     }
 
     @Override
@@ -166,7 +166,7 @@ public String getWriteableName() {
 
         @Override
         public TransportVersion getMinimalSupportedVersion() {
-            return TransportVersions.V_7_7_0;
+            return TransportVersions.ZERO;
         }
     }
 }

server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java

@@ -94,7 +94,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.V_7_7_0;
+        return TransportVersions.ZERO;
     }
 
     @Override
@@ -167,7 +167,7 @@ public String getWriteableName() {
 
         @Override
         public TransportVersion getMinimalSupportedVersion() {
-            return TransportVersions.V_7_7_0;
+            return TransportVersions.ZERO;
         }
     }
 }

server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java

@@ -217,7 +217,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.V_7_7_0;
+        return TransportVersions.ZERO;
     }
 
     @Override
@@ -311,7 +311,7 @@ public String getWriteableName() {
 
         @Override
         public TransportVersion getMinimalSupportedVersion() {
-            return TransportVersions.V_7_7_0;
+            return TransportVersions.ZERO;
         }
     }
 }

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -133,6 +133,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID_BACKPORT = def(8_520_00_0, Version.LUCENE_9_12_0);
     public static final IndexVersion V8_DEPRECATE_SOURCE_MODE_MAPPER = def(8_521_00_0, Version.LUCENE_9_12_0);
     public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BACKPORT = def(8_522_00_0, Version.LUCENE_9_12_0);
+    public static final IndexVersion UPGRADE_TO_LUCENE_9_12_1 = def(8_523_00_0, parseUnchecked("9.12.1"));
     public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_00_0, Version.LUCENE_10_0_0);
     public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT = def(9_001_00_0, Version.LUCENE_10_0_0);
     public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID = def(9_002_00_0, Version.LUCENE_10_0_0);

server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java

@@ -48,7 +48,7 @@ public final class NoOpEngine extends ReadOnlyEngine {
     public NoOpEngine(EngineConfig config) {
         this(
             config,
-            config.isPromotableToPrimary() ? null : new TranslogStats(0, 0, 0, 0, 0),
+            config.isPromotableToPrimary() && config.getTranslogConfig().hasTranslog() ? null : new TranslogStats(0, 0, 0, 0, 0),
             config.isPromotableToPrimary()
                 ? null
                 : new SeqNoStats(

server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java

@@ -98,7 +98,7 @@ public class ReadOnlyEngine extends Engine {
     public ReadOnlyEngine(
         EngineConfig config,
         SeqNoStats seqNoStats,
-        TranslogStats translogStats,
+        @Nullable TranslogStats translogStats,
         boolean obtainLock,
         Function<DirectoryReader, DirectoryReader> readerWrapperFunction,
         boolean requireCompleteHistory,
@@ -251,6 +251,7 @@ private static SeqNoStats buildSeqNoStats(EngineConfig config, SegmentInfos info
     }
 
     private static TranslogStats translogStats(final EngineConfig config, final SegmentInfos infos) throws IOException {
+        assert config.getTranslogConfig().hasTranslog();
         final String translogUuid = infos.getUserData().get(Translog.TRANSLOG_UUID_KEY);
         if (translogUuid == null) {
             throw new IllegalStateException("commit doesn't contain translog unique id");