Fix multi-node tests

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

.github/actions/build-opensearch/action.yml

@@ -15,7 +15,7 @@ inputs:
   plugins_output_directory:
     description: The directory to output the plugins to
     default: ""
-outputs: 
+outputs:
     distribution:
       description: The path to the OpenSearch distribution
       value: ${{ steps.determine.outputs.distribution }}
@@ -36,11 +36,11 @@ runs:
           ./opensearch/distribution/archives/linux-tar/build/distributions/opensearch-*.tar.gz
           ./opensearch/plugins/*/build/distributions/*.zip
         build_script: |
-          ./gradlew :distribution:archives:linux-tar:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }}
+          ./gradlew --stacktrace :distribution:archives:linux-tar:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }}
 
           PluginList=("analysis-icu"  "analysis-kuromoji"  "analysis-nori"  "analysis-phonetic"  "ingest-attachment"  "mapper-murmur3")
           for plugin in ${PluginList[*]}; do
-            ./gradlew :plugins:$plugin:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }}
+            ./gradlew --stacktrace :plugins:$plugin:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }}
           done
 
     - name: Determine OpenSearch distribution path and version
@@ -62,7 +62,7 @@ runs:
         cache_key_suffix: ${{ inputs.build_snapshot == 'true' && '-snapshot' || '' }}
         cached_paths: |
           ./opensearch-security/build/distributions/opensearch-security-*.zip
-        build_script: ./gradlew assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }}
+        build_script: ./gradlew --stacktrace assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }}
 
     - name: Restore or Build OpenSearch k-NN
       uses: ./client/.github/actions/cached-git-build
@@ -76,7 +76,7 @@ runs:
           ./opensearch-knn/build/distributions/opensearch-knn-*.zip
         build_script: |
           sudo apt-get install -y libopenblas-dev libomp-dev
-          ./gradlew buildJniLib assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }}
+          ./gradlew --stacktrace buildJniLib assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }}
           distributions=./build/distributions
           lib_dir=$distributions/lib
           mkdir $lib_dir

.github/workflows/integration-yaml-tests.yml

@@ -87,7 +87,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        opensearch_ref: ['1.x', '2.x', 'main']
+        include:
+          - { opensearch_ref: '1.x', java_version: '11' }
+          - { opensearch_ref: '2.x', java_version: '17' }
+          - { opensearch_ref: 'main', java_version: '21' }
     steps:
       - name: Checkout Client
         uses: actions/checkout@v4
@@ -111,8 +114,8 @@ jobs:
       - name: Set up JDK
         uses: actions/setup-java@v4
         with:
-          java-version: 21
           distribution: 'temurin'
+          java-version: ${{ matrix.java_version }}
 
       - name: Restore or Build OpenSearch
         id: opensearch_build

.github/workflows/integration.yml

@@ -50,7 +50,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-nuget-
 
-      - run: "./build.sh integrate $VERSION readonly,replicatedreadonly,writable random:test_only_one --report"
+      - run: "./build.sh integrate $VERSION readonly,multinode,writable random:test_only_one --report"
         name: Integration Tests
         working-directory: client
         env:
@@ -69,10 +69,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        opensearch_ref:
-          - '1.x'
-          - '2.x'
-          - 'main'
+        include:
+          - { opensearch_ref: '1.x', java_version: '11' }
+          - { opensearch_ref: '2.x', java_version: '17' }
+          - { opensearch_ref: 'main', java_version: '21' }
 
     steps:
       - name: Checkout Client
@@ -100,6 +100,12 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-nuget-
 
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java_version }}
+
       - name: Restore or Build OpenSearch
         id: opensearch
         uses: ./client/.github/actions/build-opensearch
@@ -109,7 +115,7 @@ jobs:
           knn_plugin: true
           plugins_output_directory: ${{ env.OPENSEARCH_PLUGINS_DIRECTORY }}
 
-      - run: "./build.sh integrate $OPENSEARCH_VERSION readonly,replicatedreadonly,writable random:test_only_one --report"
+      - run: "./build.sh integrate $OPENSEARCH_VERSION readonly,multinode,writable random:test_only_one --report"
         name: Integration Tests
         working-directory: client
         env:

abstractions/src/OpenSearch.OpenSearch.Managed/ClusterBase.cs

@@ -75,29 +75,32 @@ protected ClusterBase(TConfiguration clusterConfiguration)
 			ClusterConfiguration = clusterConfiguration;
 			ClusterMoniker = GetType().Name.Replace("Cluster", "");
 
-			NodeConfiguration Modify(NodeConfiguration n, int p)
-			{
-				ModifyNodeConfiguration(n, p);
-				return n;
-			}
-
-			var nodes =
-				(from port in Enumerable.Range(ClusterConfiguration.StartingPortNumber,
-						ClusterConfiguration.NumberOfNodes)
-					let config = new NodeConfiguration(clusterConfiguration, port, ClusterMoniker)
-					{
-						ShowOpenSearchOutputAfterStarted =
-							clusterConfiguration.ShowOpenSearchOutputAfterStarted,
-					}
-					let node = new OpenSearchNode(Modify(config, port))
-					{
-						AssumeStartedOnNotEnoughMasterPing = ClusterConfiguration.NumberOfNodes > 1,
-					}
-					select node).ToList();
-
-			var initialMasterNodes = string.Join(",", nodes.Select(n => n.NodeConfiguration.DesiredNodeName));
-			foreach (var node in nodes)
-				node.NodeConfiguration.InitialMasterNodes(initialMasterNodes);
+            var nodeConfigs = Enumerable.Range(ClusterConfiguration.StartingPortNumber, ClusterConfiguration.NumberOfNodes)
+                .Select(port => new NodeConfiguration(ClusterConfiguration, port, ClusterMoniker)
+                {
+                    ShowOpenSearchOutputAfterStarted = ClusterConfiguration.ShowOpenSearchOutputAfterStarted
+                })
+                .ToArray();
+
+            var initialClusterManagerNodes = string.Join(",", nodeConfigs.Select(n => n.DesiredNodeName));
+
+            var nodes = nodeConfigs
+                .Select(config =>
+                {
+                    if (nodeConfigs.Length > 1)
+                    {
+                        var otherNodes = nodeConfigs
+                            .Where(n => n != config)
+                            .Select(n => $"localhost:{(n.DesiredPort ?? 9200) + 100}");
+                        config.SeedHosts(string.Join(",", otherNodes));
+                    }
+
+                    config.InitialClusterManagerNodes(initialClusterManagerNodes);
+                    ModifyNodeConfiguration(config, config.DesiredPort ?? 9200);
+
+                    return new OpenSearchNode(config) { AssumeStartedOnNotEnoughClusterManagerPing = ClusterConfiguration.NumberOfNodes > 1 };
+                })
+                .ToArray();
 
 			Nodes = new ReadOnlyCollection<OpenSearchNode>(nodes);
 		}

abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/ClusterConfiguration.cs

@@ -90,11 +90,8 @@ public ClusterConfiguration(OpenSearchVersion version, Func<OpenSearchVersion, s
 			NumberOfNodes = numberOfNodes;
 
 			var fs = FileSystem;
-			Add("node.max_local_storage_nodes", numberOfNodes.ToString(CultureInfo.InvariantCulture), "1.0.0");
-
 			Add("cluster.name", clusterName);
 			Add("path.repo", fs.RepositoryPath);
-			Add("path.data", fs.DataPath);
 			var logsPathDefault = Path.Combine(fs.OpenSearchHome, "logs");
 			if (logsPathDefault != fs.LogsPath) Add("path.logs", fs.LogsPath);
 		}
@@ -122,7 +119,7 @@ public ClusterConfiguration(OpenSearchVersion version, Func<OpenSearchVersion, s
 		public bool CacheOpenSearchHomeInstallation { get; set; }
 
 		/// <summary>The node settings to apply to each started node</summary>
-		public NodeSettings DefaultNodeSettings { get; } = new NodeSettings();
+		public NodeSettings DefaultNodeSettings { get; } = new();
 
 		/// <summary>
 		///     Creates a node name

abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeConfiguration.cs

@@ -28,6 +28,7 @@
 
 using System;
 using System.Globalization;
+using System.IO;
 using OpenSearch.OpenSearch.Managed.FileSystem;
 using OpenSearch.Stack.ArtifactsApi;
 using ProcNet;
@@ -49,12 +50,18 @@ public NodeConfiguration(IClusterConfiguration<NodeFileSystem> clusterConfigurat
 			ClusterConfiguration = clusterConfiguration;
 			DesiredPort = port;
 			DesiredNodeName = CreateNodeName(port, nodePrefix) ?? clusterConfiguration.CreateNodeName(port);
-			Settings = new NodeSettings(clusterConfiguration.DefaultNodeSettings);
-
-			if (!string.IsNullOrWhiteSpace(DesiredNodeName)) Settings.Add("node.name", DesiredNodeName);
-			if (DesiredPort.HasValue)
-				Settings.Add("http.port", DesiredPort.Value.ToString(CultureInfo.InvariantCulture));
-		}
+			Settings = new NodeSettings(clusterConfiguration.DefaultNodeSettings)
+            {
+                { "path.data", Path.Combine(ClusterConfiguration.FileSystem.DataPath, DesiredNodeName) }
+            };
+
+            if (!string.IsNullOrWhiteSpace(DesiredNodeName)) Settings.Add("node.name", DesiredNodeName);
+            if (DesiredPort is { } desiredPort)
+            {
+                Settings.Add("http.port", desiredPort.ToString(CultureInfo.InvariantCulture));
+                Settings.Add("transport.port", (desiredPort + 100).ToString(CultureInfo.InvariantCulture));
+            }
+        }
 
 		private IClusterConfiguration<NodeFileSystem> ClusterConfiguration { get; }
 
@@ -87,10 +94,15 @@ public Action<StartArguments> ModifyStartArguments
 		public OpenSearchVersion Version => ClusterConfiguration.Version;
 		public string[] CommandLineArguments => Settings.ToCommandLineArguments(Version);
 
-		public void InitialMasterNodes(string initialMasterNodes) =>
-			Settings.Add("cluster.initial_master_nodes", initialMasterNodes, ">=1.0.0");
+        public void SeedHosts(string seedHosts) => Settings.Add("discovery.seed_hosts", seedHosts);
+
+        public void InitialClusterManagerNodes(string initialClusterManagerNodes)
+        {
+            Settings.Add("cluster.initial_master_nodes", initialClusterManagerNodes, ">=1.0.0 <2.0.0");
+            Settings.Add("cluster.initial_cluster_manager_nodes", initialClusterManagerNodes, ">=2.0.0");
+        }
 
-		public string AttributeKey(string attribute)
+        public string AttributeKey(string attribute)
 		{
 			var attr = "attr.";
 			return $"node.{attr}{attribute}";

abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchNode.cs

@@ -64,7 +64,7 @@ public OpenSearchNode(OpenSearchVersion version, string openSearchHome = null)
 		///     doing the election.
 		///     <para>Useful to speed up starting multi node clusters</para>
 		/// </summary>
-		public bool AssumeStartedOnNotEnoughMasterPing { get; set; }
+		public bool AssumeStartedOnNotEnoughClusterManagerPing { get; set; }
 
 		internal IConsoleLineHandler Writer { get; private set; }
 
@@ -119,7 +119,7 @@ private static void AppendPathEnvVar(string name, string value)
 
 		private bool AssumedStartedStateChecker(string section, string message)
 		{
-			if (AssumeStartedOnNotEnoughMasterPing
+			if (AssumeStartedOnNotEnoughClusterManagerPing
 				&& section.Contains("ZenDiscovery")
 				&& message.Contains("not enough master nodes discovered during pinging"))
 				return true;

abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterBase.cs

@@ -33,21 +33,13 @@ namespace OpenSearch.OpenSearch.Xunit
 	/// <summary>
 	///     Base class for a cluster that integrates with Xunit tests
 	/// </summary>
-	public abstract class XunitClusterBase : XunitClusterBase<XunitClusterConfiguration>
-	{
-		protected XunitClusterBase(XunitClusterConfiguration configuration) : base(configuration)
-		{
-		}
-	}
+	public abstract class XunitClusterBase(XunitClusterConfiguration configuration)
+        : XunitClusterBase<XunitClusterConfiguration>(configuration);
 
 	/// <summary>
 	///     Base class for a cluster that integrates with Xunit tests
 	/// </summary>
-	public abstract class XunitClusterBase<TConfiguration> : EphemeralCluster<TConfiguration>
-		where TConfiguration : XunitClusterConfiguration
-	{
-		protected XunitClusterBase(TConfiguration configuration) : base(configuration)
-		{
-		}
-	}
+	public abstract class XunitClusterBase<TConfiguration>(TConfiguration configuration)
+        : EphemeralCluster<TConfiguration>(configuration)
+        where TConfiguration : XunitClusterConfiguration;
 }

build/scripts/scripts.fsproj

@@ -32,15 +32,17 @@
     <Content Include="..\..\.github\license-header-fs.txt"><Link>license-header-fs.txt</Link></Content>
   </ItemGroup>
   <ItemGroup>
+    <PackageReference Update="FSharp.Core" Version="8.0.301" />
+
     <PackageReference Include="Bullseye" Version="5.0.0" />
     <ProjectReference Include="$(SolutionRoot)\abstractions\src\OpenSearch.OpenSearch.Managed\OpenSearch.OpenSearch.Managed.csproj" />
-    
+
     <PackageReference Include="Fake.Core.Environment" Version="6.1.0" />
     <PackageReference Include="Fake.Core.SemVer" Version="6.1.0" />
     <PackageReference Include="Fake.IO.FileSystem" Version="6.1.0" />
     <PackageReference Include="Fake.IO.Zip" Version="6.1.0" />
     <PackageReference Include="Fake.Tools.Git" Version="5.23.1" />
-    
+
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
 
     <PackageReference Include="Octokit" Version="13.0.1" />

src/ApiGenerator/Generator/ApiEndpointFactory.cs

@@ -289,6 +289,9 @@ private static string SanitizeDescription(this string description)
         {
             if (string.IsNullOrWhiteSpace(description)) return null;
 
+            description = Regex.Replace(description, "&", "&");
+            description = Regex.Replace(description, "<", "&lt;");
+            description = Regex.Replace(description, ">", "&gt;");
             description = Regex.Replace(description, @"\s+", " ");
 
             if (!description.EndsWith('.')) description += '.';

src/OpenSearch.Client/_Generated/Descriptors.Cluster.cs

@@ -336,7 +336,7 @@ public ClusterHealthDescriptor WaitForActiveShards(string waitforactiveshards) =
         public ClusterHealthDescriptor WaitForEvents(WaitForEvents? waitforevents) =>
             Qs("wait_for_events", waitforevents);
 
-        /// <summary>The request waits until the specified number N of nodes is available. It also accepts >=N, <=N, >N and <N. Alternatively, it is possible to use ge(N), le(N), gt(N) and lt(N) notation.</summary>
+        /// <summary>The request waits until the specified number N of nodes is available. It also accepts &gt;=N, &lt;=N, &gt;N and &lt;N. Alternatively, it is possible to use ge(N), le(N), gt(N) and lt(N) notation.</summary>
         public ClusterHealthDescriptor WaitForNodes(string waitfornodes) =>
             Qs("wait_for_nodes", waitfornodes);
 
@@ -350,7 +350,7 @@ public ClusterHealthDescriptor WaitForNoRelocatingShards(
             bool? waitfornorelocatingshards = true
         ) => Qs("wait_for_no_relocating_shards", waitfornorelocatingshards);
 
-        /// <summary>One of green, yellow or red. Will wait (until the timeout provided) until the status of the cluster changes to the one provided or better, i.e. green > yellow > red. By default, will not wait for any status.</summary>
+        /// <summary>One of green, yellow or red. Will wait (until the timeout provided) until the status of the cluster changes to the one provided or better, i.e. green &gt; yellow &gt; red. By default, will not wait for any status.</summary>
         public ClusterHealthDescriptor WaitForStatus(HealthStatus? waitforstatus) =>
             Qs("wait_for_status", waitforstatus);
     }

src/OpenSearch.Client/_Generated/Requests.Cluster.cs

@@ -458,8 +458,8 @@ public WaitForEvents? WaitForEvents
         }
 
         /// <summary>
-        /// The request waits until the specified number N of nodes is available. It also accepts >=N, <=N, >N and <N. Alternatively, it is possible
-        /// to use ge(N), le(N), gt(N) and lt(N) notation.
+        /// The request waits until the specified number N of nodes is available. It also accepts &gt;=N, &lt;=N, &gt;N and &lt;N. Alternatively, it
+        /// is possible to use ge(N), le(N), gt(N) and lt(N) notation.
         /// </summary>
         public string WaitForNodes
         {
@@ -489,7 +489,7 @@ public bool? WaitForNoRelocatingShards
 
         /// <summary>
         /// One of green, yellow or red. Will wait (until the timeout provided) until the status of the cluster changes to the one provided or better,
-        /// i.e. green > yellow > red. By default, will not wait for any status.
+        /// i.e. green &gt; yellow &gt; red. By default, will not wait for any status.
         /// </summary>
         public HealthStatus? WaitForStatus
         {

src/OpenSearch.Net/_Generated/Api/RequestParameters/RequestParameters.Cluster.cs

@@ -367,8 +367,8 @@ public WaitForEvents? WaitForEvents
         }
 
         /// <summary>
-        /// The request waits until the specified number N of nodes is available. It also accepts >=N, <=N, >N and <N. Alternatively, it is possible
-        /// to use ge(N), le(N), gt(N) and lt(N) notation.
+        /// The request waits until the specified number N of nodes is available. It also accepts &gt;=N, &lt;=N, &gt;N and &lt;N. Alternatively, it
+        /// is possible to use ge(N), le(N), gt(N) and lt(N) notation.
         /// </summary>
         public string WaitForNodes
         {
@@ -398,7 +398,7 @@ public bool? WaitForNoRelocatingShards
 
         /// <summary>
         /// One of green, yellow or red. Will wait (until the timeout provided) until the status of the cluster changes to the one provided or better,
-        /// i.e. green > yellow > red. By default, will not wait for any status.
+        /// i.e. green &gt; yellow &gt; red. By default, will not wait for any status.
         /// </summary>
         public HealthStatus? WaitForStatus
         {