Test fix?

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

.github/actions/build-opensearch/action.yml

@@ -15,7 +15,7 @@ inputs:
   plugins_output_directory:
     description: The directory to output the plugins to
     default: ""
-outputs: 
+outputs:
     distribution:
       description: The path to the OpenSearch distribution
       value: ${{ steps.determine.outputs.distribution }}
@@ -36,11 +36,11 @@ runs:
           ./opensearch/distribution/archives/linux-tar/build/distributions/opensearch-*.tar.gz
           ./opensearch/plugins/*/build/distributions/*.zip
         build_script: |
-          ./gradlew :distribution:archives:linux-tar:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }}
+          ./gradlew --stacktrace :distribution:archives:linux-tar:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }}
 
           PluginList=("analysis-icu"  "analysis-kuromoji"  "analysis-nori"  "analysis-phonetic"  "ingest-attachment"  "mapper-murmur3")
           for plugin in ${PluginList[*]}; do
-            ./gradlew :plugins:$plugin:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }}
+            ./gradlew --stacktrace :plugins:$plugin:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }}
           done
 
     - name: Determine OpenSearch distribution path and version
@@ -62,7 +62,7 @@ runs:
         cache_key_suffix: ${{ inputs.build_snapshot == 'true' && '-snapshot' || '' }}
         cached_paths: |
           ./opensearch-security/build/distributions/opensearch-security-*.zip
-        build_script: ./gradlew assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }}
+        build_script: ./gradlew --stacktrace assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }}
 
     - name: Restore or Build OpenSearch k-NN
       uses: ./client/.github/actions/cached-git-build
@@ -76,7 +76,7 @@ runs:
           ./opensearch-knn/build/distributions/opensearch-knn-*.zip
         build_script: |
           sudo apt-get install -y libopenblas-dev libomp-dev
-          ./gradlew buildJniLib assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }}
+          ./gradlew --stacktrace buildJniLib assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }}
           distributions=./build/distributions
           lib_dir=$distributions/lib
           mkdir $lib_dir

.github/workflows/integration.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches-ignore:
       - 'dependabot/**'
-  pull_request: {}
+  pull_request: { }
 
 env:
   OPENSEARCH_PLUGINS_DIRECTORY: /tmp/opensearch-plugins
@@ -57,11 +57,14 @@ jobs:
           VERSION: ${{ matrix.version }}
 
       - name: Upload test report
-        if: failure()
+        if: always()
+        #failure()
         uses: actions/upload-artifact@v3
         with:
           name: report-${{ matrix.version }}
-          path: client/build/output/*
+          path: |
+            client/build/output/*
+            /tmp/OpenSearchManaged/opensearch-${{ matrix.version }}/*/logs/**/*
 
   integration-opensearch-unreleased:
     name: Integration OpenSearch Unreleased

abstractions/src/OpenSearch.OpenSearch.Managed/ClusterBase.cs

@@ -75,29 +75,26 @@ protected ClusterBase(TConfiguration clusterConfiguration)
 			ClusterConfiguration = clusterConfiguration;
 			ClusterMoniker = GetType().Name.Replace("Cluster", "");
 
-			NodeConfiguration Modify(NodeConfiguration n, int p)
-			{
-				ModifyNodeConfiguration(n, p);
-				return n;
-			}
-
-			var nodes =
-				(from port in Enumerable.Range(ClusterConfiguration.StartingPortNumber,
-						ClusterConfiguration.NumberOfNodes)
-					let config = new NodeConfiguration(clusterConfiguration, port, ClusterMoniker)
-					{
-						ShowOpenSearchOutputAfterStarted =
-							clusterConfiguration.ShowOpenSearchOutputAfterStarted,
-					}
-					let node = new OpenSearchNode(Modify(config, port))
-					{
-						AssumeStartedOnNotEnoughMasterPing = ClusterConfiguration.NumberOfNodes > 1,
-					}
-					select node).ToList();
-
-			var initialMasterNodes = string.Join(",", nodes.Select(n => n.NodeConfiguration.DesiredNodeName));
-			foreach (var node in nodes)
-				node.NodeConfiguration.InitialMasterNodes(initialMasterNodes);
+            var nodeConfigs = Enumerable.Range(ClusterConfiguration.StartingPortNumber, ClusterConfiguration.NumberOfNodes)
+                .Select(port => new NodeConfiguration(ClusterConfiguration, port, ClusterMoniker)
+                {
+                    ShowOpenSearchOutputAfterStarted = ClusterConfiguration.ShowOpenSearchOutputAfterStarted
+                })
+                .ToArray();
+
+            var seedHosts = string.Join(",", nodeConfigs.Select(n => $"localhost:{(n.DesiredPort ?? 9200) + 100}"));
+            var initialClusterManagerNodes = string.Join(",", nodeConfigs.Select(n => n.DesiredNodeName));
+
+            var nodes = nodeConfigs
+                .Select(config =>
+                {
+                    config.SeedHosts(seedHosts);
+                    config.InitialClusterManagerNodes(initialClusterManagerNodes);
+                    ModifyNodeConfiguration(config, config.DesiredPort ?? 9200);
+
+                    return new OpenSearchNode(config) { AssumeStartedOnNotEnoughClusterManagerPing = ClusterConfiguration.NumberOfNodes > 1 };
+                })
+                .ToArray();
 
 			Nodes = new ReadOnlyCollection<OpenSearchNode>(nodes);
 		}
@@ -132,11 +129,11 @@ public IDisposable Start(IConsoleLineHandler writer, TimeSpan waitForStarted)
 
 			var subscriptions = new Subscriptions();
 
-			foreach (var node in Nodes)
-			{
-				subscriptions.Add(node.SubscribeLines(writer));
-				if (node.WaitForStarted(waitForStarted)) continue;
+            foreach (var node in Nodes) subscriptions.Add(node.SubscribeLines(writer));
 
+            var waitHandles = Nodes.Select(w => w.StartedHandle).ToArray();
+            if (!WaitHandle.WaitAll(waitHandles, waitForStarted))
+            {
 				var nodeExceptions = Nodes.Select(n => n.LastSeenException).Where(e => e != null).ToList();
 				writer?.WriteError(
 					$"{{{GetType().Name}.{nameof(Start)}}} cluster did not start after {waitForStarted}");

abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeConfiguration.cs

@@ -52,9 +52,12 @@ public NodeConfiguration(IClusterConfiguration<NodeFileSystem> clusterConfigurat
 			Settings = new NodeSettings(clusterConfiguration.DefaultNodeSettings);
 
 			if (!string.IsNullOrWhiteSpace(DesiredNodeName)) Settings.Add("node.name", DesiredNodeName);
-			if (DesiredPort.HasValue)
-				Settings.Add("http.port", DesiredPort.Value.ToString(CultureInfo.InvariantCulture));
-		}
+            if (DesiredPort is { } desiredPort)
+            {
+                Settings.Add("http.port", desiredPort.ToString(CultureInfo.InvariantCulture));
+                Settings.Add("transport.port", (desiredPort + 100).ToString(CultureInfo.InvariantCulture));
+            }
+        }
 
 		private IClusterConfiguration<NodeFileSystem> ClusterConfiguration { get; }
 
@@ -87,10 +90,15 @@ public Action<StartArguments> ModifyStartArguments
 		public OpenSearchVersion Version => ClusterConfiguration.Version;
 		public string[] CommandLineArguments => Settings.ToCommandLineArguments(Version);
 
-		public void InitialMasterNodes(string initialMasterNodes) =>
-			Settings.Add("cluster.initial_master_nodes", initialMasterNodes, ">=1.0.0");
+        public void SeedHosts(string seedHosts) => Settings.Add("discovery.seed_hosts", seedHosts);
+
+        public void InitialClusterManagerNodes(string initialClusterManagerNodes)
+        {
+            Settings.Add("cluster.initial_master_nodes", initialClusterManagerNodes, ">=1.0.0 <2.0.0");
+            Settings.Add("cluster.initial_cluster_manager_nodes", initialClusterManagerNodes, ">=2.0.0");
+        }
 
-		public string AttributeKey(string attribute)
+        public string AttributeKey(string attribute)
 		{
 			var attr = "attr.";
 			return $"node.{attr}{attribute}";

abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchNode.cs

@@ -64,7 +64,7 @@ public OpenSearchNode(OpenSearchVersion version, string openSearchHome = null)
 		///     doing the election.
 		///     <para>Useful to speed up starting multi node clusters</para>
 		/// </summary>
-		public bool AssumeStartedOnNotEnoughMasterPing { get; set; }
+		public bool AssumeStartedOnNotEnoughClusterManagerPing { get; set; }
 
 		internal IConsoleLineHandler Writer { get; private set; }
 
@@ -119,7 +119,7 @@ private static void AppendPathEnvVar(string name, string value)
 
 		private bool AssumedStartedStateChecker(string section, string message)
 		{
-			if (AssumeStartedOnNotEnoughMasterPing
+			if (AssumeStartedOnNotEnoughClusterManagerPing
 				&& section.Contains("ZenDiscovery")
 				&& message.Contains("not enough master nodes discovered during pinging"))
 				return true;

build/scripts/scripts.fsproj

@@ -32,15 +32,17 @@
     <Content Include="..\..\.github\license-header-fs.txt"><Link>license-header-fs.txt</Link></Content>
   </ItemGroup>
   <ItemGroup>
+    <PackageReference Update="FSharp.Core" Version="8.0.301" />
+
     <PackageReference Include="Bullseye" Version="5.0.0" />
     <ProjectReference Include="$(SolutionRoot)\abstractions\src\OpenSearch.OpenSearch.Managed\OpenSearch.OpenSearch.Managed.csproj" />
-    
+
     <PackageReference Include="Fake.Core.Environment" Version="6.1.0" />
     <PackageReference Include="Fake.Core.SemVer" Version="6.1.0" />
     <PackageReference Include="Fake.IO.FileSystem" Version="6.1.0" />
     <PackageReference Include="Fake.IO.Zip" Version="6.1.0" />
     <PackageReference Include="Fake.Tools.Git" Version="5.23.1" />
-    
+
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
 
     <PackageReference Include="Octokit" Version="13.0.1" />

src/ApiGenerator/Generator/ApiEndpointFactory.cs

@@ -289,6 +289,9 @@ private static string SanitizeDescription(this string description)
         {
             if (string.IsNullOrWhiteSpace(description)) return null;
 
+            description = Regex.Replace(description, "&", "&");
+            description = Regex.Replace(description, "<", "&lt;");
+            description = Regex.Replace(description, ">", "&gt;");
             description = Regex.Replace(description, @"\s+", " ");
 
             if (!description.EndsWith('.')) description += '.';

src/OpenSearch.Client/_Generated/Descriptors.Cluster.cs

@@ -336,7 +336,7 @@ public ClusterHealthDescriptor WaitForActiveShards(string waitforactiveshards) =
         public ClusterHealthDescriptor WaitForEvents(WaitForEvents? waitforevents) =>
             Qs("wait_for_events", waitforevents);
 
-        /// <summary>The request waits until the specified number N of nodes is available. It also accepts >=N, <=N, >N and <N. Alternatively, it is possible to use ge(N), le(N), gt(N) and lt(N) notation.</summary>
+        /// <summary>The request waits until the specified number N of nodes is available. It also accepts &gt;=N, &lt;=N, &gt;N and &lt;N. Alternatively, it is possible to use ge(N), le(N), gt(N) and lt(N) notation.</summary>
         public ClusterHealthDescriptor WaitForNodes(string waitfornodes) =>
             Qs("wait_for_nodes", waitfornodes);
 
@@ -350,7 +350,7 @@ public ClusterHealthDescriptor WaitForNoRelocatingShards(
             bool? waitfornorelocatingshards = true
         ) => Qs("wait_for_no_relocating_shards", waitfornorelocatingshards);
 
-        /// <summary>One of green, yellow or red. Will wait (until the timeout provided) until the status of the cluster changes to the one provided or better, i.e. green > yellow > red. By default, will not wait for any status.</summary>
+        /// <summary>One of green, yellow or red. Will wait (until the timeout provided) until the status of the cluster changes to the one provided or better, i.e. green &gt; yellow &gt; red. By default, will not wait for any status.</summary>
         public ClusterHealthDescriptor WaitForStatus(HealthStatus? waitforstatus) =>
             Qs("wait_for_status", waitforstatus);
     }

src/OpenSearch.Client/_Generated/Requests.Cluster.cs

@@ -458,8 +458,8 @@ public WaitForEvents? WaitForEvents
         }
 
         /// <summary>
-        /// The request waits until the specified number N of nodes is available. It also accepts >=N, <=N, >N and <N. Alternatively, it is possible
-        /// to use ge(N), le(N), gt(N) and lt(N) notation.
+        /// The request waits until the specified number N of nodes is available. It also accepts &gt;=N, &lt;=N, &gt;N and &lt;N. Alternatively, it
+        /// is possible to use ge(N), le(N), gt(N) and lt(N) notation.
         /// </summary>
         public string WaitForNodes
         {
@@ -489,7 +489,7 @@ public bool? WaitForNoRelocatingShards
 
         /// <summary>
         /// One of green, yellow or red. Will wait (until the timeout provided) until the status of the cluster changes to the one provided or better,
-        /// i.e. green > yellow > red. By default, will not wait for any status.
+        /// i.e. green &gt; yellow &gt; red. By default, will not wait for any status.
         /// </summary>
         public HealthStatus? WaitForStatus
         {

src/OpenSearch.Net/_Generated/Api/RequestParameters/RequestParameters.Cluster.cs

@@ -367,8 +367,8 @@ public WaitForEvents? WaitForEvents
         }
 
         /// <summary>
-        /// The request waits until the specified number N of nodes is available. It also accepts >=N, <=N, >N and <N. Alternatively, it is possible
-        /// to use ge(N), le(N), gt(N) and lt(N) notation.
+        /// The request waits until the specified number N of nodes is available. It also accepts &gt;=N, &lt;=N, &gt;N and &lt;N. Alternatively, it
+        /// is possible to use ge(N), le(N), gt(N) and lt(N) notation.
         /// </summary>
         public string WaitForNodes
         {
@@ -398,7 +398,7 @@ public bool? WaitForNoRelocatingShards
 
         /// <summary>
         /// One of green, yellow or red. Will wait (until the timeout provided) until the status of the cluster changes to the one provided or better,
-        /// i.e. green > yellow > red. By default, will not wait for any status.
+        /// i.e. green &gt; yellow &gt; red. By default, will not wait for any status.
         /// </summary>
         public HealthStatus? WaitForStatus
         {

tests/Tests.Core/ManagedOpenSearch/Clusters/ReadOnlyCluster.cs

@@ -41,12 +41,20 @@ public ReadOnlyCluster() : base(Knn, MapperMurmur3, Security) { }
 
 	public class ReplicatedReadOnlyCluster : ClientTestClusterBase
 	{
-		public ReplicatedReadOnlyCluster() : base(new ClientTestClusterConfiguration(numberOfNodes: 2, plugins: new[] {Knn, MapperMurmur3, Security})) { }
+        public ReplicatedReadOnlyCluster() : base(new ClientTestClusterConfiguration(numberOfNodes: 2, plugins: [Knn, MapperMurmur3, Security])
+        {
+            NoCleanupAfterNodeStopped = true
+        })
+        {
+        }
 
 		protected override void SeedNode() => new DefaultSeeder(Client, new IndexSettings
 		{
-			NumberOfShards = 2,
+			NumberOfShards = 1,
 			NumberOfReplicas = 1
-		}).SeedNode();
+		})
+        {
+            IncludeRemoteCluster = false
+        }.SeedNode();
 	}
 }

tests/Tests.Core/ManagedOpenSearch/NodeSeeders/DefaultSeeder.cs

@@ -66,6 +66,8 @@ public DefaultSeeder(IOpenSearchClient client) : this(client, null) { }
 
 		private IIndexSettings IndexSettings { get; }
 
+        public bool IncludeRemoteCluster { get; set; } = true;
+
 		public void SeedNode()
 		{
 			var alreadySeeded = false;
@@ -117,12 +119,9 @@ public async Task ClusterSettingsAsync()
 				{ "cluster.routing.use_adaptive_replica_selection", true }
 			};
 
-			clusterConfiguration += new RemoteClusterConfiguration
-			{
-				{ RemoteClusterName, "127.0.0.1:9300" }
-			};
+            if (IncludeRemoteCluster) clusterConfiguration += new RemoteClusterConfiguration { { RemoteClusterName, "127.0.0.1:9300" } };
 
-			var putSettingsResponse = await Client.Cluster.PutSettingsAsync(new ClusterPutSettingsRequest
+            var putSettingsResponse = await Client.Cluster.PutSettingsAsync(new ClusterPutSettingsRequest
 			{
 				Transient = clusterConfiguration
 			}).ConfigureAwait(false);

tests/Tests/Cat/CatSegmentReplication/CatSegmentReplicationApiTests.cs

@@ -7,6 +7,7 @@
 
 using System;
 using System.Linq;
+using System.Threading;
 using FluentAssertions;
 using OpenSearch.Client;
 using OpenSearch.Net;
@@ -45,9 +46,11 @@ protected override void ExpectResponse(CatResponse<CatSegmentReplicationRecord>
 		response.Records.Should().NotBeEmpty().And.AllSatisfy(r => r.ShardId.Should().StartWith($"[{IndexName}]"));
 
 	protected override void IntegrationSetup(IOpenSearchClient client, CallUniqueValues values)
-	{
+    {
 		var resp = client.Indices.Create(IndexName, d => d
 			.Settings(s => s
+                .NumberOfShards(1)
+                .NumberOfReplicas(1)
 				.Setting("index.replication.type", "SEGMENT")));
 		resp.ShouldBeValid();
 
@@ -56,9 +59,16 @@ protected override void IntegrationSetup(IOpenSearchClient client, CallUniqueVal
 			.IndexMany(Enumerable.Range(0, 10).Select(i => new Doc { Id = i }))
 			.Refresh(Refresh.WaitFor));
 		bulkResp.ShouldBeValid();
-	}
 
-	protected override void IntegrationTeardown(IOpenSearchClient client, CallUniqueValues values) => client.Indices.Delete(IndexName);
+        var healthResp = client.Cluster.Health(IndexName, d => d
+            .WaitForStatus(HealthStatus.Green)
+            .WaitForNodes("2")
+            .Timeout("30s")
+            .Level(ClusterHealthLevel.Shards)
+            .WaitForNoInitializingShards()
+            .WaitForNoRelocatingShards());
+        healthResp.ShouldBeValid();
+    }
 
 	public class Doc
 	{