Support the neural query type and text_embedding ingest processor…

… type (opensearch-project#636)

* Implement `text_embedding` ingest pipeline processor

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Implement `neural` search query type

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Add changelog entry and fix license headers

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Improve test

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

* Separate neural query cluster configuration

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

---------

Signed-off-by: Thomas Farr <tsfarr@amazon.com>
(cherry picked from commit 9e7b819)

.github/workflows/integration.yml

@@ -56,6 +56,11 @@ jobs:
         env:
           VERSION: ${{ matrix.version }}
 
+      # Run neural query integration tests separately as they use a significant amount of memory on their own
+      - run: "./build.sh integrate ${{ matrix.version }} neuralquery random:test_only_one --report"
+        name: Neural Query Integration Tests
+        working-directory: client
+
       - name: Upload test report
         if: failure()
         uses: actions/upload-artifact@v3

CHANGELOG.md

@@ -3,6 +3,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased]
 ### Added
+- Added support for the `neural` query type and `text_embedding` ingest processor type ([#636](https://github.com/opensearch-project/opensearch-net/pull/636))
 - Added support for the `Cat.PitSegments` and `Cat.SegmentReplication` APIs ([#527](https://github.com/opensearch-project/opensearch-net/pull/527))
 - Added support for `MinScore` on `ScriptScoreQuery` ([#624](https://github.com/opensearch-project/opensearch-net/pull/624))
 - Added support for serializing the `DateOnly` and `TimeOnly` types ([#734](https://github.com/opensearch-project/opensearch-net/pull/734))

abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterConfiguration.cs

@@ -75,7 +75,7 @@ public EphemeralClusterConfiguration(OpenSearchVersion version, ClusterFeatures
     ///     This can be useful to fail early when subsequent operations are relying on installation
     ///     succeeding.
     /// </summary>
-    public bool ValidatePluginsToInstall { get; } = true;
+    public bool ValidatePluginsToInstall { get; set; } = true;
 
     public bool EnableSsl => Features.HasFlag(ClusterFeatures.SSL);
 

abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs

@@ -61,38 +61,37 @@ public override void Run(IEphemeralCluster<EphemeralClusterConfiguration> cluste
 					.Where(p => !p.IsValid(v))
 					.Select(p => p.SubProductName).ToList();
 				if (invalidPlugins.Any())
-					throw new OpenSearchCleanExitException(
-						$"Can not install the following plugins for version {v}: {string.Join(", ", invalidPlugins)} ");
-			}
+                {
+                    throw new OpenSearchCleanExitException(
+                        $"Can not install the following plugins for version {v}: {string.Join(", ", invalidPlugins)} ");
+                }
+            }
 
 			foreach (var plugin in requiredPlugins)
 			{
-				var includedByDefault = plugin.IsIncludedOutOfTheBox(v);
-				if (includedByDefault)
+                if (plugin.IsIncludedOutOfTheBox(v))
 				{
 					cluster.Writer?.WriteDiagnostic(
 						$"{{{nameof(InstallPlugins)}}} SKIP plugin [{plugin.SubProductName}] shipped OOTB as of: {{{plugin.ShippedByDefaultAsOf}}}");
 					continue;
 				}
 
-				var validForCurrentVersion = plugin.IsValid(v);
-				if (!validForCurrentVersion)
+                if (!plugin.IsValid(v))
 				{
 					cluster.Writer?.WriteDiagnostic(
 						$"{{{nameof(InstallPlugins)}}} SKIP plugin [{plugin.SubProductName}] not valid for version: {{{v}}}");
 					continue;
 				}
 
-				var alreadyInstalled = AlreadyInstalled(fs, plugin.SubProductName);
-				if (alreadyInstalled)
+                if (AlreadyInstalled(fs, plugin.SubProductName))
 				{
 					cluster.Writer?.WriteDiagnostic(
 						$"{{{nameof(InstallPlugins)}}} SKIP plugin [{plugin.SubProductName}] already installed");
 					continue;
 				}
 
 				cluster.Writer?.WriteDiagnostic(
-					$"{{{nameof(InstallPlugins)}}} attempting install [{plugin.SubProductName}] as it's not OOTB: {{{plugin.ShippedByDefaultAsOf}}} and valid for {v}: {{{plugin.IsValid(v)}}}");
+					$"{{{nameof(InstallPlugins)}}} attempting install [{plugin.SubProductName}] as it's not OOTB: {{{plugin.ShippedByDefaultAsOf}}} and valid for {v}");
 
 				var homeConfigPath = Path.Combine(fs.OpenSearchHome, "config");
 

abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IntegrationTestDiscoverer.cs

@@ -35,102 +35,109 @@
 using Xunit;
 using Xunit.Abstractions;
 using Xunit.Sdk;
-using Enumerable = System.Linq.Enumerable;
 
-namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing
+namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing;
+
+/// <summary>
+/// A Xunit test that should be skipped, and a reason why.
+/// </summary>
+public abstract class SkipTestAttributeBase : Attribute
 {
-	/// <summary>
-	///     An Xunit test that should be skipped, and a reason why.
-	/// </summary>
-	public abstract class SkipTestAttributeBase : Attribute
-	{
-		/// <summary>
-		///     Whether the test should be skipped
-		/// </summary>
-		public abstract bool Skip { get; }
+    /// <summary>
+    ///     Whether the test should be skipped
+    /// </summary>
+    public abstract bool Skip { get; }
+
+    /// <summary>
+    ///     The reason why the test should be skipped
+    /// </summary>
+    public abstract string Reason { get; }
+}
 
-		/// <summary>
-		///     The reason why the test should be skipped
-		/// </summary>
-		public abstract string Reason { get; }
-	}
+/// <summary>
+///     An Xunit integration test
+/// </summary>
+[XunitTestCaseDiscoverer("OpenSearch.OpenSearch.Xunit.XunitPlumbing.IntegrationTestDiscoverer",
+    "OpenSearch.OpenSearch.Xunit")]
+public class I : FactAttribute
+{
+}
 
-	/// <summary>
-	///     An Xunit integration test
-	/// </summary>
-	[XunitTestCaseDiscoverer("OpenSearch.OpenSearch.Xunit.XunitPlumbing.IntegrationTestDiscoverer",
-		"OpenSearch.OpenSearch.Xunit")]
-	public class I : FactAttribute
-	{
-	}
+/// <summary>
+///     A test discoverer used to discover integration tests cases attached
+///     to test methods that are attributed with <see cref="I" /> attribute
+/// </summary>
+public class IntegrationTestDiscoverer : OpenSearchTestCaseDiscoverer
+{
+    public IntegrationTestDiscoverer(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink)
+    {
+    }
 
-	/// <summary>
-	///     A test discoverer used to discover integration tests cases attached
-	///     to test methods that are attributed with <see cref="I" /> attribute
-	/// </summary>
-	public class IntegrationTestDiscoverer : OpenSearchTestCaseDiscoverer
-	{
-		public IntegrationTestDiscoverer(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink)
-		{
-		}
+    /// <inheritdoc />
+    protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod,
+        out string skipReason)
+    {
+        skipReason = null;
+        var runIntegrationTests =
+            discoveryOptions.GetValue<bool>(nameof(OpenSearchXunitRunOptions.RunIntegrationTests));
+        if (!runIntegrationTests) return true;
 
-		/// <inheritdoc />
-		protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod,
-			out string skipReason)
-		{
-			skipReason = null;
-			var runIntegrationTests =
-				discoveryOptions.GetValue<bool>(nameof(OpenSearchXunitRunOptions.RunIntegrationTests));
-			if (!runIntegrationTests) return true;
+        var cluster = TestAssemblyRunner.GetClusterForClass(testMethod.TestClass.Class);
+        if (cluster == null)
+        {
+            skipReason +=
+                $"{testMethod.TestClass.Class.Name} does not define a cluster through IClusterFixture or {nameof(IntegrationTestClusterAttribute)}";
+            return true;
+        }
 
-			var cluster = TestAssemblyRunner.GetClusterForClass(testMethod.TestClass.Class);
-			if (cluster == null)
-			{
-				skipReason +=
-					$"{testMethod.TestClass.Class.Name} does not define a cluster through IClusterFixture or {nameof(IntegrationTestClusterAttribute)}";
-				return true;
-			}
+        var openSearchVersion =
+            discoveryOptions.GetValue<OpenSearchVersion>(nameof(OpenSearchXunitRunOptions.Version));
 
-			var openSearchVersion =
-				discoveryOptions.GetValue<OpenSearchVersion>(nameof(OpenSearchXunitRunOptions.Version));
+        // Skip if the version we are testing against is attributed to be skipped do not run the test nameof(SkipVersionAttribute.Ranges)
+        var skipVersionAttribute = GetAttributes<SkipVersionAttribute>(testMethod).FirstOrDefault();
+        if (skipVersionAttribute != null)
+        {
+            var skipVersionRanges =
+                skipVersionAttribute.GetNamedArgument<IList<Range>>(nameof(SkipVersionAttribute.Ranges)) ??
+                new List<Range>();
+            if (openSearchVersion == null && skipVersionRanges.Count > 0)
+            {
+                skipReason = $"{nameof(SkipVersionAttribute)} has ranges defined for this test but " +
+                    $"no {nameof(OpenSearchXunitRunOptions.Version)} has been provided to {nameof(OpenSearchXunitRunOptions)}";
+                return true;
+            }
 
-			// Skip if the version we are testing against is attributed to be skipped do not run the test nameof(SkipVersionAttribute.Ranges)
-			var skipVersionAttribute = Enumerable.FirstOrDefault(GetAttributes<SkipVersionAttribute>(testMethod));
-			if (skipVersionAttribute != null)
-			{
-				var skipVersionRanges =
-					skipVersionAttribute.GetNamedArgument<IList<Range>>(nameof(SkipVersionAttribute.Ranges)) ??
-					new List<Range>();
-				if (openSearchVersion == null && skipVersionRanges.Count > 0)
-				{
-					skipReason = $"{nameof(SkipVersionAttribute)} has ranges defined for this test but " +
-					             $"no {nameof(OpenSearchXunitRunOptions.Version)} has been provided to {nameof(OpenSearchXunitRunOptions)}";
-					return true;
-				}
+            if (openSearchVersion != null)
+            {
+                var reason = skipVersionAttribute.GetNamedArgument<string>(nameof(SkipVersionAttribute.Reason));
+                foreach (var range in skipVersionRanges)
+                {
+                    // inrange takes prereleases into account
+                    if (!openSearchVersion.InRange(range)) continue;
+                    skipReason =
+                        $"{nameof(SkipVersionAttribute)} has range {range} that {openSearchVersion} satisfies";
+                    if (!string.IsNullOrWhiteSpace(reason)) skipReason += $": {reason}";
+                    return true;
+                }
+            }
+        }
 
-				if (openSearchVersion != null)
-				{
-					var reason = skipVersionAttribute.GetNamedArgument<string>(nameof(SkipVersionAttribute.Reason));
-					for (var index = 0; index < skipVersionRanges.Count; index++)
-					{
-						var range = skipVersionRanges[index];
-						// inrange takes prereleases into account
-						if (!openSearchVersion.InRange(range)) continue;
-						skipReason =
-							$"{nameof(SkipVersionAttribute)} has range {range} that {openSearchVersion} satisfies";
-						if (!string.IsNullOrWhiteSpace(reason)) skipReason += $": {reason}";
-						return true;
-					}
-				}
-			}
+        // Skip if a prerelease version and has SkipPrereleaseVersionsAttribute
+        var skipPrerelease = GetAttributes<SkipPrereleaseVersionsAttribute>(testMethod).FirstOrDefault();
+        if (openSearchVersion != null && openSearchVersion.IsPreRelease && skipPrerelease != null)
+        {
+            skipReason = $"{nameof(SkipPrereleaseVersionsAttribute)} has been applied to this test";
+            var reason = skipPrerelease.GetNamedArgument<string>(nameof(SkipVersionAttribute.Reason));
+            if (!string.IsNullOrWhiteSpace(reason)) skipReason += $": {reason}";
+            return true;
+        }
 
-			var skipTests = GetAttributes<SkipTestAttributeBase>(testMethod)
-				.FirstOrDefault(a => a.GetNamedArgument<bool>(nameof(SkipTestAttributeBase.Skip)));
+        var skipTests = GetAttributes<SkipTestAttributeBase>(testMethod)
+            .FirstOrDefault(a => a.GetNamedArgument<bool>(nameof(SkipTestAttributeBase.Skip)));
 
-			if (skipTests == null) return false;
+        if (skipTests == null) return false;
 
-			skipReason = skipTests.GetNamedArgument<string>(nameof(SkipTestAttributeBase.Reason));
-			return true;
-		}
-	}
+        skipReason = skipTests.GetNamedArgument<string>(nameof(SkipTestAttributeBase.Reason));
+        return true;
+    }
 }

abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/SkipPrereleaseVersionsAttribute.cs

@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+using System;
+
+namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing;
+
+/// <summary>
+/// A Xunit test that should be skipped for prerelease OpenSearch versions, and a reason why.
+/// </summary>
+public class SkipPrereleaseVersionsAttribute : Attribute
+{
+    public SkipPrereleaseVersionsAttribute(string reason) => Reason = reason;
+
+    /// <summary>
+    /// The reason why the test should be skipped
+    /// </summary>
+    public string Reason { get; }
+}

abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/SkipVersionAttribute.cs

@@ -31,35 +31,34 @@
 using System.Linq;
 using SemanticVersioning;
 
-namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing
+namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing;
+
+/// <summary>
+/// A Xunit test that should be skipped for given OpenSearch versions, and a reason why.
+/// </summary>
+public class SkipVersionAttribute : Attribute
 {
-	/// <summary>
-	///     An Xunit test that should be skipped for given OpenSearch versions, and a reason why.
-	/// </summary>
-	public class SkipVersionAttribute : Attribute
-	{
-		// ReSharper disable once UnusedParameter.Local
-		// reason is used to allow the test its used on to self document why its been put in place
-		public SkipVersionAttribute(string skipVersionRangesSeparatedByComma, string reason)
-		{
-			Reason = reason;
-			Ranges = string.IsNullOrEmpty(skipVersionRangesSeparatedByComma)
-				? new List<Range>()
-				: skipVersionRangesSeparatedByComma.Split(',')
-					.Select(r => r.Trim())
-					.Where(r => !string.IsNullOrWhiteSpace(r))
-					.Select(r => new Range(r))
-					.ToList();
-		}
+    // ReSharper disable once UnusedParameter.Local
+    // reason is used to allow the test its used on to self document why its been put in place
+    public SkipVersionAttribute(string skipVersionRangesSeparatedByComma, string reason)
+    {
+        Reason = reason;
+        Ranges = string.IsNullOrEmpty(skipVersionRangesSeparatedByComma)
+            ? new List<Range>()
+            : skipVersionRangesSeparatedByComma.Split(',')
+                .Select(r => r.Trim())
+                .Where(r => !string.IsNullOrWhiteSpace(r))
+                .Select(r => new Range(r))
+                .ToList();
+    }
 
-		/// <summary>
-		///     The reason why the test should be skipped
-		/// </summary>
-		public string Reason { get; }
+    /// <summary>
+    ///     The reason why the test should be skipped
+    /// </summary>
+    public string Reason { get; }
 
-		/// <summary>
-		///     The version ranges for which the test should be skipped
-		/// </summary>
-		public IList<Range> Ranges { get; }
-	}
+    /// <summary>
+    ///     The version ranges for which the test should be skipped
+    /// </summary>
+    public IList<Range> Ranges { get; }
 }

abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/OpenSearchPlugin.cs

@@ -27,6 +27,7 @@
 */
 
 using System;
+using Version = SemanticVersioning.Version;
 
 namespace OpenSearch.Stack.ArtifactsApi.Products
 {
@@ -81,5 +82,9 @@ public OpenSearchPlugin(string plugin, Func<OpenSearchVersion, bool> isValid = n
 		public static OpenSearchPlugin DeleteByQuery { get; } = new("delete-by-query", version => version < "1.0.0");
 
 		public static OpenSearchPlugin Knn { get; } = new("opensearch-knn");
-	}
+
+        public static OpenSearchPlugin MachineLearning { get; } = new("opensearch-ml", v => v.BaseVersion() >= new Version("1.3.0") && !v.IsPreRelease);
+
+        public static OpenSearchPlugin NeuralSearch { get; } = new("opensearch-neural-search", v => v.BaseVersion() >= new Version("2.4.0") && !v.IsPreRelease);
+    }
 }