Add support for Point In Time APIs (opensearch-project#405)

* Add `CreatePit` & `DeletePit` methods Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Add `DeleteAllPits` and `GetAllPits` methods Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Add PIT search support Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Add integration tests Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Add docs Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Add changelog entry Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Correct Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Fix changelog typo Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Add tests for CreatePit Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Add tests for DeletePit Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Fixing tests Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Cleanup Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Add tests for GetAllPits & DeleteAllPits Signed-off-by: Thomas Farr <tsfarr@amazon.com> * PitSearch tests Signed-off-by: Thomas Farr <tsfarr@amazon.com> * call isolated Signed-off-by: Thomas Farr <tsfarr@amazon.com> * writable cluster Signed-off-by: Thomas Farr <tsfarr@amazon.com> --------- Signed-off-by: Thomas Farr <tsfarr@amazon.com>
Xtansia · Nov 5, 2023 · 5111427 · 5111427
1 parent 94a287c
commit 5111427
Show file tree

Hide file tree

Showing 54 changed files with 1,981 additions and 73 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,9 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Removed support for the `net461` target ([#256](https://github.com/opensearch-project/opensearch-net/pull/256))
 - Fixed naming of `ClusterManagerTimeout` and `MasterTimeout` properties from `*TimeSpanout` in the low-level client ([#332](https://github.com/opensearch-project/opensearch-net/pull/332))
 
+### Added
+- Added support for point-in-time search and associated APIs ([#405](https://github.com/opensearch-project/opensearch-net/pull/405))
+
 ### Removed
 - Removed the `Features` API which is not supported by OpenSearch from the low-level client ([#331](https://github.com/opensearch-project/opensearch-net/pull/331))
 
@@ -107,4 +110,4 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 [Unreleased]: https://github.com/opensearch-project/opensearch-net/compare/v1.5.0...main
 [1.5.0]: https://github.com/opensearch-project/opensearch-net/compare/v1.4.0...v1.5.0
 [1.4.0]: https://github.com/opensearch-project/opensearch-net/compare/v1.3.0...v1.4.0
-[1.3.0]: https://github.com/opensearch-project/opensearch-net/compare/v1.2.0...v1.3.0
+[1.3.0]: https://github.com/opensearch-project/opensearch-net/compare/v1.2.0...v1.3.0
diff --git a/guides/search.md b/guides/search.md
@@ -141,8 +141,41 @@ var page3 = await client.ScrollAsync<Movie>("1m", page2.ScrollId);
 Console.WriteLine(string.Join('\n', page3.Documents));
 ```
 
+### Pagination with Point in Time
+The scroll example above has one weakness: if the index is updated while you are scrolling through the results, they will be paginated inconsistently. To avoid this, you should use the "Point in Time" feature. The following example demonstrates how to use the `point_in_time` and `pit_id` parameters to paginate through the search results:
+
+```csharp
+var pitResp = await client.CreatePitAsync("movies", p => p.KeepAlive("1m"));
+
+var page1 = await client.SearchAsync<Movie>(s => s
+    .Query(_ => query)
+    .Sort(_ => sort)
+    .Size(2)
+    .PointInTime(p => p.Id(pitResp.PitId).KeepAlive("1m")));
+var page2 = await client.SearchAsync<Movie>(s => s
+    .Query(_ => query)
+    .Sort(_ => sort)
+    .Size(2)
+    .PointInTime(p => p.Id(pitResp.PitId).KeepAlive("1m"))
+    .SearchAfter(page1.Hits.Last().Sorts));
+var page3 = await client.SearchAsync<Movie>(s => s
+    .Query(_ => query)
+    .Sort(_ => sort)
+    .Size(2)
+    .PointInTime(p => p.Id(pitResp.PitId).KeepAlive("1m"))
+    .SearchAfter(page2.Hits.Last().Sorts));
+
+foreach (var doc in page1.Documents.Concat(page2.Documents).Concat(page3.Documents))
+{
+    Console.WriteLine(doc.Title);
+}
+
+await client.DeletePitAsync(p => p.PitId(pitResp.PitId));
+```
+
+Note that a point-in-time is associated with an index or a set of index. So, when performing a search with a point-in-time, you DO NOT specify the index in the search.
 
 ## Cleanup
 ```csharp
 await client.Indices.DeleteAsync("movies");
-```
+```
diff --git a/src/ApiGenerator/Configuration/CodeConfiguration.cs b/src/ApiGenerator/Configuration/CodeConfiguration.cs
@@ -39,6 +39,9 @@ public static class CodeConfiguration
 	{
 		private static readonly Glob[] OperationsToInclude =
 		{
+			new("{create,delete}_pit"),
+			new("{delete,get}_all_pits"),
+
 			new("cluster.allocation_explain"),
 			new("cluster.delete_component_template"),
 			new("cluster.delete_voting_config_exclusions"),

diff --git a/src/ApiGenerator/OpenSearch.openapi.json b/src/ApiGenerator/OpenSearch.openapi.json
@@ -5119,16 +5119,6 @@
                             "type": "string",
                             "description": "The awareness attribute for which the health is required."
                         }
-                    },
-                    {
-                        "name": "ensure_node_commissioned",
-                        "in": "query",
-                        "description": "Checks whether local node is commissioned or not. If set to true on a local call it will throw exception if node is decommissioned.",
-                        "schema": {
-                            "type": "boolean",
-                            "default": false,
-                            "description": "Checks whether local node is commissioned or not. If set to true on a local call it will throw exception if node is decommissioned."
-                        }
                     }
                 ],
                 "responses": {
@@ -5280,16 +5270,6 @@
                             "type": "string",
                             "description": "The awareness attribute for which the health is required."
                         }
-                    },
-                    {
-                        "name": "ensure_node_commissioned",
-                        "in": "query",
-                        "description": "Checks whether local node is commissioned or not. If set to true on a local call it will throw exception if node is decommissioned.",
-                        "schema": {
-                            "type": "boolean",
-                            "default": false,
-                            "description": "Checks whether local node is commissioned or not. If set to true on a local call it will throw exception if node is decommissioned."
-                        }
                     }
                 ],
                 "responses": {
@@ -6831,6 +6811,18 @@
                             "deprecated": true
                         }
                     },
+                    {
+                        "name": "cluster_manager_timeout",
+                        "in": "query",
+                        "description": "Operation timeout for connection to cluster-manager node.",
+                        "schema": {
+                            "type": "string",
+                            "pattern": "^([0-9]+)(?:d|h|m|s|ms|micros|nanos)$",
+                            "description": "Operation timeout for connection to cluster-manager node.",
+                            "x-version-added": "2.0.0",
+                            "x-data-type": "time"
+                        }
+                    },
                     {
                         "name": "local",
                         "in": "query",
@@ -8536,6 +8528,18 @@
                             "deprecated": true
                         }
                     },
+                    {
+                        "name": "cluster_manager_timeout",
+                        "in": "query",
+                        "description": "Operation timeout for connection to cluster-manager node.",
+                        "schema": {
+                            "type": "string",
+                            "pattern": "^([0-9]+)(?:d|h|m|s|ms|micros|nanos)$",
+                            "description": "Operation timeout for connection to cluster-manager node.",
+                            "x-version-added": "2.0.0",
+                            "x-data-type": "time"
+                        }
+                    },
                     {
                         "name": "local",
                         "in": "query",
@@ -18594,6 +18598,18 @@
                             "deprecated": true
                         }
                     },
+                    {
+                        "name": "cluster_manager_timeout",
+                        "in": "query",
+                        "description": "Operation timeout for connection to cluster-manager node.",
+                        "schema": {
+                            "type": "string",
+                            "pattern": "^([0-9]+)(?:d|h|m|s|ms|micros|nanos)$",
+                            "description": "Operation timeout for connection to cluster-manager node.",
+                            "x-version-added": "2.0.0",
+                            "x-data-type": "time"
+                        }
+                    },
                     {
                         "name": "local",
                         "in": "query",
@@ -19451,6 +19467,18 @@
                             "description": "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)."
                         }
                     },
+                    {
+                        "name": "cluster_manager_timeout",
+                        "in": "query",
+                        "description": "Operation timeout for connection to cluster-manager node.",
+                        "schema": {
+                            "type": "string",
+                            "pattern": "^([0-9]+)(?:d|h|m|s|ms|micros|nanos)$",
+                            "description": "Operation timeout for connection to cluster-manager node.",
+                            "x-version-added": "2.0.0",
+                            "x-data-type": "time"
+                        }
+                    },
                     {
                         "name": "expand_wildcards",
                         "in": "query",
@@ -25302,6 +25330,18 @@
                             "$ref": "#/components/schemas/ExpandWildcards"
                         }
                     },
+                    {
+                        "name": "cluster_manager_timeout",
+                        "in": "query",
+                        "description": "Operation timeout for connection to cluster-manager node.",
+                        "schema": {
+                            "type": "string",
+                            "pattern": "^([0-9]+)(?:d|h|m|s|ms|micros|nanos)$",
+                            "description": "Operation timeout for connection to cluster-manager node.",
+                            "x-version-added": "2.0.0",
+                            "x-data-type": "time"
+                        }
+                    },
                     {
                         "name": "wait_for_active_shards",
                         "in": "query",
@@ -26631,7 +26671,8 @@
                         "description": "Specify the keep alive for point in time.",
                         "schema": {
                             "type": "string",
-                            "description": "Specify the keep alive for point in time."
+                            "description": "Specify the keep alive for point in time.",
+                          "x-data-type": "time"
                         }
                     },
                     {
@@ -30868,6 +30909,7 @@
             },
             "DeletePit_BodyParams": {
                 "type": "object",
+                "description": "The point-in-time ids to be deleted",
                 "properties": {
                     "pit_id": {
                         "type": "array",
@@ -31170,6 +31212,9 @@
                             "type": "string"
                         }
                     },
+                    "dls": {
+                        "type": "string"
+                    },
                     "fls": {
                         "type": "array",
                         "items": {
@@ -31885,19 +31930,22 @@
                     "description": {
                         "type": "string"
                     },
-                    "cluster_permission": {
+                    "cluster_permissions": {
                         "type": "array",
                         "items": {
                             "type": "string"
                         }
                     },
-                    "index_permission": {
-                        "$ref": "#/components/schemas/IndexPermission"
+                    "index_permissions": {
+                        "type": "array",
+                        "items": {
+                            "$ref": "#/components/schemas/IndexPermission"
+                        }
                     },
                     "tenant_permissions": {
                         "type": "array",
                         "items": {
-                            "type": "string"
+                            "$ref": "#/components/schemas/TenantPermission"
                         }
                     },
                     "static": {
@@ -32220,6 +32268,23 @@
                     }
                 }
             },
+            "TenantPermission": {
+                "type": "object",
+                "properties": {
+                    "tenant_patterns": {
+                        "type": "array",
+                        "items": {
+                            "type": "string"
+                        }
+                    },
+                    "allowed_actions": {
+                        "type": "array",
+                        "items": {
+                            "type": "string"
+                        }
+                    }
+                }
+            },
             "TenantsMap": {
                 "type": "object",
                 "additionalProperties": {

diff --git a/src/ApiGenerator/Views/HighLevel/Descriptors/Descriptors.cshtml b/src/ApiGenerator/Views/HighLevel/Descriptors/Descriptors.cshtml
@@ -17,7 +17,7 @@ using System.Linq.Expressions;
 
 using OpenSearch.Net;
 using OpenSearch.Net.Utf8Json;
-@if (ns != CsharpNames.RootNamespace)
+@if (ns != null)
 {
     <text>using OpenSearch.Net@(ns);
 </text>

diff --git a/src/ApiGenerator/Views/HighLevel/Requests/Requests.cshtml b/src/ApiGenerator/Views/HighLevel/Requests/Requests.cshtml
@@ -17,7 +17,7 @@ using System.Linq.Expressions;
 using System.Runtime.Serialization;
 using OpenSearch.Net;
 using OpenSearch.Net.Utf8Json;
-@if (ns != CsharpNames.RootNamespace)
+@if (ns != null)
 {
     <text>using OpenSearch.Net@(ns);
 </text>

diff --git a/src/OpenSearch.Client/Aggregations/Bucket/Composite/CompositeAggregationSource.cs b/src/OpenSearch.Client/Aggregations/Bucket/Composite/CompositeAggregationSource.cs
@@ -30,6 +30,7 @@
 using System.Collections.Generic;
 using System.Linq.Expressions;
 using System.Runtime.Serialization;
+using OpenSearch.Net.Extensions;
 using OpenSearch.Net.Utf8Json;
 using OpenSearch.Net.Utf8Json.Internal;
 using OpenSearch.Net.Utf8Json.Resolvers;

diff --git a/src/OpenSearch.Client/Cat/CatHelpResponseBuilder.cs b/src/OpenSearch.Client/Cat/CatHelpResponseBuilder.cs
@@ -31,6 +31,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using OpenSearch.Net;
+using OpenSearch.Net.Extensions;
 
 namespace OpenSearch.Client.Specification.CatApi
 {

diff --git a/src/OpenSearch.Client/CommonAbstractions/Extensions/Extensions.cs b/src/OpenSearch.Client/CommonAbstractions/Extensions/Extensions.cs
@@ -114,13 +114,6 @@ internal static string ToEnumValue<T>(this T enumValue) where T : struct
 			return null;
 		}
 
-		internal static string Utf8String(this ref ArraySegment<byte> segment) =>
-			StringEncoding.UTF8.GetString(segment.Array, segment.Offset, segment.Count);
-
-		internal static string Utf8String(this byte[] bytes) => bytes == null ? null : Encoding.UTF8.GetString(bytes, 0, bytes.Length);
-
-		internal static byte[] Utf8Bytes(this string s) => s.IsNullOrEmpty() ? null : Encoding.UTF8.GetBytes(s);
-
 		internal static bool IsNullOrEmpty(this IndexName value) => value == null || value.GetHashCode() == 0;
 
 		internal static bool IsNullable(this Type type) =>

diff --git a/src/OpenSearch.Client/CommonOptions/Sorting/SortFormatter.cs b/src/OpenSearch.Client/CommonOptions/Sorting/SortFormatter.cs
@@ -27,6 +27,7 @@
 */
 
 using System.Collections.Generic;
+using OpenSearch.Net.Extensions;
 using OpenSearch.Net.Utf8Json;
 using OpenSearch.Net.Utf8Json.Internal;
 using OpenSearch.Net.Utf8Json.Resolvers;

diff --git a/src/OpenSearch.Client/Indices/MappingManagement/GetFieldMapping/FieldMappingFormatter.cs b/src/OpenSearch.Client/Indices/MappingManagement/GetFieldMapping/FieldMappingFormatter.cs
@@ -27,6 +27,7 @@
 */
 
 using System.Collections.Generic;
+using OpenSearch.Net.Extensions;
 using OpenSearch.Net.Utf8Json;
 using OpenSearch.Net.Utf8Json.Internal;
 using OpenSearch.Net.Utf8Json.Resolvers;

diff --git a/src/OpenSearch.Client/OpenSearch.Client.csproj b/src/OpenSearch.Client/OpenSearch.Client.csproj
@@ -19,6 +19,7 @@
     <ProjectReference Include="..\OpenSearch.Net\OpenSearch.Net.csproj" />
   </ItemGroup>
   <ItemGroup>
+    <InternalsVisibleTo Include="Tests" Key="$(ExposedPublicKey)" />
     <InternalsVisibleTo Include="OpenSearch.Net.CustomDynamicObjectResolver" Key="$(ExposedPublicKey)" />
     <InternalsVisibleTo Include="OpenSearch.Net.DynamicCompositeResolver" Key="$(ExposedPublicKey)" />
     <InternalsVisibleTo Include="OpenSearch.Net.DynamicObjectResolverAllowPrivateFalseExcludeNullFalseNameMutateOriginal" Key="$(ExposedPublicKey)" />

diff --git a/...OpenSearch.Client/QueryDsl/Compound/FunctionScore/Functions/ScoreFunctionJsonFormatter.cs b/...OpenSearch.Client/QueryDsl/Compound/FunctionScore/Functions/ScoreFunctionJsonFormatter.cs
@@ -28,6 +28,7 @@
 
 using System;
 using OpenSearch.Net;
+using OpenSearch.Net.Extensions;
 using OpenSearch.Net.Utf8Json;
 using OpenSearch.Net.Utf8Json.Internal;
 

diff --git a/src/OpenSearch.Client/QueryDsl/Geo/BoundingBox/GeoBoundingBoxQuery.cs b/src/OpenSearch.Client/QueryDsl/Geo/BoundingBox/GeoBoundingBoxQuery.cs
@@ -27,6 +27,7 @@
 */
 
 using System;
+using OpenSearch.Net.Extensions;
 using OpenSearch.Net.Utf8Json;
 using OpenSearch.Net.Utf8Json.Internal;
 

diff --git a/src/OpenSearch.Client/QueryDsl/Geo/Distance/GeoDistanceQuery.cs b/src/OpenSearch.Client/QueryDsl/Geo/Distance/GeoDistanceQuery.cs
@@ -26,6 +26,7 @@
 *  under the License.
 */
 
+using OpenSearch.Net.Extensions;
 using OpenSearch.Net.Utf8Json;
 using OpenSearch.Net.Utf8Json.Internal;
 

diff --git a/src/OpenSearch.Client/QueryDsl/Geo/Polygon/GeoPolygonQuery.cs b/src/OpenSearch.Client/QueryDsl/Geo/Polygon/GeoPolygonQuery.cs
@@ -27,6 +27,7 @@
 */
 
 using System.Collections.Generic;
+using OpenSearch.Net.Extensions;
 using OpenSearch.Net.Utf8Json;
 using OpenSearch.Net.Utf8Json.Internal;
 

diff --git a/src/OpenSearch.Client/QueryDsl/Geo/Shape/GeoShapeQueryFormatter.cs b/src/OpenSearch.Client/QueryDsl/Geo/Shape/GeoShapeQueryFormatter.cs
@@ -27,6 +27,7 @@
 */
 
 using System;
+using OpenSearch.Net.Extensions;
 using OpenSearch.Net.Utf8Json;
 using OpenSearch.Net.Utf8Json.Internal;
 

diff --git a/src/OpenSearch.Client/QueryDsl/Specialized/Shape/CartesianPoint.cs b/src/OpenSearch.Client/QueryDsl/Specialized/Shape/CartesianPoint.cs
@@ -32,6 +32,7 @@
 using OpenSearch.Net.Utf8Json;
 using OpenSearch.Net.Utf8Json.Internal;
 using OpenSearch.Client;
+using OpenSearch.Net.Extensions;
 
 namespace OpenSearch.Client
 {