Update by query API (#3597)

(cherry picked from commit 04f09fb)

src/CodeGeneration/ApiGenerator/CodeConfiguration.cs

@@ -38,7 +38,6 @@ public static class CodeConfiguration
 			"ml.find_file_structure.json",
 			"monitoring.bulk.json",
 			"delete_by_query_rethrottle.json",
-			"update_by_query_rethrottle.json",
 
 			"security.delete_privileges.json",
 			"security.get_privileges.json",

src/Elasticsearch.Net/Domain/RequestParameters/RequestParameters.Generated.cs

@@ -2213,6 +2213,13 @@ public class UpdateByQueryRequestParameters : RequestParameters<UpdateByQueryReq
 		///<summary>The number of slices this task should be divided into. Defaults to 1 meaning the task isn't sliced into subtasks.</summary>
 		public long? Slices { get => Q<long?>("slices"); set => Q("slices", value); }
 	}
+	///<summary>Request options for UpdateByQueryRethrottle<pre>https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update-by-query.html</pre></summary>
+	public class UpdateByQueryRethrottleRequestParameters : RequestParameters<UpdateByQueryRethrottleRequestParameters> 
+	{
+		public override HttpMethod DefaultHttpMethod => HttpMethod.POST;
+		///<summary>The throttle to set on this request in floating sub-requests per second. -1 means set no throttle.</summary>
+		public long? RequestsPerSecond { get => Q<long?>("requests_per_second"); set => Q("requests_per_second", value); }
+	}
 	///<summary>Request options for CcrDeleteAutoFollowPattern<pre>https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-delete-auto-follow-pattern.html</pre></summary>
 	public class DeleteAutoFollowPatternRequestParameters : RequestParameters<DeleteAutoFollowPatternRequestParameters> 
 	{

src/Elasticsearch.Net/ElasticLowLevelClient.Generated.cs

@@ -2532,6 +2532,16 @@ public TResponse UpdateByQuery<TResponse>(string index, PostData body, UpdateByQ
 		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
 		public Task<TResponse> UpdateByQueryAsync<TResponse>(string index, PostData body, UpdateByQueryRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
 			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync<TResponse>(POST, Url($"{index.NotNull("index")}/_update_by_query"), ctx, body, _params(requestParameters));
+		///<summary>POST on /_update_by_query/{task_id}/_rethrottle <para>https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update-by-query.html</para></summary>
+		///<param name="task_id">The task id to rethrottle</param>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		public TResponse UpdateByQueryRethrottle<TResponse>(string task_id, UpdateByQueryRethrottleRequestParameters requestParameters = null)
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequest<TResponse>(POST, Url($"_update_by_query/{task_id.NotNull("task_id")}/_rethrottle"), null, _params(requestParameters));
+		///<summary>POST on /_update_by_query/{task_id}/_rethrottle <para>https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update-by-query.html</para></summary>
+		///<param name="task_id">The task id to rethrottle</param>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		public Task<TResponse> UpdateByQueryRethrottleAsync<TResponse>(string task_id, UpdateByQueryRethrottleRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync<TResponse>(POST, Url($"_update_by_query/{task_id.NotNull("task_id")}/_rethrottle"), ctx, null, _params(requestParameters));
 		///<summary>DELETE on /_ccr/auto_follow/{name} <para>https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-delete-auto-follow-pattern.html</para></summary>
 		///<param name="name">The name of the auto follow pattern.</param>
 		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>

src/Elasticsearch.Net/IElasticLowLevelClient.Generated.cs

@@ -2034,6 +2034,14 @@ public partial interface IElasticLowLevelClient
 		///<param name="body">The search definition using the Query DSL</param>
 		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
 		Task<TResponse> UpdateByQueryAsync<TResponse>(string index, PostData body, UpdateByQueryRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
+		///<summary>POST on /_update_by_query/{task_id}/_rethrottle <para>https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update-by-query.html</para></summary>
+		///<param name="task_id">The task id to rethrottle</param>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		TResponse UpdateByQueryRethrottle<TResponse>(string task_id, UpdateByQueryRethrottleRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
+		///<summary>POST on /_update_by_query/{task_id}/_rethrottle <para>https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update-by-query.html</para></summary>
+		///<param name="task_id">The task id to rethrottle</param>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		Task<TResponse> UpdateByQueryRethrottleAsync<TResponse>(string task_id, UpdateByQueryRethrottleRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
 		///<summary>DELETE on /_ccr/auto_follow/{name} <para>https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-delete-auto-follow-pattern.html</para></summary>
 		///<param name="name">The name of the auto follow pattern.</param>
 		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>

src/Nest/Cluster/TaskManagement/ListTasks/ListTasksResponse.cs

@@ -24,6 +24,10 @@ public class ListTasksResponse : ResponseBase, IListTasksResponse
 
 	public class TaskExecutingNode
 	{
+		[DataMember(Name = "attributes")]
+		[JsonFormatter(typeof(VerbatimInterfaceReadOnlyDictionaryKeysFormatter<string, string>))]
+		public IReadOnlyDictionary<string, string> Attributes { get; internal set; } = EmptyReadOnly<string, string>.Dictionary;
+
 		[DataMember(Name = "host")]
 		public string Host { get; internal set; }
 
@@ -33,6 +37,9 @@ public class TaskExecutingNode
 		[DataMember(Name = "name")]
 		public string Name { get; internal set; }
 
+		[DataMember(Name = "roles")]
+		public IEnumerable<string> Roles { get; internal set; }
+
 		[DataMember(Name = "tasks")]
 		public IReadOnlyDictionary<TaskId, TaskState> Tasks { get; internal set; } = EmptyReadOnly<TaskId, TaskState>.Dictionary;
 

src/Nest/Document/Multiple/UpdateByQueryRethrottle/ElasticClient-UpdateByQueryRethrottle.cs

@@ -0,0 +1,62 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Elasticsearch.Net;
+
+namespace Nest
+{
+	public partial interface IElasticClient
+	{
+		/// <summary>
+		/// Rethrottles a running update by query. Rethrottling that speeds up the query takes effect immediately
+		/// but rethrotting that slows down the query will take effect after completing the current batch. This prevents scroll timeouts.
+		/// <para> </para>
+		/// <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update-by-query.html#docs-update-by-query-rethrottle">https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update-by-query.html#docs-update-by-query-rethrottle</a>
+		/// </summary>
+		IListTasksResponse UpdateByQueryRethrottle(TaskId taskId, Func<UpdateByQueryRethrottleDescriptor, IUpdateByQueryRethrottleRequest> selector = null);
+
+ 		/// <inheritdoc cref="UpdateByQueryRethrottle(Nest.TaskId,System.Func{Nest.UpdateByQueryRethrottleDescriptor,Nest.IUpdateByQueryRethrottleRequest})" />
+		IListTasksResponse UpdateByQueryRethrottle(IUpdateByQueryRethrottleRequest request);
+
+ 		/// <inheritdoc cref="UpdateByQueryRethrottle(Nest.TaskId,System.Func{Nest.UpdateByQueryRethrottleDescriptor,Nest.IUpdateByQueryRethrottleRequest})" />
+		Task<IListTasksResponse> UpdateByQueryRethrottleAsync(TaskId taskId,
+			Func<UpdateByQueryRethrottleDescriptor, IUpdateByQueryRethrottleRequest> selector = null,
+			CancellationToken cancellationToken = default(CancellationToken)
+		);
+
+ 		/// <inheritdoc cref="UpdateByQueryRethrottle(Nest.TaskId,System.Func{Nest.UpdateByQueryRethrottleDescriptor,Nest.IUpdateByQueryRethrottleRequest})" />
+		Task<IListTasksResponse> UpdateByQueryRethrottleAsync(IUpdateByQueryRethrottleRequest request,
+			CancellationToken cancellationToken = default(CancellationToken)
+		);
+	}
+
+ 	public partial class ElasticClient
+	{
+		/// <inheritdoc />
+		public IListTasksResponse UpdateByQueryRethrottle(TaskId taskId, Func<UpdateByQueryRethrottleDescriptor, IUpdateByQueryRethrottleRequest> selector = null) =>
+			UpdateByQueryRethrottle(selector.InvokeOrDefault(new UpdateByQueryRethrottleDescriptor(taskId)));
+
+ 		/// <inheritdoc />
+		public IListTasksResponse UpdateByQueryRethrottle(IUpdateByQueryRethrottleRequest request) =>
+			Dispatcher.Dispatch<IUpdateByQueryRethrottleRequest, UpdateByQueryRethrottleRequestParameters, ListTasksResponse>(
+				request,
+				(p, d) => LowLevelDispatch.UpdateByQueryRethrottleDispatch<ListTasksResponse>(p)
+			);
+
+ 		/// <inheritdoc />
+		public Task<IListTasksResponse> UpdateByQueryRethrottleAsync(TaskId taskId, Func<UpdateByQueryRethrottleDescriptor, IUpdateByQueryRethrottleRequest> selector = null,
+			CancellationToken cancellationToken = default(CancellationToken)
+		) =>
+			UpdateByQueryRethrottleAsync(selector.InvokeOrDefault(new UpdateByQueryRethrottleDescriptor(taskId)), cancellationToken);
+
+ 		/// <inheritdoc />
+		public Task<IListTasksResponse> UpdateByQueryRethrottleAsync(IUpdateByQueryRethrottleRequest request,
+			CancellationToken cancellationToken = default(CancellationToken)
+		) =>
+			Dispatcher.DispatchAsync<IUpdateByQueryRethrottleRequest, UpdateByQueryRethrottleRequestParameters, ListTasksResponse, IListTasksResponse>(
+				request,
+				cancellationToken,
+				(p, d, c) => LowLevelDispatch.UpdateByQueryRethrottleDispatchAsync<ListTasksResponse>(p, c)
+			);
+	}
+}

src/Nest/Document/Multiple/UpdateByQueryRethrottle/UpdateByQueryRethrottleRequest.cs

@@ -0,0 +1,9 @@
+namespace Nest
+{
+	[MapsApi("update_by_query_rethrottle")]
+	public partial interface IUpdateByQueryRethrottleRequest { }
+
+	public partial class UpdateByQueryRethrottleRequest : IUpdateByQueryRethrottleRequest { }
+
+	public partial class UpdateByQueryRethrottleDescriptor : IUpdateByQueryRethrottleRequest { }
+}

src/Nest/_Generated/_Descriptors.generated.cs

@@ -4024,6 +4024,23 @@ public UpdateByQueryDescriptor() : this(typeof(T)){}
 		///<summary>The number of slices this task should be divided into. Defaults to 1 meaning the task isn't sliced into subtasks.</summary>
 		public UpdateByQueryDescriptor<T> Slices(long? slices) => Qs("slices", slices);
 	}
+	///<summary>descriptor for UpdateByQueryRethrottle <pre>https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update-by-query.html</pre></summary>
+	public partial class UpdateByQueryRethrottleDescriptor  : RequestDescriptorBase<UpdateByQueryRethrottleDescriptor,UpdateByQueryRethrottleRequestParameters, IUpdateByQueryRethrottleRequest>, IUpdateByQueryRethrottleRequest
+	{ 
+		///<summary>/_update_by_query/{task_id}/_rethrottle</summary>
+		///<param name="task_id">this parameter is required</param>
+		public UpdateByQueryRethrottleDescriptor(TaskId task_id) : base(r => r.Required("task_id", task_id)){}
+		///<summary>Used for serialization purposes, making sure we have a parameterless constructor</summary>
+		[SerializationConstructor]
+		internal UpdateByQueryRethrottleDescriptor() : base(){}
+		// values part of the url path
+		TaskId IUpdateByQueryRethrottleRequest.TaskId => Self.RouteValues.Get<TaskId>("task_id");
+
+		// Request parameters
+
+		///<summary>The throttle to set on this request in floating sub-requests per second. -1 means set no throttle.</summary>
+		public UpdateByQueryRethrottleDescriptor RequestsPerSecond(long? requestsPerSecond) => Qs("requests_per_second", requestsPerSecond);
+	}
 	///<summary>descriptor for CcrDeleteAutoFollowPattern <pre>https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-delete-auto-follow-pattern.html</pre></summary>
 	public partial class DeleteAutoFollowPatternDescriptor  : RequestDescriptorBase<DeleteAutoFollowPatternDescriptor,DeleteAutoFollowPatternRequestParameters, IDeleteAutoFollowPatternRequest>, IDeleteAutoFollowPatternRequest
 	{ 

src/Nest/_Generated/_LowLevelDispatch.generated.cs

@@ -2866,6 +2866,28 @@ internal partial class LowLevelDispatch
 			throw InvalidDispatch("UpdateByQuery", p, new [] { POST }, "/{index}/_update_by_query");
 		}
 
+		internal TResponse UpdateByQueryRethrottleDispatch<TResponse>(IRequest<UpdateByQueryRethrottleRequestParameters> p) where TResponse : class, IElasticsearchResponse, new()
+		{
+			switch(p.HttpMethod)
+			{
+				case POST:
+					if (AllSetNoFallback(p.RouteValues.TaskId)) return _lowLevel.UpdateByQueryRethrottle<TResponse>(p.RouteValues.TaskId,p.RequestParameters);
+					break;
+			}
+			throw InvalidDispatch("UpdateByQueryRethrottle", p, new [] { POST }, "/_update_by_query/{task_id}/_rethrottle");
+		}
+
+		internal Task<TResponse> UpdateByQueryRethrottleDispatchAsync<TResponse>(IRequest<UpdateByQueryRethrottleRequestParameters> p, CancellationToken ct) where TResponse : class, IElasticsearchResponse, new()
+		{
+			switch(p.HttpMethod)
+			{
+				case POST:
+					if (AllSetNoFallback(p.RouteValues.TaskId)) return _lowLevel.UpdateByQueryRethrottleAsync<TResponse>(p.RouteValues.TaskId,p.RequestParameters,ct);
+					break;
+			}
+			throw InvalidDispatch("UpdateByQueryRethrottle", p, new [] { POST }, "/_update_by_query/{task_id}/_rethrottle");
+		}
+
 		internal TResponse CcrDeleteAutoFollowPatternDispatch<TResponse>(IRequest<DeleteAutoFollowPatternRequestParameters> p) where TResponse : class, IElasticsearchResponse, new()
 		{
 			switch(p.HttpMethod)

src/Nest/_Generated/_Requests.generated.cs

@@ -6609,6 +6609,31 @@ public UpdateByQueryRequest(Indices index) : base(index){}
 		public UpdateByQueryRequest() : base(typeof(T)){}
 	}
 
+	[InterfaceDataContract]
+	public partial interface IUpdateByQueryRethrottleRequest : IRequest<UpdateByQueryRethrottleRequestParameters>
+	{
+		[IgnoreDataMember]
+			TaskId TaskId { get; }
+	}
+
+	///<summary>Request parameters for UpdateByQueryRethrottle <pre>https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update-by-query.html</pre></summary>
+	public partial class UpdateByQueryRethrottleRequest : PlainRequestBase<UpdateByQueryRethrottleRequestParameters>, IUpdateByQueryRethrottleRequest
+	{
+		protected IUpdateByQueryRethrottleRequest Self => this;
+		///<summary>/_update_by_query/{task_id}/_rethrottle</summary>
+		///<param name="task_id">this parameter is required</param>
+		public UpdateByQueryRethrottleRequest(TaskId task_id) : base(r => r.Required("task_id", task_id)){}
+		///<summary>Used for serialization purposes, making sure we have a parameterless constructor</summary>
+		[SerializationConstructor]
+		internal UpdateByQueryRethrottleRequest() : base(){}
+		// values part of the url path
+		[IgnoreDataMember]
+		TaskId IUpdateByQueryRethrottleRequest.TaskId => Self.RouteValues.Get<TaskId>("task_id");
+
+		// Request parameters
+		///<summary>The throttle to set on this request in floating sub-requests per second. -1 means set no throttle.</summary>
+		public long? RequestsPerSecond { get => Q<long?>("requests_per_second"); set => Q("requests_per_second", value); }
+	}
 	[InterfaceDataContract]
 	public partial interface IUpdateDatafeedRequest : IRequest<UpdateDatafeedRequestParameters>
 	{

src/Tests/Tests/CodeStandards/NamingConventions.doc.cs

@@ -125,7 +125,8 @@ public void ParityBetweenRequestsAndResponses()
 				typeof(IndicesShardStoresRequest),
 				typeof(RenderSearchTemplateRequest),
 				typeof(MultiSearchTemplateRequest),
-				typeof(CreateRequest<>)
+				typeof(CreateRequest<>),
+				typeof(UpdateByQueryRethrottleRequest) // uses ListTasksResponse
 			};
 
 			var types = typeof(IRequest).Assembly.GetTypes();

src/Tests/Tests/Document/Multiple/UpdateByQueryRethrottle/UpdateByQueryRethrottleApiTests.cs

@@ -0,0 +1,109 @@
+using System;
+using System.Linq;
+using Elasticsearch.Net;
+using FluentAssertions;
+using Nest;
+using Tests.Document.Multiple.Reindex;
+using Tests.Domain;
+using Tests.Framework;
+using Tests.Framework.Integration;
+using Tests.Core.Extensions;
+
+namespace Tests.Document.Multiple.UpdateByQueryRethrottle
+{
+	public class UpdateByQueryRethrottleApiTests
+		: ApiIntegrationTestBase<ReindexCluster, IListTasksResponse, IUpdateByQueryRethrottleRequest,
+			UpdateByQueryRethrottleDescriptor, UpdateByQueryRethrottleRequest>
+	{
+		protected const string TaskIdKey = "taskId";
+
+		public UpdateByQueryRethrottleApiTests(ReindexCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
+
+		protected override bool ExpectIsValid => true;
+		protected override object ExpectJson => null;
+		protected override int ExpectStatusCode => 200;
+
+		protected override Func<UpdateByQueryRethrottleDescriptor, IUpdateByQueryRethrottleRequest> Fluent => d => d
+			.RequestsPerSecond(-1);
+
+		protected override HttpMethod HttpMethod => HttpMethod.POST;
+
+		protected override UpdateByQueryRethrottleRequest Initializer => new UpdateByQueryRethrottleRequest(TaskId)
+		{
+			RequestsPerSecond = -1
+		};
+
+		protected override bool SupportsDeserialization => false;
+		protected TaskId TaskId => RanIntegrationSetup ? ExtendedValue<TaskId>(TaskIdKey) : "foo:1";
+
+		protected override string UrlPath => $"/_update_by_query/{TaskId.NodeId}%3A{TaskId.TaskNumber}/_rethrottle?requests_per_second=-1";
+
+		protected override UpdateByQueryRethrottleDescriptor NewDescriptor() => new UpdateByQueryRethrottleDescriptor(TaskId);
+
+		protected override void IntegrationSetup(IElasticClient client, CallUniqueValues values)
+		{
+			foreach (var callUniqueValue in values)
+			{
+				client.IndexMany(Project.Projects, callUniqueValue.Value);
+				client.Refresh(callUniqueValue.Value);
+			}
+		}
+
+		protected override LazyResponses ClientUsage() => Calls(
+			(client, f) => client.UpdateByQueryRethrottle(TaskId, f),
+			(client, f) => client.UpdateByQueryRethrottleAsync(TaskId, f),
+			(client, r) => client.UpdateByQueryRethrottle(r),
+			(client, r) => client.UpdateByQueryRethrottleAsync(r)
+		);
+
+		protected override void OnBeforeCall(IElasticClient client)
+		{
+			client.IndexMany(Project.Projects, CallIsolatedValue);
+			client.Refresh(CallIsolatedValue);
+
+			var updateByQuery = client.UpdateByQuery<Project>(u => u
+				.Index(CallIsolatedValue)
+				.Conflicts(Conflicts.Proceed)
+				.Query(q => q.MatchAll())
+				.Script(s => s.Source("ctx._source.numberOfCommits+=10"))
+				.Refresh()
+				.RequestsPerSecond(1)
+				.WaitForCompletion(false)
+			);
+
+			updateByQuery.ShouldBeValid();
+			ExtendedValue(TaskIdKey, updateByQuery.Task);
+		}
+
+		protected override void ExpectResponse(IListTasksResponse response)
+		{
+			response.ShouldBeValid();
+
+			response.Nodes.Should().NotBeEmpty().And.HaveCount(1);
+			var node = response.Nodes.First().Value;
+
+			node.Name.Should().NotBeNullOrEmpty();
+			node.TransportAddress.Should().NotBeNullOrEmpty();
+			node.Host.Should().NotBeNullOrEmpty();
+			node.Ip.Should().NotBeNullOrEmpty();
+			node.Roles.Should().NotBeEmpty();
+			node.Attributes.Should().NotBeEmpty();
+
+			node.Tasks.Should().NotBeEmpty().And.HaveCount(1);
+			node.Tasks.First().Key.Should().Be(TaskId);
+
+			var task = node.Tasks.First().Value;
+
+			task.Node.Should().NotBeNullOrEmpty().And.Be(TaskId.NodeId);
+			task.Id.Should().Be(TaskId.TaskNumber);
+			task.Type.Should().NotBeNullOrEmpty();
+			task.Action.Should().NotBeNullOrEmpty();
+
+			task.Status.RequestsPerSecond.Should().Be(-1);
+
+			task.StartTimeInMilliseconds.Should().BeGreaterThan(0);
+			task.RunningTimeInNanoSeconds.Should().BeGreaterThan(0);
+			task.Cancellable.Should().BeTrue();
+		}
+	}
+}