Skip to content

Commit

Permalink
feat(clients): allow batch size on objects helper [skip-bc] (#4172) (…
Browse files Browse the repository at this point in the history 
…generated) [skip ci]

Co-authored-by: Clément Vannicatte <vannicattec@gmail.com>
  • Loading branch information
@algolia-bot @shortcuts
algolia-bot and shortcuts committed Nov 29, 2024
1 parent 2325c61 commit b54f1c3
Show file tree
Hide file tree
Showing 9 changed files with 164 additions and 28 deletions.
82 changes: 79 additions & 3 deletions ...s/algoliasearch-client-java/algoliasearch/src/main/java/com/algolia/api/SearchClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6612,7 +6612,31 @@ public <T> List<BatchResponse> saveObjects(String indexName, Iterable<T> objects
* the transporter requestOptions. (optional)
*/
public <T> List<BatchResponse> saveObjects(String indexName, Iterable<T> objects, boolean waitForTasks, RequestOptions requestOptions) {
return chunkedBatch(indexName, objects, Action.ADD_OBJECT, waitForTasks, 1000, requestOptions);
return saveObjects(indexName, objects, false, 1000, requestOptions);
}

/**
* Helper: Saves the given array of objects in the given index. The `chunkedBatch` helper is used
* under the hood, which creates a `batch` requests with at most 1000 objects in it.
*
* @param indexName The `indexName` to replace `objects` in.
* @param objects The array of `objects` to store in the given Algolia `indexName`.
* @param waitForTasks - Whether or not we should wait until every `batch` tasks has been
* processed, this operation may slow the total execution time of this method but is more
* reliable.
* @param batchSize The size of the chunk of `objects`. The number of `batch` calls will be equal
* to `length(objects) / batchSize`.
* @param requestOptions The requestOptions to send along with the query, they will be merged with
* the transporter requestOptions. (optional)
*/
public <T> List<BatchResponse> saveObjects(
String indexName,
Iterable<T> objects,
boolean waitForTasks,
int batchSize,
RequestOptions requestOptions
) {
return chunkedBatch(indexName, objects, Action.ADD_OBJECT, waitForTasks, batchSize, requestOptions);
}

/**
Expand Down Expand Up @@ -6652,6 +6676,30 @@ public List<BatchResponse> deleteObjects(String indexName, List<String> objectID
* the transporter requestOptions. (optional)
*/
public List<BatchResponse> deleteObjects(String indexName, List<String> objectIDs, boolean waitForTasks, RequestOptions requestOptions) {
return deleteObjects(indexName, objectIDs, false, 1000, null);
}

/**
* Helper: Deletes every records for the given objectIDs. The `chunkedBatch` helper is used under
* the hood, which creates a `batch` requests with at most 1000 objectIDs in it.
*
* @param indexName The `indexName` to delete `objectIDs` from.
* @param objectIDs The array of `objectIDs` to delete from the `indexName`.
* @param waitForTasks - Whether or not we should wait until every `batch` tasks has been
* processed, this operation may slow the total execution time of this method but is more
* reliable.
* @param batchSize The size of the chunk of `objects`. The number of `batch` calls will be equal
* to `length(objects) / batchSize`.
* @param requestOptions The requestOptions to send along with the query, they will be merged with
* the transporter requestOptions. (optional)
*/
public List<BatchResponse> deleteObjects(
String indexName,
List<String> objectIDs,
boolean waitForTasks,
int batchSize,
RequestOptions requestOptions
) {
List<Map<String, String>> objects = new ArrayList<>();

for (String id : objectIDs) {
Expand All @@ -6660,7 +6708,7 @@ public List<BatchResponse> deleteObjects(String indexName, List<String> objectID
objects.add(obj);
}

return chunkedBatch(indexName, objects, Action.DELETE_OBJECT, waitForTasks, 1000, requestOptions);
return chunkedBatch(indexName, objects, Action.DELETE_OBJECT, waitForTasks, batchSize, requestOptions);
}

/**
Expand Down Expand Up @@ -6720,13 +6768,41 @@ public <T> List<BatchResponse> partialUpdateObjects(
boolean createIfNotExists,
boolean waitForTasks,
RequestOptions requestOptions
) {
return partialUpdateObjects(indexName, objects, createIfNotExists, waitForTasks, 1000, null);
}

/**
* Helper: Replaces object content of all the given objects according to their respective
* `objectID` field. The `chunkedBatch` helper is used under the hood, which creates a `batch`
* requests with at most 1000 objects in it.
*
* @param indexName The `indexName` to update `objects` in.
* @param objects The array of `objects` to update in the given Algolia `indexName`.
* @param createIfNotExists To be provided if non-existing objects are passed, otherwise, the call
* will fail.
* @param waitForTasks - Whether or not we should wait until every `batch` tasks has been
* processed, this operation may slow the total execution time of this method but is more
* reliable.
* @param batchSize The size of the chunk of `objects`. The number of `batch` calls will be equal
* to `length(objects) / batchSize`.
* @param requestOptions The requestOptions to send along with the query, they will be merged with
* the transporter requestOptions. (optional)
*/
public <T> List<BatchResponse> partialUpdateObjects(
String indexName,
Iterable<T> objects,
boolean createIfNotExists,
boolean waitForTasks,
int batchSize,
RequestOptions requestOptions
) {
return chunkedBatch(
indexName,
objects,
createIfNotExists ? Action.PARTIAL_UPDATE_OBJECT : Action.PARTIAL_UPDATE_OBJECT_NO_CREATE,
waitForTasks,
1000,
batchSize,
requestOptions
);
}
Expand Down
9 changes: 6 additions & 3 deletions clients/algoliasearch-client-javascript/packages/client-search/model/clientMethodProps.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -824,21 +824,24 @@ export type SearchClientNodeHelpers = {
getSecuredApiKeyRemainingValidity: (opts: GetSecuredApiKeyRemainingValidityOptions) => number;
};

export type DeleteObjectsOptions = Pick<ChunkedBatchOptions, 'indexName' | 'waitForTasks'> & {
export type DeleteObjectsOptions = Pick<ChunkedBatchOptions, 'indexName' | 'waitForTasks' | 'batchSize'> & {
/**
* The objectIDs to delete.
*/
objectIDs: string[];
};

export type PartialUpdateObjectsOptions = Pick<ChunkedBatchOptions, 'indexName' | 'objects' | 'waitForTasks'> & {
export type PartialUpdateObjectsOptions = Pick<
ChunkedBatchOptions,
'indexName' | 'objects' | 'waitForTasks' | 'batchSize'
> & {
/**
*To be provided if non-existing objects are passed, otherwise, the call will fail.
*/
createIfNotExists?: boolean;
};

export type SaveObjectsOptions = Pick<ChunkedBatchOptions, 'indexName' | 'objects' | 'waitForTasks'>;
export type SaveObjectsOptions = Pick<ChunkedBatchOptions, 'indexName' | 'objects' | 'waitForTasks' | 'batchSize'>;

export type ChunkedBatchOptions = ReplaceAllObjectsOptions & {
/**
Expand Down
16 changes: 12 additions & 4 deletions clients/algoliasearch-client-javascript/packages/client-search/src/searchClient.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -550,14 +550,18 @@ export function createSearchClient({
* @param saveObjects - The `saveObjects` object.
* @param saveObjects.indexName - The `indexName` to save `objects` in.
* @param saveObjects.objects - The array of `objects` to store in the given Algolia `indexName`.
* @param chunkedBatch.batchSize - The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.
* @param saveObjects.waitForTasks - Whether or not we should wait until every `batch` tasks has been processed, this operation may slow the total execution time of this method but is more reliable.
* @param requestOptions - The requestOptions to send along with the query, they will be forwarded to the `batch` method and merged with the transporter requestOptions.
*/
async saveObjects(
{ indexName, objects, waitForTasks }: SaveObjectsOptions,
{ indexName, objects, waitForTasks, batchSize }: SaveObjectsOptions,
requestOptions?: RequestOptions,
): Promise<BatchResponse[]> {
return await this.chunkedBatch({ indexName, objects, action: 'addObject', waitForTasks }, requestOptions);
return await this.chunkedBatch(
{ indexName, objects, action: 'addObject', waitForTasks, batchSize },
requestOptions,
);
},

/**
Expand All @@ -567,11 +571,12 @@ export function createSearchClient({
* @param deleteObjects - The `deleteObjects` object.
* @param deleteObjects.indexName - The `indexName` to delete `objectIDs` from.
* @param deleteObjects.objectIDs - The objectIDs to delete.
* @param chunkedBatch.batchSize - The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.
* @param deleteObjects.waitForTasks - Whether or not we should wait until every `batch` tasks has been processed, this operation may slow the total execution time of this method but is more reliable.
* @param requestOptions - The requestOptions to send along with the query, they will be forwarded to the `batch` method and merged with the transporter requestOptions.
*/
async deleteObjects(
{ indexName, objectIDs, waitForTasks }: DeleteObjectsOptions,
{ indexName, objectIDs, waitForTasks, batchSize }: DeleteObjectsOptions,
requestOptions?: RequestOptions,
): Promise<BatchResponse[]> {
return await this.chunkedBatch(
Expand All @@ -580,6 +585,7 @@ export function createSearchClient({
objects: objectIDs.map((objectID) => ({ objectID })),
action: 'deleteObject',
waitForTasks,
batchSize,
},
requestOptions,
);
Expand All @@ -593,18 +599,20 @@ export function createSearchClient({
* @param partialUpdateObjects.indexName - The `indexName` to update `objects` in.
* @param partialUpdateObjects.objects - The array of `objects` to update in the given Algolia `indexName`.
* @param partialUpdateObjects.createIfNotExists - To be provided if non-existing objects are passed, otherwise, the call will fail..
* @param chunkedBatch.batchSize - The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.
* @param partialUpdateObjects.waitForTasks - Whether or not we should wait until every `batch` tasks has been processed, this operation may slow the total execution time of this method but is more reliable.
* @param requestOptions - The requestOptions to send along with the query, they will be forwarded to the `getTask` method and merged with the transporter requestOptions.
*/
async partialUpdateObjects(
{ indexName, objects, createIfNotExists, waitForTasks }: PartialUpdateObjectsOptions,
{ indexName, objects, createIfNotExists, waitForTasks, batchSize }: PartialUpdateObjectsOptions,
requestOptions?: RequestOptions,
): Promise<BatchResponse[]> {
return await this.chunkedBatch(
{
indexName,
objects,
action: createIfNotExists ? 'partialUpdateObject' : 'partialUpdateObjectNoCreate',
batchSize,
waitForTasks,
},
requestOptions,
Expand Down
15 changes: 9 additions & 6 deletions clients/algoliasearch-client-php/lib/Api/SearchClient.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2945,31 +2945,33 @@ public function replaceAllObjects($indexName, $objects, $batchSize = 1000, $requ
*
* @param string $indexName the `indexName` to replace `objects` in
* @param array $objects the array of `objects` to store in the given Algolia `indexName`
* @param array $batchSize The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.
* @param array $requestOptions Request options
* @param bool $waitForTasks Whether or not we should wait until every `batch` tasks has been processed, this operation may slow the total execution time of this method but is more reliable
*/
public function saveObjects($indexName, $objects, $requestOptions = [], $waitForTasks = false)
public function saveObjects($indexName, $objects, $batchSize = 1000, $requestOptions = [], $waitForTasks = false)
{
return $this->chunkedBatch($indexName, $objects, 'addObject', $waitForTasks, 1000, $requestOptions);
return $this->chunkedBatch($indexName, $objects, 'addObject', $waitForTasks, $batchSize, $requestOptions);
}

/**
* Helper: Deletes every records for the given objectIDs. The `chunkedBatch` helper is used under the hood, which creates a `batch` requests with at most 1000 objectIDs in it.
*
* @param string $indexName the `indexName` to delete `objectIDs` from
* @param array $objectIDs the `objectIDs` to delete
* @param array $batchSize The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.
* @param array $requestOptions Request options
* @param bool $waitForTasks Whether or not we should wait until every `batch` tasks has been processed, this operation may slow the total execution time of this method but is more reliable
*/
public function deleteObjects($indexName, $objectIDs, $requestOptions = [], $waitForTasks = false)
public function deleteObjects($indexName, $objectIDs, $batchSize = 1000, $requestOptions = [], $waitForTasks = false)
{
$objects = [];

foreach ($objectIDs as $id) {
$objects[] = ['objectID' => $id];
}

return $this->chunkedBatch($indexName, $objects, 'deleteObject', $waitForTasks, 1000, $requestOptions);
return $this->chunkedBatch($indexName, $objects, 'deleteObject', $waitForTasks, $batchSize, $requestOptions);
}

/**
Expand All @@ -2978,12 +2980,13 @@ public function deleteObjects($indexName, $objectIDs, $requestOptions = [], $wai
* @param string $indexName the `indexName` to replace `objects` in
* @param array $objects the array of `objects` to store in the given Algolia `indexName`
* @param bool $createIfNotExists To be provided if non-existing objects are passed, otherwise, the call will fail..
* @param array $batchSize The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.
* @param array $requestOptions Request options
* @param bool $waitForTasks Whether or not we should wait until every `batch` tasks has been processed, this operation may slow the total execution time of this method but is more reliable
*/
public function partialUpdateObjects($indexName, $objects, $createIfNotExists, $requestOptions = [], $waitForTasks = false)
public function partialUpdateObjects($indexName, $objects, $createIfNotExists, $batchSize = 1000, $requestOptions = [], $waitForTasks = false)
{
return $this->chunkedBatch($indexName, $objects, (true == $createIfNotExists) ? 'partialUpdateObject' : 'partialUpdateObjectNoCreate', $waitForTasks, 1000, $requestOptions);
return $this->chunkedBatch($indexName, $objects, (true == $createIfNotExists) ? 'partialUpdateObject' : 'partialUpdateObjectNoCreate', $waitForTasks, $batchSize, $requestOptions);
}

/**
Expand Down
12 changes: 12 additions & 0 deletions clients/algoliasearch-client-python/algoliasearch/search/client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -504,6 +504,7 @@ async def save_objects(
index_name: str,
objects: List[Dict[str, Any]],
wait_for_tasks: bool = False,
batch_size: int = 1000,
request_options: Optional[Union[dict, RequestOptions]] = None,
) -> List[BatchResponse]:
"""
Expand All @@ -514,6 +515,7 @@ async def save_objects(
objects=objects,
action=Action.ADDOBJECT,
wait_for_tasks=wait_for_tasks,
batch_size=batch_size,
request_options=request_options,
)

Expand All @@ -522,6 +524,7 @@ async def delete_objects(
index_name: str,
object_ids: List[str],
wait_for_tasks: bool = False,
batch_size: int = 1000,
request_options: Optional[Union[dict, RequestOptions]] = None,
) -> List[BatchResponse]:
"""
Expand All @@ -532,6 +535,7 @@ async def delete_objects(
objects=[{"objectID": id} for id in object_ids],
action=Action.DELETEOBJECT,
wait_for_tasks=wait_for_tasks,
batch_size=batch_size,
request_options=request_options,
)

Expand All @@ -541,6 +545,7 @@ async def partial_update_objects(
objects: List[Dict[str, Any]],
create_if_not_exists: bool = False,
wait_for_tasks: bool = False,
batch_size: int = 1000,
request_options: Optional[Union[dict, RequestOptions]] = None,
) -> List[BatchResponse]:
"""
Expand All @@ -553,6 +558,7 @@ async def partial_update_objects(
if create_if_not_exists
else Action.PARTIALUPDATEOBJECTNOCREATE,
wait_for_tasks=wait_for_tasks,
batch_size=batch_size,
request_options=request_options,
)

Expand Down Expand Up @@ -5526,6 +5532,7 @@ def save_objects(
index_name: str,
objects: List[Dict[str, Any]],
wait_for_tasks: bool = False,
batch_size: int = 1000,
request_options: Optional[Union[dict, RequestOptions]] = None,
) -> List[BatchResponse]:
"""
Expand All @@ -5536,6 +5543,7 @@ def save_objects(
objects=objects,
action=Action.ADDOBJECT,
wait_for_tasks=wait_for_tasks,
batch_size=batch_size,
request_options=request_options,
)

Expand All @@ -5544,6 +5552,7 @@ def delete_objects(
index_name: str,
object_ids: List[str],
wait_for_tasks: bool = False,
batch_size: int = 1000,
request_options: Optional[Union[dict, RequestOptions]] = None,
) -> List[BatchResponse]:
"""
Expand All @@ -5554,6 +5563,7 @@ def delete_objects(
objects=[{"objectID": id} for id in object_ids],
action=Action.DELETEOBJECT,
wait_for_tasks=wait_for_tasks,
batch_size=batch_size,
request_options=request_options,
)

Expand All @@ -5563,6 +5573,7 @@ def partial_update_objects(
objects: List[Dict[str, Any]],
create_if_not_exists: bool = False,
wait_for_tasks: bool = False,
batch_size: int = 1000,
request_options: Optional[Union[dict, RequestOptions]] = None,
) -> List[BatchResponse]:
"""
Expand All @@ -5575,6 +5586,7 @@ def partial_update_objects(
if create_if_not_exists
else Action.PARTIALUPDATEOBJECTNOCREATE,
wait_for_tasks=wait_for_tasks,
batch_size=batch_size,
request_options=request_options,
)

Expand Down
Loading

0 comments on commit b54f1c3

Please sign in to comment.