diff --git a/README.md b/README.md index 61a4faa3..49b50d7f 100644 --- a/README.md +++ b/README.md @@ -86,8 +86,6 @@ Samples are in the [`samples/`](https://github.com/googleapis/java-retail/tree/m | Events Create Big Query Table | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java) | | Events Create Gcs Bucket | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java) | | Update User Events Json | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/events/setup/UpdateUserEventsJson.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/events/setup/UpdateUserEventsJson.java) | -| Create Test Resources | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/init/CreateTestResources.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/init/CreateTestResources.java) | -| Remove Test Resources | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/init/RemoveTestResources.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/init/RemoveTestResources.java) | | Add Fulfillment Places | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/product/AddFulfillmentPlaces.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/product/AddFulfillmentPlaces.java) | | Create Product | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/product/CreateProduct.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/product/CreateProduct.java) | | Crud Product | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/product/CrudProduct.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/product/CrudProduct.java) | @@ -101,6 +99,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/java-retail/tree/m | Update Product | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/product/UpdateProduct.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/product/UpdateProduct.java) | | Products Create Bigquery Table | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java) | | Products Create Gcs Bucket | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateGcsBucket.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateGcsBucket.java) | +| Remove Products Resources | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/product/setup/RemoveProductsResources.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/product/setup/RemoveProductsResources.java) | | Search Simple Query | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/search/SearchSimpleQuery.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/search/SearchSimpleQuery.java) | | Search With Boost Spec | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/search/SearchWithBoostSpec.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/search/SearchWithBoostSpec.java) | | Search With Facet Spec | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/search/SearchWithFacetSpec.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/search/SearchWithFacetSpec.java) | diff --git a/samples/interactive-tutorials/src/main/java/init/CreateTestResources.java b/samples/interactive-tutorials/src/main/java/init/CreateTestResources.java deleted file mode 100644 index 5a0f6cc6..00000000 --- a/samples/interactive-tutorials/src/main/java/init/CreateTestResources.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package init; - -import com.google.cloud.ServiceOptions; -import com.google.cloud.retail.v2.GcsSource; -import com.google.cloud.retail.v2.ImportErrorsConfig; -import com.google.cloud.retail.v2.ImportMetadata; -import com.google.cloud.retail.v2.ImportProductsRequest; -import com.google.cloud.retail.v2.ImportProductsRequest.ReconciliationMode; -import com.google.cloud.retail.v2.ImportProductsResponse; -import com.google.cloud.retail.v2.ProductInputConfig; -import com.google.cloud.retail.v2.ProductServiceClient; -import com.google.longrunning.Operation; -import com.google.longrunning.OperationsClient; -import events.setup.EventsCreateBigQueryTable; -import events.setup.EventsCreateGcsBucket; -import java.io.IOException; -import java.util.Collections; -import product.setup.ProductsCreateBigqueryTable; -import product.setup.ProductsCreateGcsBucket; - -public class CreateTestResources { - - public static void main(String... args) throws IOException, InterruptedException { - // TODO(developer): Replace these variables before running the sample. - String projectId = ServiceOptions.getDefaultProjectId(); - String bucketName = System.getenv("BUCKET_NAME"); - String gcsBucket = String.format("gs://%s", System.getenv("BUCKET_NAME")); - String gcsErrorBucket = String.format("%s/errors", gcsBucket); - String branchName = - String.format( - "projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId); - - ProductsCreateGcsBucket.main(); - EventsCreateGcsBucket.main(); - importProductsFromGcs(bucketName, gcsErrorBucket, branchName); - ProductsCreateBigqueryTable.main(); - EventsCreateBigQueryTable.main(); - } - - public static void importProductsFromGcs( - String bucketName, String gcsErrorBucket, String branchName) - throws IOException, InterruptedException { - GcsSource gcsSource = - GcsSource.newBuilder() - .addAllInputUris( - Collections.singleton(String.format("gs://%s/%s", bucketName, "products.json"))) - .build(); - ProductInputConfig inputConfig = - ProductInputConfig.newBuilder().setGcsSource(gcsSource).build(); - System.out.println("GRS source: " + gcsSource.getInputUrisList()); - - ImportErrorsConfig errorsConfig = - ImportErrorsConfig.newBuilder().setGcsPrefix(gcsErrorBucket).build(); - ImportProductsRequest importRequest = - ImportProductsRequest.newBuilder() - .setParent(branchName) - .setReconciliationMode(ReconciliationMode.INCREMENTAL) - .setInputConfig(inputConfig) - .setErrorsConfig(errorsConfig) - .build(); - System.out.println("Import products from google cloud source request: " + importRequest); - - try (ProductServiceClient serviceClient = ProductServiceClient.create()) { - String operationName = serviceClient.importProductsCallable().call(importRequest).getName(); - System.out.printf("OperationName = %s\n", operationName); - - OperationsClient operationsClient = serviceClient.getOperationsClient(); - Operation operation = operationsClient.getOperation(operationName); - - while (!operation.getDone()) { - System.out.println("Please wait till operation is completed."); - // Keep polling the operation periodically until the import task is done. - Thread.sleep(30_000); - operation = operationsClient.getOperation(operationName); - } - - System.out.println("Import products operation is completed."); - - if (operation.hasMetadata()) { - ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class); - System.out.printf( - "Number of successfully imported products: %s\n", metadata.getSuccessCount()); - System.out.printf( - "Number of failures during the importing: %s\n", metadata.getFailureCount()); - } - - if (operation.hasResponse()) { - ImportProductsResponse response = - operation.getResponse().unpack(ImportProductsResponse.class); - System.out.printf("Operation result: %s", response); - } - } - } -} diff --git a/samples/interactive-tutorials/src/main/java/init/TEST_RESOURCES_SETUP_CLEANUP.md b/samples/interactive-tutorials/src/main/java/init/TEST_RESOURCES_SETUP_CLEANUP.md deleted file mode 100644 index 0b9df6d1..00000000 --- a/samples/interactive-tutorials/src/main/java/init/TEST_RESOURCES_SETUP_CLEANUP.md +++ /dev/null @@ -1,49 +0,0 @@ -# How to set up/ tear down the test resources - -## Required environment variables - -To successfully import the catalog data for tests, the following environment variables should be -set: - -- PROJECT_ID -- PROJECT_NUMBER -- BUCKET_NAME - -The Secret Manager name is set in .kokoro/presubmit/common.cfg file, SECRET_MANAGER_KEYS variable. - -## Import catalog data - -There is a JSON file with valid products prepared in the `product` directory: -`resources/products.json`. - -Run the `CreateTestResources` to perform the following actions: - -- create the GCS bucket ; -- upload the product data from `resources/products.json` file to the bucket; -- import products to the default branch of the Retail catalog; -- upload the product data from `resources/user_events.json` file to the bucket; -- create a BigQuery dataset and table `products`; -- insert products from resources/products.json to the created products table; -- create a BigQuery dataset and table `events`; -- insert user events from resources/user_events.json to the created events table; - -``` -mvn compile exec:java -Dexec.mainClass="init.CreateTestResources" -``` - -In the result 316 products should be created in the test project catalog. - -## Remove catalog data - -Run the `RemoveTestResources` to perform the following actions: - -- remove all objects from the GCS bucket ; -- remove the bucket; -- delete all products from the Retail catalog; -- remove all objects from the GCS bucket ; -- remove dataset `products` along with tables; -- remove dataset `user_events` along with tables; - -``` -mvn compile exec:java -Dexec.mainClass="init.RemoveTestResources" -``` \ No newline at end of file diff --git a/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java b/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java index 291720c1..38935f9d 100644 --- a/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java +++ b/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java @@ -33,11 +33,11 @@ import com.google.longrunning.Operation; import com.google.longrunning.OperationsClient; import java.io.IOException; +import java.util.concurrent.TimeUnit; public class ImportProductsBigQueryTable { public static void main(String[] args) throws IOException, InterruptedException { - // TODO(developer): Replace these variables before running the sample. String projectId = ServiceOptions.getDefaultProjectId(); String branchName = String.format( @@ -45,24 +45,18 @@ public static void main(String[] args) throws IOException, InterruptedException String datasetId = "products"; String tableId = "products"; // TO CHECK ERROR HANDLING USE THE TABLE WITH INVALID PRODUCTS: - // TABLE_ID = "products_some_invalid" - String dataSchema = "product"; - // TRY THE FULL RECONCILIATION MODE HERE: - ReconciliationMode reconciliationMode = ReconciliationMode.INCREMENTAL; + // tableId = "products_some_invalid" - ImportProductsRequest importBigQueryRequest = - getImportProductsBigQueryRequest( - reconciliationMode, projectId, datasetId, tableId, dataSchema, branchName); - waitForOperationCompletion(importBigQueryRequest); + importProductsFromBigQuery(projectId, branchName, datasetId, tableId); } - public static ImportProductsRequest getImportProductsBigQueryRequest( - ReconciliationMode reconciliationMode, - String projectId, - String datasetId, - String tableId, - String dataSchema, - String branchName) { + public static void importProductsFromBigQuery( + String projectId, String branchName, String datasetId, String tableId) + throws IOException, InterruptedException { + // TRY THE FULL RECONCILIATION MODE HERE: + ReconciliationMode reconciliationMode = ReconciliationMode.INCREMENTAL; + String dataSchema = "product"; + BigQuerySource bigQuerySource = BigQuerySource.newBuilder() .setProjectId(projectId) @@ -82,30 +76,29 @@ public static ImportProductsRequest getImportProductsBigQueryRequest( .build(); System.out.printf("Import products from big query table request: %s%n", importRequest); - return importRequest; - } - - public static void waitForOperationCompletion(ImportProductsRequest importRequest) - throws IOException, InterruptedException { + // Initialize client that will be used to send requests. This client only + // needs to be created once, and can be reused for multiple requests. After + // completing all of your requests, call the "close" method on the client to + // safely clean up any remaining background resources. try (ProductServiceClient serviceClient = ProductServiceClient.create()) { String operationName = serviceClient.importProductsCallable().call(importRequest).getName(); - System.out.printf("OperationName = %s\n", operationName); + System.out.printf("OperationName = %s%n", operationName); OperationsClient operationsClient = serviceClient.getOperationsClient(); Operation operation = operationsClient.getOperation(operationName); while (!operation.getDone()) { // Keep polling the operation periodically until the import task is done. - Thread.sleep(30_000); + TimeUnit.SECONDS.sleep(30); operation = operationsClient.getOperation(operationName); } if (operation.hasMetadata()) { ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class); System.out.printf( - "Number of successfully imported products: %s\n", metadata.getSuccessCount()); + "Number of successfully imported products: %s%n", metadata.getSuccessCount()); System.out.printf( - "Number of failures during the importing: %s\n", metadata.getFailureCount()); + "Number of failures during the importing: %s%n", metadata.getFailureCount()); } if (operation.hasResponse()) { diff --git a/samples/interactive-tutorials/src/main/java/product/ImportProductsGcs.java b/samples/interactive-tutorials/src/main/java/product/ImportProductsGcs.java index 16322466..b1d72c57 100644 --- a/samples/interactive-tutorials/src/main/java/product/ImportProductsGcs.java +++ b/samples/interactive-tutorials/src/main/java/product/ImportProductsGcs.java @@ -22,6 +22,8 @@ package product; +import com.google.api.gax.rpc.InvalidArgumentException; +import com.google.api.gax.rpc.PermissionDeniedException; import com.google.cloud.ServiceOptions; import com.google.cloud.retail.v2.GcsSource; import com.google.cloud.retail.v2.ImportErrorsConfig; @@ -34,37 +36,42 @@ import com.google.longrunning.Operation; import com.google.longrunning.OperationsClient; import java.io.IOException; +import java.time.Instant; import java.util.Collections; +import java.util.concurrent.TimeUnit; public class ImportProductsGcs { public static void main(String[] args) throws IOException, InterruptedException { - // TODO(developer): Replace these variables before running the sample. String projectId = ServiceOptions.getDefaultProjectId(); String branchName = String.format( "projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId); + + String bucketName = System.getenv("BUCKET_NAME"); + String gcsBucket = String.format("gs://%s", bucketName); + String gcsErrorBucket = String.format("%s/errors", gcsBucket); + // To check error handling, use an invalid catalog in request // branchName = // String.format("projects/%s/locations/global/catalogs/invalid_catalog/branches/default_branch", projectId); - String gcsBucket = String.format("gs://%s", System.getenv("BUCKET_NAME")); - String gcsErrorBucket = String.format("%s/errors", gcsBucket); String gcsProductsObject = "products.json"; // To check error handling, use an invalid product JSON. - // gcsProductsObject = "products_some_invalid.json"; + // gcsProductsObject = "products_some_invalid.json" - ImportProductsRequest importGcsRequest = - getImportProductsGcsRequest(gcsProductsObject, gcsBucket, gcsErrorBucket, branchName); - waitForOperationCompletion(importGcsRequest); + importProductsFromGcs(branchName, gcsBucket, gcsProductsObject); } - public static ImportProductsRequest getImportProductsGcsRequest( - String gcsObjectName, String gcsBucket, String gcsErrorBucket, String branchName) { + public static void importProductsFromGcs( + String branchName, String gcsBucket, String gcsProductsObject) + throws IOException, InterruptedException { + String gcsErrorBucket = String.format("%s/errors", gcsBucket); + GcsSource gcsSource = GcsSource.newBuilder() .addAllInputUris( - Collections.singleton(String.format("%s/%s", gcsBucket, gcsObjectName))) + Collections.singleton(String.format("%s/%s", gcsBucket, gcsProductsObject))) .build(); ProductInputConfig inputConfig = @@ -83,39 +90,53 @@ public static ImportProductsRequest getImportProductsGcsRequest( .setErrorsConfig(errorsConfig) .build(); - System.out.println("Import products from google cloud source request: " + importRequest); - - return importRequest; - } + System.out.printf("Import products from google cloud source request: %s%n", importRequest); - public static void waitForOperationCompletion(ImportProductsRequest importRequest) - throws IOException, InterruptedException { + // Initialize client that will be used to send requests. This client only + // needs to be created once, and can be reused for multiple requests. After + // completing all of your requests, call the "close" method on the client to + // safely clean up any remaining background resources. try (ProductServiceClient serviceClient = ProductServiceClient.create()) { String operationName = serviceClient.importProductsCallable().call(importRequest).getName(); - System.out.printf("OperationName = %s\n", operationName); + + System.out.println("The operation was started."); + System.out.printf("OperationName = %s%n", operationName); OperationsClient operationsClient = serviceClient.getOperationsClient(); Operation operation = operationsClient.getOperation(operationName); - while (!operation.getDone()) { - // Keep polling the operation periodically until the import task is done. - Thread.sleep(30_000); + Instant deadline = Instant.now().plusSeconds(60); + + while (!operation.getDone() || Instant.now().isBefore(deadline)) { + System.out.println("Please wait till operation is done."); + TimeUnit.SECONDS.sleep(30); operation = operationsClient.getOperation(operationName); } if (operation.hasMetadata()) { ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class); System.out.printf( - "Number of successfully imported products: %s\n", metadata.getSuccessCount()); + "Number of successfully imported products: %s%n", metadata.getSuccessCount()); System.out.printf( - "Number of failures during the importing: %s\n", metadata.getFailureCount()); + "Number of failures during the importing: %s%n", metadata.getFailureCount()); + } else { + System.out.println("Metadata is empty."); } if (operation.hasResponse()) { ImportProductsResponse response = operation.getResponse().unpack(ImportProductsResponse.class); System.out.printf("Operation result: %s%n", response); + } else { + System.out.println("Operation result is empty."); } + } catch (InvalidArgumentException e) { + System.out.printf( + "%s%n'%s' file does not exist in the bucket. Please " + + "make sure you have followed the setting up instructions.", + e.getMessage(), gcsProductsObject); + } catch (PermissionDeniedException e) { + System.out.println(e.getMessage()); } } } diff --git a/samples/interactive-tutorials/src/main/java/product/ImportProductsInlineSource.java b/samples/interactive-tutorials/src/main/java/product/ImportProductsInlineSource.java index 3f8b2639..bf0cbf94 100644 --- a/samples/interactive-tutorials/src/main/java/product/ImportProductsInlineSource.java +++ b/samples/interactive-tutorials/src/main/java/product/ImportProductsInlineSource.java @@ -22,6 +22,7 @@ package product; +import com.google.api.gax.rpc.InvalidArgumentException; import com.google.cloud.ServiceOptions; import com.google.cloud.retail.v2.ColorInfo; import com.google.cloud.retail.v2.FulfillmentInfo; @@ -42,24 +43,23 @@ import java.util.Collections; import java.util.List; import java.util.UUID; +import java.util.concurrent.TimeUnit; public class ImportProductsInlineSource { public static void main(String[] args) throws IOException, InterruptedException { - // TODO(developer): Replace these variables before running the sample. String projectId = ServiceOptions.getDefaultProjectId(); String branchName = String.format( "projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId); - ImportProductsRequest importRequest = getImportProductsInlineRequest(getProducts(), branchName); - waitForOperationCompletion(importRequest); + importProductsInlineSource(branchName); } - public static ImportProductsRequest getImportProductsInlineRequest( - List productsToImport, String branchName) { + public static void importProductsInlineSource(String branchName) + throws IOException, InterruptedException { ProductInlineSource inlineSource = - ProductInlineSource.newBuilder().addAllProducts(productsToImport).build(); + ProductInlineSource.newBuilder().addAllProducts(getProducts()).build(); ProductInputConfig inputConfig = ProductInputConfig.newBuilder().setProductInlineSource(inlineSource).build(); @@ -72,7 +72,41 @@ public static ImportProductsRequest getImportProductsInlineRequest( System.out.printf("Import products from inline source request: %s%n", importRequest); - return importRequest; + // Initialize client that will be used to send requests. This client only + // needs to be created once, and can be reused for multiple requests. After + // completing all of your requests, call the "close" method on the client to + // safely clean up any remaining background resources. + try (ProductServiceClient serviceClient = ProductServiceClient.create()) { + String operationName = serviceClient.importProductsCallable().call(importRequest).getName(); + System.out.printf("OperationName = %s%n", operationName); + + OperationsClient operationsClient = serviceClient.getOperationsClient(); + Operation operation = operationsClient.getOperation(operationName); + + long assuredBreak = System.currentTimeMillis() + 60000; // 60 seconds delay + + while (!operation.getDone() || System.currentTimeMillis() < assuredBreak) { + // Keep polling the operation periodically until the import task is done. + TimeUnit.SECONDS.sleep(30); + operation = operationsClient.getOperation(operationName); + } + + if (operation.hasMetadata()) { + ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class); + System.out.printf( + "Number of successfully imported products: %s%n", metadata.getSuccessCount()); + System.out.printf( + "Number of failures during the importing: %s%n", metadata.getFailureCount()); + } + + if (operation.hasResponse()) { + ImportProductsResponse response = + operation.getResponse().unpack(ImportProductsResponse.class); + System.out.printf("Operation result: %s%n", response); + } + } catch (InvalidArgumentException e) { + System.out.println(e.getMessage()); + } } public static List getProducts() { @@ -175,37 +209,6 @@ public static List getProducts() { return products; } - - public static void waitForOperationCompletion(ImportProductsRequest importRequest) - throws IOException, InterruptedException { - try (ProductServiceClient serviceClient = ProductServiceClient.create()) { - String operationName = serviceClient.importProductsCallable().call(importRequest).getName(); - System.out.printf("OperationName = %s\n", operationName); - - OperationsClient operationsClient = serviceClient.getOperationsClient(); - Operation operation = operationsClient.getOperation(operationName); - - while (!operation.getDone()) { - // Keep polling the operation periodically until the import task is done. - Thread.sleep(30_000); - operation = operationsClient.getOperation(operationName); - } - - if (operation.hasMetadata()) { - ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class); - System.out.printf( - "Number of successfully imported products: %s\n", metadata.getSuccessCount()); - System.out.printf( - "Number of failures during the importing: %s\n", metadata.getFailureCount()); - } - - if (operation.hasResponse()) { - ImportProductsResponse response = - operation.getResponse().unpack(ImportProductsResponse.class); - System.out.printf("Operation result: %s%n", response); - } - } - } } // [END retail_import_products_from_inline_source] diff --git a/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java b/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java index 361144a8..f5244b78 100644 --- a/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java +++ b/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java @@ -30,16 +30,15 @@ public class ProductsCreateBigqueryTable { - public static void main(String... args) throws IOException { + public static void main(String[] args) throws IOException { String dataset = "products"; String validProductsTable = "products"; String invalidProductsTable = "products_some_invalid"; String productSchemaFilePath = "src/main/resources/product_schema.json"; String validProductsSourceFile = - String.format("gs://%s/products.json", ProductsCreateGcsBucket.getBucketName()); + String.format("gs://%s/products.json", System.getenv("BUCKET_NAME")); String invalidProductsSourceFile = - String.format( - "gs://%s/products_some_invalid.json", ProductsCreateGcsBucket.getBucketName()); + String.format("gs://%s/products_some_invalid.json", System.getenv("BUCKET_NAME")); BufferedReader bufferedReader = new BufferedReader(new FileReader(productSchemaFilePath)); String jsonToString = bufferedReader.lines().collect(Collectors.joining()); diff --git a/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateGcsBucket.java b/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateGcsBucket.java index 1e5caad4..80ed6285 100644 --- a/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateGcsBucket.java +++ b/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateGcsBucket.java @@ -38,16 +38,20 @@ public class ProductsCreateGcsBucket { String.format("%s_products_%s", PROJECT_ID, CURRENT_DATE.getSeconds()); public static void main(String... args) throws IOException { - createBucket(BUCKET_NAME); - System.out.printf("Products gcs bucket %s was created.", BUCKET_NAME); + createGcsBucketAndUploadData(BUCKET_NAME); + } + + public static void createGcsBucketAndUploadData(String bucketName) throws IOException { + createBucket(bucketName); + System.out.printf("Products gcs bucket %s was created.%n", bucketName); - uploadObject(BUCKET_NAME, "products.json", "src/main/resources/products.json"); - System.out.printf("File 'products.json' was uploaded into bucket '%s'.", BUCKET_NAME); + uploadObject(bucketName, "products.json", "src/main/resources/products.json"); + System.out.printf("File 'products.json' was uploaded into bucket '%s'.%n", bucketName); uploadObject( - BUCKET_NAME, "products_some_invalid.json", "src/main/resources/products_some_invalid.json"); + bucketName, "products_some_invalid.json", "src/main/resources/products_some_invalid.json"); System.out.printf( - "File 'products_some_invalid.json' was uploaded into bucket '%s'.", BUCKET_NAME); + "File 'products_some_invalid.json' was uploaded into bucket '%s'.%n", bucketName); } public static String getBucketName() { diff --git a/samples/interactive-tutorials/src/main/java/init/RemoveTestResources.java b/samples/interactive-tutorials/src/main/java/product/setup/RemoveProductsResources.java similarity index 85% rename from samples/interactive-tutorials/src/main/java/init/RemoveTestResources.java rename to samples/interactive-tutorials/src/main/java/product/setup/RemoveProductsResources.java index c48ceff2..de7aea24 100644 --- a/samples/interactive-tutorials/src/main/java/init/RemoveTestResources.java +++ b/samples/interactive-tutorials/src/main/java/product/setup/RemoveProductsResources.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package init; +package product.setup; import static setup.SetupCleanup.deleteBucket; import static setup.SetupCleanup.deleteDataset; @@ -28,10 +28,9 @@ import com.google.cloud.retail.v2.ProductServiceClient.ListProductsPagedResponse; import java.io.IOException; -public class RemoveTestResources { +public class RemoveProductsResources { - public static void main(String... args) throws IOException { - // TODO(developer): Replace these variables before running the sample. + public static void main(String[] args) throws IOException { String projectId = ServiceOptions.getDefaultProjectId(); String bucketName = System.getenv("BUCKET_NAME"); String branchName = @@ -41,12 +40,15 @@ public static void main(String... args) throws IOException { deleteBucket(bucketName); deleteAllProducts(branchName); deleteDataset(projectId, "products"); - deleteDataset(projectId, "user_events"); } public static void deleteAllProducts(String branchName) throws IOException { System.out.println("Deleting products in process, please wait..."); + // Initialize client that will be used to send requests. This client only + // needs to be created once, and can be reused for multiple requests. After + // completing all of your requests, call the "close" method on the client to + // safely clean up any remaining background resources. try (ProductServiceClient productServiceClient = ProductServiceClient.create()) { ListProductsRequest listRequest = ListProductsRequest.newBuilder().setParent(branchName).build(); diff --git a/samples/interactive-tutorials/src/test/java/product/ImportProductsBigQueryTableTest.java b/samples/interactive-tutorials/src/test/java/product/ImportProductsBigQueryTableTest.java index 58191f37..fce668a3 100644 --- a/samples/interactive-tutorials/src/test/java/product/ImportProductsBigQueryTableTest.java +++ b/samples/interactive-tutorials/src/test/java/product/ImportProductsBigQueryTableTest.java @@ -17,12 +17,9 @@ package product; import static com.google.common.truth.Truth.assertThat; -import static product.ImportProductsBigQueryTable.getImportProductsBigQueryRequest; -import static product.ImportProductsBigQueryTable.waitForOperationCompletion; +import static product.ImportProductsBigQueryTable.importProductsFromBigQuery; import com.google.cloud.ServiceOptions; -import com.google.cloud.retail.v2.ImportProductsRequest; -import com.google.cloud.retail.v2.ImportProductsRequest.ReconciliationMode; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; @@ -30,8 +27,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import product.setup.ProductsCreateBigqueryTable; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +@RunWith(JUnit4.class) public class ImportProductsBigQueryTableTest { private ByteArrayOutputStream bout; @@ -39,33 +38,46 @@ public class ImportProductsBigQueryTableTest { @Before public void setUp() throws IOException, InterruptedException, ExecutionException { + bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + originalPrintStream = System.out; + System.setOut(out); + } + + @Test + public void testValidImportProductsBigQueryTable() throws IOException, InterruptedException { String projectId = ServiceOptions.getDefaultProjectId(); String branchName = String.format( "projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId); String datasetId = "products"; String tableId = "products"; - String dataSchema = "product"; - ReconciliationMode reconciliationMode = ReconciliationMode.INCREMENTAL; - bout = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(bout); - originalPrintStream = System.out; - System.setOut(out); - ProductsCreateBigqueryTable.main(); - ImportProductsRequest importBigQueryRequest = - getImportProductsBigQueryRequest( - reconciliationMode, projectId, datasetId, tableId, dataSchema, branchName); - waitForOperationCompletion(importBigQueryRequest); + importProductsFromBigQuery(projectId, branchName, datasetId, tableId); + + String outputResult = bout.toString(); + + assertThat(outputResult).contains("Import products from big query table request"); + assertThat(outputResult).contains("Number of successfully imported products:"); + assertThat(outputResult).contains("Number of failures during the importing: 0"); } @Test - public void testImportProductsBigQueryTable() { + public void testInvalidImportProductsBigQueryTable() throws IOException, InterruptedException { + String projectId = ServiceOptions.getDefaultProjectId(); + String branchName = + String.format( + "projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId); + String datasetId = "products"; + String tableId = "products_some_invalid"; + + importProductsFromBigQuery(projectId, branchName, datasetId, tableId); + String outputResult = bout.toString(); assertThat(outputResult).contains("Import products from big query table request"); - assertThat(outputResult).contains("Number of successfully imported products"); - assertThat(outputResult).contains("Number of failures during the importing"); + assertThat(outputResult).contains("Number of successfully imported products:"); + assertThat(outputResult).contains("Number of failures during the importing:"); } @After diff --git a/samples/interactive-tutorials/src/test/java/product/ImportProductsGcsTest.java b/samples/interactive-tutorials/src/test/java/product/ImportProductsGcsTest.java index 7ae805aa..22594f6b 100644 --- a/samples/interactive-tutorials/src/test/java/product/ImportProductsGcsTest.java +++ b/samples/interactive-tutorials/src/test/java/product/ImportProductsGcsTest.java @@ -17,11 +17,9 @@ package product; import static com.google.common.truth.Truth.assertThat; -import static product.ImportProductsGcs.getImportProductsGcsRequest; -import static product.ImportProductsGcs.waitForOperationCompletion; +import static product.ImportProductsGcs.importProductsFromGcs; import com.google.cloud.ServiceOptions; -import com.google.cloud.retail.v2.ImportProductsRequest; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; @@ -29,8 +27,11 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; import product.setup.ProductsCreateGcsBucket; +@RunWith(JUnit4.class) public class ImportProductsGcsTest { private ByteArrayOutputStream bout; @@ -39,31 +40,28 @@ public class ImportProductsGcsTest { @Before public void setUp() throws IOException, InterruptedException, ExecutionException { ProductsCreateGcsBucket.main(); - String projectId = ServiceOptions.getDefaultProjectId(); String branchName = String.format( "projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId); - String gcsBucket = String.format("gs://%s", ProductsCreateGcsBucket.getBucketName()); - String gcsErrorBucket = String.format("%s/errors", gcsBucket); + String bucketName = ProductsCreateGcsBucket.getBucketName(); + String gcsBucket = String.format("gs://%s", bucketName); String gscProductsObject = "products.json"; bout = new ByteArrayOutputStream(); PrintStream out = new PrintStream(bout); originalPrintStream = System.out; System.setOut(out); - ImportProductsRequest importGcsRequest = - getImportProductsGcsRequest(gscProductsObject, gcsBucket, gcsErrorBucket, branchName); - waitForOperationCompletion(importGcsRequest); + importProductsFromGcs(branchName, gcsBucket, gscProductsObject); } @Test - public void testImportProductsGcs() { + public void testValidImportProductsGcs() { String outputResult = bout.toString(); assertThat(outputResult).contains("Import products from google cloud source request"); - assertThat(outputResult).contains("Number of successfully imported products"); - assertThat(outputResult).contains("Number of failures during the importing"); + assertThat(outputResult).contains("Number of successfully imported products:"); + assertThat(outputResult).contains("Number of failures during the importing: 0"); } @After diff --git a/samples/interactive-tutorials/src/test/java/product/ImportProductsInlineSourceTest.java b/samples/interactive-tutorials/src/test/java/product/ImportProductsInlineSourceTest.java index eb4678a9..b5a424dc 100644 --- a/samples/interactive-tutorials/src/test/java/product/ImportProductsInlineSourceTest.java +++ b/samples/interactive-tutorials/src/test/java/product/ImportProductsInlineSourceTest.java @@ -17,12 +17,9 @@ package product; import static com.google.common.truth.Truth.assertThat; -import static product.ImportProductsInlineSource.getImportProductsInlineRequest; -import static product.ImportProductsInlineSource.getProducts; -import static product.ImportProductsInlineSource.waitForOperationCompletion; +import static product.ImportProductsInlineSource.importProductsInlineSource; import com.google.cloud.ServiceOptions; -import com.google.cloud.retail.v2.ImportProductsRequest; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; @@ -30,7 +27,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +@RunWith(JUnit4.class) public class ImportProductsInlineSourceTest { private ByteArrayOutputStream bout; @@ -47,8 +47,7 @@ public void setUp() throws IOException, InterruptedException, ExecutionException originalPrintStream = System.out; System.setOut(out); - ImportProductsRequest importRequest = getImportProductsInlineRequest(getProducts(), branchName); - waitForOperationCompletion(importRequest); + importProductsInlineSource(branchName); } @Test