diff --git a/retail/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java b/retail/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java index 08389cf33c5..73c98d75f95 100644 --- a/retail/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java +++ b/retail/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java @@ -20,7 +20,7 @@ package events; -import com.google.cloud.ServiceOptions; +import com.google.api.gax.rpc.NotFoundException; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.retail.v2.BigQuerySource; import com.google.cloud.retail.v2.ImportMetadata; @@ -31,19 +31,22 @@ import com.google.longrunning.Operation; import com.google.longrunning.OperationsClient; import java.io.IOException; +import java.time.Instant; +import java.util.concurrent.TimeUnit; public class ImportUserEventsBigQuery { public static void main(String[] args) throws IOException, InterruptedException { // TODO(developer): Replace these variables before running the sample. - String projectId = ServiceOptions.getDefaultProjectId(); + String projectId = "your-project-id"; String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); - // TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: defaultCatalog = - // "invalid_catalog_name" + // TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: + // defaultCatalog = "invalid_catalog_name"; String datasetId = "user_events"; String tableId = "events"; - // TO CHECK ERROR HANDLING USE THE TABLE OF INVALID USER EVENTS: tableId = "events_some_invalid" + // TO CHECK ERROR HANDLING USE THE TABLE OF INVALID USER EVENTS: + // tableId = "events_some_invalid"; importUserEventsFromBigQuery(projectId, defaultCatalog, datasetId, tableId); } @@ -51,6 +54,7 @@ public static void main(String[] args) throws IOException, InterruptedException public static void importUserEventsFromBigQuery( String projectId, String defaultCatalog, String datasetId, String tableId) throws IOException, InterruptedException { + try { String dataSchema = "user_event"; @@ -73,30 +77,32 @@ public static void importUserEventsFromBigQuery( System.out.printf("Import user events from BigQuery source request: %s%n", importRequest); - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. After completing all of your requests, call - // the "close" method on the client to safely clean up any remaining background resources. + // Initialize client that will be used to send requests. This client only + // needs to be created once, and can be reused for multiple requests. After + // completing all of your requests, call the "close" method on the client to + // safely clean up any remaining background resources. try (UserEventServiceClient serviceClient = UserEventServiceClient.create()) { String operationName = serviceClient.importUserEventsCallable().call(importRequest).getName(); - System.out.printf("OperationName = %s\n", operationName); + System.out.printf("OperationName = %s%n", operationName); OperationsClient operationsClient = serviceClient.getOperationsClient(); Operation operation = operationsClient.getOperation(operationName); - while (!operation.getDone()) { + Instant deadline = Instant.now().plusSeconds(60); + + while (!operation.getDone() || Instant.now().isBefore(deadline)) { // Keep polling the operation periodically until the import task is done. - int awaitDuration = 30000; - Thread.sleep(awaitDuration); + TimeUnit.SECONDS.sleep(30); operation = operationsClient.getOperation(operationName); } if (operation.hasMetadata()) { ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class); System.out.printf( - "Number of successfully imported events: %s\n", metadata.getSuccessCount()); + "Number of successfully imported events: %s%n", metadata.getSuccessCount()); System.out.printf( - "Number of failures during the importing: %s\n", metadata.getFailureCount()); + "Number of failures during the importing: %s%n", metadata.getFailureCount()); } if (operation.hasResponse()) { @@ -107,6 +113,8 @@ public static void importUserEventsFromBigQuery( } } catch (BigQueryException e) { System.out.printf("Exception message: %s", e.getMessage()); + } catch (NotFoundException e) { + System.out.printf("Catalog name is not found.%n%s%n", e.getMessage()); } } } diff --git a/retail/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java b/retail/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java index aae1a455521..6abb57bd848 100644 --- a/retail/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java +++ b/retail/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java @@ -21,8 +21,7 @@ package events; import com.google.api.gax.rpc.InvalidArgumentException; -import com.google.cloud.ServiceOptions; -import com.google.cloud.bigquery.BigQueryException; +import com.google.api.gax.rpc.PermissionDeniedException; import com.google.cloud.retail.v2.GcsSource; import com.google.cloud.retail.v2.ImportErrorsConfig; import com.google.cloud.retail.v2.ImportMetadata; @@ -32,91 +31,99 @@ import com.google.cloud.retail.v2.UserEventServiceClient; import com.google.longrunning.Operation; import com.google.longrunning.OperationsClient; -import events.setup.EventsCreateGcsBucket; import java.io.IOException; +import java.time.Instant; +import java.util.concurrent.TimeUnit; public class ImportUserEventsGcs { public static void main(String[] args) throws IOException, InterruptedException { // TODO(developer): Replace these variables before running the sample. - String projectId = ServiceOptions.getDefaultProjectId(); + String projectId = "your-project-id"; String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); - // TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: defaultCatalog = - // "invalid_catalog_name" - String gcsEventsObject = "user_events.json"; - // TO CHECK ERROR HANDLING USE THE JSON WITH INVALID USER EVENT: gcsEventsObject = - // "user_events_some_invalid.json" - - importUserEventsFromGcs(gcsEventsObject, defaultCatalog); + // TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: + // defaultCatalog = "invalid_catalog_name"; + String bucketName = System.getenv("EVENTS_BUCKET_NAME"); + String gcsUserEventsObject = "user_events.json"; + // TO CHECK ERROR HANDLING USE THE JSON WITH INVALID USER EVENT: + // gcsUserEventsObject = "user_events_some_invalid.json"; + + importUserEventsFromGcs(defaultCatalog, bucketName, gcsUserEventsObject); } - public static void importUserEventsFromGcs(String gcsEventsObject, String defaultCatalog) + public static void importUserEventsFromGcs( + String defaultCatalog, String bucketName, String gcsUserEventsObject) throws IOException, InterruptedException { - try { - String gcsBucket = String.format("gs://%s", EventsCreateGcsBucket.getBucketName()); - String gcsErrorsBucket = String.format("%s/error", gcsBucket); - - GcsSource gcsSource = - GcsSource.newBuilder() - .addInputUris(String.format("%s/%s", gcsBucket, gcsEventsObject)) - .build(); - - UserEventInputConfig inputConfig = - UserEventInputConfig.newBuilder().setGcsSource(gcsSource).build(); - - ImportErrorsConfig errorsConfig = - ImportErrorsConfig.newBuilder().setGcsPrefix(gcsErrorsBucket).build(); - - ImportUserEventsRequest importRequest = - ImportUserEventsRequest.newBuilder() - .setParent(defaultCatalog) - .setInputConfig(inputConfig) - .setErrorsConfig(errorsConfig) - .build(); - - System.out.printf("Import user events from google cloud source request: %s%n", importRequest); - - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. After completing all of your requests, call - // the "close" method on the client to safely clean up any remaining background resources. - try (UserEventServiceClient serviceClient = UserEventServiceClient.create()) { - String operationName = - serviceClient.importUserEventsCallable().call(importRequest).getName(); - - System.out.printf("OperationName = %s\n", operationName); - - OperationsClient operationsClient = serviceClient.getOperationsClient(); - Operation operation = operationsClient.getOperation(operationName); - - while (!operation.getDone()) { - // Keep polling the operation periodically until the import task is done. - int awaitDuration = 30000; - Thread.sleep(awaitDuration); - operation = operationsClient.getOperation(operationName); - } - - if (operation.hasMetadata()) { - ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class); - System.out.printf( - "Number of successfully imported events: %s\n", metadata.getSuccessCount()); - System.out.printf( - "Number of failures during the importing: %s\n", metadata.getFailureCount()); - } - - if (operation.hasResponse()) { - ImportUserEventsResponse response = - operation.getResponse().unpack(ImportUserEventsResponse.class); - System.out.printf("Operation result: %s%n", response); - } - } catch (InvalidArgumentException e) { + String gcsBucket = String.format("gs://%s", bucketName); + String gcsErrorsBucket = String.format("%s/error", gcsBucket); + + GcsSource gcsSource = + GcsSource.newBuilder() + .addInputUris(String.format("%s/%s", gcsBucket, gcsUserEventsObject)) + .build(); + + UserEventInputConfig inputConfig = + UserEventInputConfig.newBuilder().setGcsSource(gcsSource).build(); + + System.out.println("GCS source: " + gcsSource.getInputUrisList()); + + ImportErrorsConfig errorsConfig = + ImportErrorsConfig.newBuilder().setGcsPrefix(gcsErrorsBucket).build(); + + ImportUserEventsRequest importRequest = + ImportUserEventsRequest.newBuilder() + .setParent(defaultCatalog) + .setInputConfig(inputConfig) + .setErrorsConfig(errorsConfig) + .build(); + System.out.printf("Import user events from google cloud source request: %s%n", importRequest); + + // Initialize client that will be used to send requests. This client only + // needs to be created once, and can be reused for multiple requests. After + // completing all of your requests, call the "close" method on the client to + // safely clean up any remaining background resources. + try (UserEventServiceClient serviceClient = UserEventServiceClient.create()) { + String operationName = serviceClient.importUserEventsCallable().call(importRequest).getName(); + + System.out.println("The operation was started."); + System.out.printf("OperationName = %s%n", operationName); + + OperationsClient operationsClient = serviceClient.getOperationsClient(); + Operation operation = operationsClient.getOperation(operationName); + + Instant deadline = Instant.now().plusSeconds(60); + + while (!operation.getDone() || Instant.now().isBefore(deadline)) { + System.out.println("Please wait till operation is done."); + TimeUnit.SECONDS.sleep(30); + operation = operationsClient.getOperation(operationName); + } + + if (operation.hasMetadata()) { + ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class); + System.out.printf( + "Number of successfully imported events: %s%n", metadata.getSuccessCount()); System.out.printf( - "Given GCS input path was not found. %n%s%n " - + "Please run CreateTestResources class to create resources.", - e.getMessage()); + "Number of failures during the importing: %s%n", metadata.getFailureCount()); + } else { + System.out.println("Metadata is empty."); + } + + if (operation.hasResponse()) { + ImportUserEventsResponse response = + operation.getResponse().unpack(ImportUserEventsResponse.class); + System.out.printf("Operation result: %s%n", response); + } else { + System.out.println("Operation result is empty."); } - } catch (BigQueryException e) { - System.out.printf("Exception message: %s", e.getMessage()); + } catch (InvalidArgumentException e) { + System.out.printf( + "%s%n'%s' file does not exist in the bucket. Please " + + "make sure you have followed the setting up instructions.", + e.getMessage(), gcsUserEventsObject); + } catch (PermissionDeniedException e) { + System.out.println(e.getMessage()); } } } diff --git a/retail/interactive-tutorials/src/main/java/events/ImportUserEventsInline.java b/retail/interactive-tutorials/src/main/java/events/ImportUserEventsInline.java index 693a6718f0e..44cf661eaad 100644 --- a/retail/interactive-tutorials/src/main/java/events/ImportUserEventsInline.java +++ b/retail/interactive-tutorials/src/main/java/events/ImportUserEventsInline.java @@ -21,7 +21,6 @@ package events; import com.google.api.gax.longrunning.OperationFuture; -import com.google.cloud.ServiceOptions; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.retail.v2.ImportMetadata; import com.google.cloud.retail.v2.ImportUserEventsRequest; @@ -44,7 +43,7 @@ public class ImportUserEventsInline { public static void main(String[] args) throws IOException, ExecutionException, InterruptedException { // TODO(developer): Replace these variables before running the sample. - String projectId = ServiceOptions.getDefaultProjectId(); + String projectId = "your-project-id"; String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); @@ -88,9 +87,10 @@ public static void importUserEventsFromInlineSource(String defaultCatalog) .build(); System.out.printf("Import user events from inline source request: %s%n", importRequest); - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. After completing all of your requests, call - // the "close" method on the client to safely clean up any remaining background resources. + // Initialize client that will be used to send requests. This client only + // needs to be created once, and can be reused for multiple requests. After + // completing all of your requests, call the "close" method on the client to + // safely clean up any remaining background resources. try (UserEventServiceClient userEventServiceClient = UserEventServiceClient.create()) { OperationFuture importOperation = userEventServiceClient.importUserEventsAsync(importRequest); diff --git a/retail/interactive-tutorials/src/main/java/events/PurgeUserEvent.java b/retail/interactive-tutorials/src/main/java/events/PurgeUserEvent.java index 4f5f902108a..31e3d680a67 100644 --- a/retail/interactive-tutorials/src/main/java/events/PurgeUserEvent.java +++ b/retail/interactive-tutorials/src/main/java/events/PurgeUserEvent.java @@ -23,7 +23,6 @@ import static setup.SetupCleanup.writeUserEvent; import com.google.api.gax.longrunning.OperationFuture; -import com.google.cloud.ServiceOptions; import com.google.cloud.retail.v2.PurgeMetadata; import com.google.cloud.retail.v2.PurgeUserEventsRequest; import com.google.cloud.retail.v2.PurgeUserEventsResponse; @@ -37,7 +36,7 @@ public class PurgeUserEvent { public static void main(String[] args) throws IOException, ExecutionException, InterruptedException { // TODO(developer): Replace these variables before running the sample. - String projectId = ServiceOptions.getDefaultProjectId(); + String projectId = "your-project-id"; String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); // visitorId generated randomly. @@ -50,9 +49,10 @@ public static void callPurgeUserEvents(String defaultCatalog, String visitorId) throws IOException, ExecutionException, InterruptedException { writeUserEvent(visitorId); - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. After completing all of your requests, call - // the "close" method on the client to safely clean up any remaining background resources. + // Initialize client that will be used to send requests. This client only + // needs to be created once, and can be reused for multiple requests. After + // completing all of your requests, call the "close" method on the client to + // safely clean up any remaining background resources. try (UserEventServiceClient userEventServiceClient = UserEventServiceClient.create()) { PurgeUserEventsRequest purgeUserEventsRequest = PurgeUserEventsRequest.newBuilder() diff --git a/retail/interactive-tutorials/src/main/java/events/RejoinUserEvent.java b/retail/interactive-tutorials/src/main/java/events/RejoinUserEvent.java index 9577db5177f..d97cb8fdfbf 100644 --- a/retail/interactive-tutorials/src/main/java/events/RejoinUserEvent.java +++ b/retail/interactive-tutorials/src/main/java/events/RejoinUserEvent.java @@ -24,7 +24,6 @@ import static setup.SetupCleanup.writeUserEvent; import com.google.api.gax.longrunning.OperationFuture; -import com.google.cloud.ServiceOptions; import com.google.cloud.retail.v2.RejoinUserEventsMetadata; import com.google.cloud.retail.v2.RejoinUserEventsRequest; import com.google.cloud.retail.v2.RejoinUserEventsRequest.UserEventRejoinScope; @@ -39,7 +38,7 @@ public class RejoinUserEvent { public static void main(String[] args) throws IOException, ExecutionException, InterruptedException { // TODO(developer): Replace these variables before running the sample. - String projectId = ServiceOptions.getDefaultProjectId(); + String projectId = "your-project-id"; String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); // visitorId generated randomly. @@ -52,9 +51,10 @@ public static void callRejoinUserEvents(String defaultCatalog, String visitorId) throws IOException, ExecutionException, InterruptedException { writeUserEvent(visitorId); - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. After completing all of your requests, call - // the "close" method on the client to safely clean up any remaining background resources. + // Initialize client that will be used to send requests. This client only + // needs to be created once, and can be reused for multiple requests. After + // completing all of your requests, call the "close" method on the client to + // safely clean up any remaining background resources. try (UserEventServiceClient userEventServiceClient = UserEventServiceClient.create()) { RejoinUserEventsRequest rejoinUserEventsRequest = RejoinUserEventsRequest.newBuilder() diff --git a/retail/interactive-tutorials/src/main/java/events/WriteUserEvent.java b/retail/interactive-tutorials/src/main/java/events/WriteUserEvent.java index a3455e1f09e..b4b20a59680 100644 --- a/retail/interactive-tutorials/src/main/java/events/WriteUserEvent.java +++ b/retail/interactive-tutorials/src/main/java/events/WriteUserEvent.java @@ -22,7 +22,6 @@ import static setup.SetupCleanup.purgeUserEvent; -import com.google.cloud.ServiceOptions; import com.google.cloud.retail.v2.UserEvent; import com.google.cloud.retail.v2.UserEventServiceClient; import com.google.cloud.retail.v2.WriteUserEventRequest; @@ -37,7 +36,7 @@ public class WriteUserEvent { public static void main(String[] args) throws IOException, ExecutionException, InterruptedException { // TODO(developer): Replace these variables before running the sample. - String projectId = ServiceOptions.getDefaultProjectId(); + String projectId = "your-project-id"; String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); // visitorId generated randomly. @@ -48,9 +47,10 @@ public static void main(String[] args) public static void writeUserEvent(String defaultCatalog, String visitorId) throws IOException, ExecutionException, InterruptedException { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. After completing all of your requests, call - // the "close" method on the client to safely clean up any remaining background resources. + // Initialize client that will be used to send requests. This client only + // needs to be created once, and can be reused for multiple requests. After + // completing all of your requests, call the "close" method on the client to + // safely clean up any remaining background resources. try (UserEventServiceClient userEventServiceClient = UserEventServiceClient.create()) { Timestamp timestamp = Timestamp.newBuilder().setSeconds(Instant.now().getEpochSecond()).build(); diff --git a/retail/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java b/retail/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java index 5b0cce2f38b..3bedc9f1184 100644 --- a/retail/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java +++ b/retail/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java @@ -30,16 +30,15 @@ public class EventsCreateBigQueryTable { - public static void main(String... args) throws IOException { + public static void main(String[] args) throws IOException { String dataset = "user_events"; String validEventsTable = "events"; String invalidEventsTable = "events_some_invalid"; String eventsSchemaFilePath = "src/main/resources/events_schema.json"; String validEventsSourceFile = - String.format("gs://%s/user_events.json", EventsCreateGcsBucket.getBucketName()); + String.format("gs://%s/user_events.json", System.getenv("EVENTS_BUCKET_NAME")); String invalidEventsSourceFile = - String.format( - "gs://%s/user_events_some_invalid.json", EventsCreateGcsBucket.getBucketName()); + String.format("gs://%s/user_events_some_invalid.json", System.getenv("EVENTS_BUCKET_NAME")); BufferedReader bufferedReader = new BufferedReader(new FileReader(eventsSchemaFilePath)); String jsonToString = bufferedReader.lines().collect(Collectors.joining()); diff --git a/retail/interactive-tutorials/src/main/java/init/RemoveTestResources.java b/retail/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java similarity index 67% rename from retail/interactive-tutorials/src/main/java/init/RemoveTestResources.java rename to retail/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java index c48ceff2552..05b19e33937 100644 --- a/retail/interactive-tutorials/src/main/java/init/RemoveTestResources.java +++ b/retail/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java @@ -14,36 +14,46 @@ * limitations under the License. */ -package init; +package events.setup; import static setup.SetupCleanup.deleteBucket; import static setup.SetupCleanup.deleteDataset; import com.google.api.gax.rpc.PermissionDeniedException; import com.google.cloud.ServiceOptions; -import com.google.cloud.retail.v2.DeleteProductRequest; -import com.google.cloud.retail.v2.ListProductsRequest; -import com.google.cloud.retail.v2.Product; -import com.google.cloud.retail.v2.ProductServiceClient; +import com.google.cloud.retail.v2.*; import com.google.cloud.retail.v2.ProductServiceClient.ListProductsPagedResponse; import java.io.IOException; -public class RemoveTestResources { +public class RemoveEventsResources { - public static void main(String... args) throws IOException { - // TODO(developer): Replace these variables before running the sample. + public static void main(String[] args) throws IOException { String projectId = ServiceOptions.getDefaultProjectId(); - String bucketName = System.getenv("BUCKET_NAME"); + String bucketName = System.getenv("EVENTS_BUCKET_NAME"); String branchName = String.format( "projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId); deleteBucket(bucketName); - deleteAllProducts(branchName); - deleteDataset(projectId, "products"); + deleteAllEvents(branchName); deleteDataset(projectId, "user_events"); } + public static void deleteAllEvents(String branchName) throws IOException { + System.out.println("Deleting events in process, please wait..."); + + // Initialize client that will be used to send requests. This client only + // needs to be created once, and can be reused for multiple requests. After + // completing all of your requests, call the "close" method on the client to + // safely clean up any remaining background resources. + try (UserEventServiceClient eventServiceClient = UserEventServiceClient.create()) { + PurgeUserEventsRequest purgeUserEventsRequest = + PurgeUserEventsRequest.newBuilder().setParent(branchName).build(); + eventServiceClient.purgeUserEventsAsync(purgeUserEventsRequest); + System.out.printf("Events were deleted from %s%n", branchName); + } + } + public static void deleteAllProducts(String branchName) throws IOException { System.out.println("Deleting products in process, please wait..."); diff --git a/retail/interactive-tutorials/src/main/java/events/setup/UpdateUserEventsJson.java b/retail/interactive-tutorials/src/main/java/events/setup/UpdateUserEventsJson.java index b23da9afcd6..e1fd5590338 100644 --- a/retail/interactive-tutorials/src/main/java/events/setup/UpdateUserEventsJson.java +++ b/retail/interactive-tutorials/src/main/java/events/setup/UpdateUserEventsJson.java @@ -1,46 +1,53 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package events.setup; -import java.io.BufferedReader; -import java.io.FileReader; +import java.io.BufferedWriter; import java.io.FileWriter; import java.io.IOException; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.List; -import org.json.JSONObject; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.temporal.ChronoUnit; public class UpdateUserEventsJson { - public static void main(String[] args) { - try { - String timestamp = LocalDateTime.now().minusDays(1).toString(); - String filename = "src/main/resources/user_events.json"; - updateFields(timestamp, filename); - filename = "src/main/resources/user_events_some_invalid.json"; - updateFields(timestamp, filename); - } catch (IOException e) { - e.printStackTrace(); - } + public static void main(String[] args) throws IOException { + // TODO(developer): Replace these variables before running the sample. + String filePath = "src/main/resources/user_events.json"; + + updateEventsTimestamp(filePath); } - private static void updateFields(String timestamp, String filename) throws IOException { - List newLines = new ArrayList<>(); - try (BufferedReader file = new BufferedReader(new FileReader(filename))) { - String line = file.readLine(); - String field = "eventTime"; - while (line != null) { - JSONObject object = new JSONObject(line); - object.put(field, timestamp); - newLines.add(object.toString()); - line = file.readLine(); - } - } - try (FileWriter file = new FileWriter(filename)) { - for (String event : newLines) { - file.write(event); - file.write("\n"); - } - System.out.println("Successfully updated json file!"); - } + public static void updateEventsTimestamp(String jsonFile) throws IOException { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + Timestamp yesterdayDate = Timestamp.from(Instant.now().minus(1, ChronoUnit.DAYS)); + + String json = new String(Files.readAllBytes(Paths.get(jsonFile))); + json = + json.replaceAll( + "(\"eventTime\"\\s*:\\s*\"(\\d{4}-\\d{2}-\\d{2}(T.*Z)?))", + "\"eventTime\":\"" + dateFormat.format(yesterdayDate) + ""); + + BufferedWriter writer = new BufferedWriter(new FileWriter(jsonFile)); + writer.write(json); + System.out.printf("User events file '%s' updated.%n", jsonFile); + writer.close(); } } diff --git a/retail/interactive-tutorials/src/main/java/init/CreateTestResources.java b/retail/interactive-tutorials/src/main/java/init/CreateTestResources.java deleted file mode 100644 index 5a0f6cc6072..00000000000 --- a/retail/interactive-tutorials/src/main/java/init/CreateTestResources.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package init; - -import com.google.cloud.ServiceOptions; -import com.google.cloud.retail.v2.GcsSource; -import com.google.cloud.retail.v2.ImportErrorsConfig; -import com.google.cloud.retail.v2.ImportMetadata; -import com.google.cloud.retail.v2.ImportProductsRequest; -import com.google.cloud.retail.v2.ImportProductsRequest.ReconciliationMode; -import com.google.cloud.retail.v2.ImportProductsResponse; -import com.google.cloud.retail.v2.ProductInputConfig; -import com.google.cloud.retail.v2.ProductServiceClient; -import com.google.longrunning.Operation; -import com.google.longrunning.OperationsClient; -import events.setup.EventsCreateBigQueryTable; -import events.setup.EventsCreateGcsBucket; -import java.io.IOException; -import java.util.Collections; -import product.setup.ProductsCreateBigqueryTable; -import product.setup.ProductsCreateGcsBucket; - -public class CreateTestResources { - - public static void main(String... args) throws IOException, InterruptedException { - // TODO(developer): Replace these variables before running the sample. - String projectId = ServiceOptions.getDefaultProjectId(); - String bucketName = System.getenv("BUCKET_NAME"); - String gcsBucket = String.format("gs://%s", System.getenv("BUCKET_NAME")); - String gcsErrorBucket = String.format("%s/errors", gcsBucket); - String branchName = - String.format( - "projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId); - - ProductsCreateGcsBucket.main(); - EventsCreateGcsBucket.main(); - importProductsFromGcs(bucketName, gcsErrorBucket, branchName); - ProductsCreateBigqueryTable.main(); - EventsCreateBigQueryTable.main(); - } - - public static void importProductsFromGcs( - String bucketName, String gcsErrorBucket, String branchName) - throws IOException, InterruptedException { - GcsSource gcsSource = - GcsSource.newBuilder() - .addAllInputUris( - Collections.singleton(String.format("gs://%s/%s", bucketName, "products.json"))) - .build(); - ProductInputConfig inputConfig = - ProductInputConfig.newBuilder().setGcsSource(gcsSource).build(); - System.out.println("GRS source: " + gcsSource.getInputUrisList()); - - ImportErrorsConfig errorsConfig = - ImportErrorsConfig.newBuilder().setGcsPrefix(gcsErrorBucket).build(); - ImportProductsRequest importRequest = - ImportProductsRequest.newBuilder() - .setParent(branchName) - .setReconciliationMode(ReconciliationMode.INCREMENTAL) - .setInputConfig(inputConfig) - .setErrorsConfig(errorsConfig) - .build(); - System.out.println("Import products from google cloud source request: " + importRequest); - - try (ProductServiceClient serviceClient = ProductServiceClient.create()) { - String operationName = serviceClient.importProductsCallable().call(importRequest).getName(); - System.out.printf("OperationName = %s\n", operationName); - - OperationsClient operationsClient = serviceClient.getOperationsClient(); - Operation operation = operationsClient.getOperation(operationName); - - while (!operation.getDone()) { - System.out.println("Please wait till operation is completed."); - // Keep polling the operation periodically until the import task is done. - Thread.sleep(30_000); - operation = operationsClient.getOperation(operationName); - } - - System.out.println("Import products operation is completed."); - - if (operation.hasMetadata()) { - ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class); - System.out.printf( - "Number of successfully imported products: %s\n", metadata.getSuccessCount()); - System.out.printf( - "Number of failures during the importing: %s\n", metadata.getFailureCount()); - } - - if (operation.hasResponse()) { - ImportProductsResponse response = - operation.getResponse().unpack(ImportProductsResponse.class); - System.out.printf("Operation result: %s", response); - } - } - } -} diff --git a/retail/interactive-tutorials/src/main/java/init/TEST_RESOURCES_SETUP_CLEANUP.md b/retail/interactive-tutorials/src/main/java/init/TEST_RESOURCES_SETUP_CLEANUP.md deleted file mode 100644 index 0b9df6d113a..00000000000 --- a/retail/interactive-tutorials/src/main/java/init/TEST_RESOURCES_SETUP_CLEANUP.md +++ /dev/null @@ -1,49 +0,0 @@ -# How to set up/ tear down the test resources - -## Required environment variables - -To successfully import the catalog data for tests, the following environment variables should be -set: - -- PROJECT_ID -- PROJECT_NUMBER -- BUCKET_NAME - -The Secret Manager name is set in .kokoro/presubmit/common.cfg file, SECRET_MANAGER_KEYS variable. - -## Import catalog data - -There is a JSON file with valid products prepared in the `product` directory: -`resources/products.json`. - -Run the `CreateTestResources` to perform the following actions: - -- create the GCS bucket ; -- upload the product data from `resources/products.json` file to the bucket; -- import products to the default branch of the Retail catalog; -- upload the product data from `resources/user_events.json` file to the bucket; -- create a BigQuery dataset and table `products`; -- insert products from resources/products.json to the created products table; -- create a BigQuery dataset and table `events`; -- insert user events from resources/user_events.json to the created events table; - -``` -mvn compile exec:java -Dexec.mainClass="init.CreateTestResources" -``` - -In the result 316 products should be created in the test project catalog. - -## Remove catalog data - -Run the `RemoveTestResources` to perform the following actions: - -- remove all objects from the GCS bucket ; -- remove the bucket; -- delete all products from the Retail catalog; -- remove all objects from the GCS bucket ; -- remove dataset `products` along with tables; -- remove dataset `user_events` along with tables; - -``` -mvn compile exec:java -Dexec.mainClass="init.RemoveTestResources" -``` \ No newline at end of file diff --git a/retail/interactive-tutorials/src/test/java/events/ImportUserEventsBigQueryTest.java b/retail/interactive-tutorials/src/test/java/events/ImportUserEventsBigQueryTest.java index 2f465467228..a7f9a7a6716 100644 --- a/retail/interactive-tutorials/src/test/java/events/ImportUserEventsBigQueryTest.java +++ b/retail/interactive-tutorials/src/test/java/events/ImportUserEventsBigQueryTest.java @@ -17,9 +17,10 @@ package events; import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import com.google.api.gax.rpc.InvalidArgumentException; import com.google.cloud.ServiceOptions; -import events.setup.EventsCreateBigQueryTable; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; @@ -27,7 +28,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +@RunWith(JUnit4.class) public class ImportUserEventsBigQueryTest { private ByteArrayOutputStream bout; @@ -35,28 +39,60 @@ public class ImportUserEventsBigQueryTest { @Before public void setUp() throws IOException, InterruptedException, ExecutionException { + bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + originalPrintStream = System.out; + System.setOut(out); + } + + @Test + public void testValidImportUserEventsBigQuery() throws IOException, InterruptedException { String projectId = ServiceOptions.getDefaultProjectId(); String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); String datasetId = "user_events"; String tableId = "events"; - bout = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(bout); - originalPrintStream = System.out; - System.setOut(out); - EventsCreateBigQueryTable.main(); ImportUserEventsBigQuery.importUserEventsFromBigQuery( projectId, defaultCatalog, datasetId, tableId); + + String outputResult = bout.toString(); + + assertThat(outputResult).contains("Import user events from BigQuery source request"); + assertThat(outputResult).contains("table_id: \"events\""); + assertThat(outputResult).contains("Number of successfully imported events:"); + assertThat(outputResult).contains("Number of failures during the importing: 0"); } @Test - public void testImportUserEventsBigQuery() { + public void testInvalidImportUserEventsBigQuery() throws IOException, InterruptedException { + String projectId = ServiceOptions.getDefaultProjectId(); + String defaultCatalog = + String.format("projects/%s/locations/global/catalogs/invalid_catalog_name", projectId); + String datasetId = "user_events"; + String tableId = "events_some_invalid"; + + ImportUserEventsBigQuery.importUserEventsFromBigQuery( + projectId, defaultCatalog, datasetId, tableId); + String outputResult = bout.toString(); - assertThat(outputResult).contains("Import user events from BigQuery source request"); - assertThat(outputResult).contains("Number of successfully imported events"); - assertThat(outputResult).contains("Number of failures during the importing"); + assertThat(outputResult).contains("table_id: \"events_some_invalid\""); + assertThat(outputResult).contains("Catalog name is not found."); + } + + @Test + public void testInvalidDefaultCatalogBigQuery() { + String projectId = ServiceOptions.getDefaultProjectId(); + String defaultCatalog = "invalid_catalog_name"; + String datasetId = "user_events"; + String tableId = "events"; + + assertThrows( + InvalidArgumentException.class, + () -> + ImportUserEventsBigQuery.importUserEventsFromBigQuery( + projectId, defaultCatalog, datasetId, tableId)); } @After diff --git a/retail/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java b/retail/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java index 78db5ff99d1..8ee3488f9f9 100644 --- a/retail/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java +++ b/retail/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java @@ -17,6 +17,7 @@ package events; import static com.google.common.truth.Truth.assertThat; +import static events.ImportUserEventsGcs.importUserEventsFromGcs; import com.google.cloud.ServiceOptions; import events.setup.EventsCreateGcsBucket; @@ -26,7 +27,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +@RunWith(JUnit4.class) public class ImportUserEventsGcsTest { private ByteArrayOutputStream bout; @@ -39,22 +43,23 @@ public void setUp() throws IOException, InterruptedException { String projectId = ServiceOptions.getDefaultProjectId(); String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); + String bucketName = EventsCreateGcsBucket.getBucketName(); String gcsEventsObject = "user_events.json"; bout = new ByteArrayOutputStream(); PrintStream out = new PrintStream(bout); originalPrintStream = System.out; System.setOut(out); - ImportUserEventsGcs.importUserEventsFromGcs(gcsEventsObject, defaultCatalog); + importUserEventsFromGcs(defaultCatalog, bucketName, gcsEventsObject); } @Test - public void testImportUserEventsGcs() { + public void testValidImportUserEventsGcs() { String outputResult = bout.toString(); assertThat(outputResult).contains("Import user events from google cloud source request"); - assertThat(outputResult).contains("Number of successfully imported events"); - assertThat(outputResult).contains("Number of failures during the importing"); + assertThat(outputResult).contains("Number of successfully imported events:"); + assertThat(outputResult).contains("Number of failures during the importing: 0"); } @After diff --git a/retail/interactive-tutorials/src/test/java/events/ImportUserEventsInlineTest.java b/retail/interactive-tutorials/src/test/java/events/ImportUserEventsInlineTest.java index f6db8372b88..f182a5ad277 100644 --- a/retail/interactive-tutorials/src/test/java/events/ImportUserEventsInlineTest.java +++ b/retail/interactive-tutorials/src/test/java/events/ImportUserEventsInlineTest.java @@ -26,7 +26,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +@RunWith(JUnit4.class) public class ImportUserEventsInlineTest { private ByteArrayOutputStream bout; diff --git a/retail/interactive-tutorials/src/test/java/events/PurgeUserEventTest.java b/retail/interactive-tutorials/src/test/java/events/PurgeUserEventTest.java index 7c4eb4460a7..13e6897021e 100644 --- a/retail/interactive-tutorials/src/test/java/events/PurgeUserEventTest.java +++ b/retail/interactive-tutorials/src/test/java/events/PurgeUserEventTest.java @@ -27,7 +27,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +@RunWith(JUnit4.class) public class PurgeUserEventTest { private ByteArrayOutputStream bout; diff --git a/retail/interactive-tutorials/src/test/java/events/RejoinUserEventTest.java b/retail/interactive-tutorials/src/test/java/events/RejoinUserEventTest.java index fd478dff8c2..446ab2d9633 100644 --- a/retail/interactive-tutorials/src/test/java/events/RejoinUserEventTest.java +++ b/retail/interactive-tutorials/src/test/java/events/RejoinUserEventTest.java @@ -27,7 +27,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +@RunWith(JUnit4.class) public class RejoinUserEventTest { private ByteArrayOutputStream bout; diff --git a/retail/interactive-tutorials/src/test/java/events/WriteUserEventTest.java b/retail/interactive-tutorials/src/test/java/events/WriteUserEventTest.java index ebff18caed1..46f5a652c90 100644 --- a/retail/interactive-tutorials/src/test/java/events/WriteUserEventTest.java +++ b/retail/interactive-tutorials/src/test/java/events/WriteUserEventTest.java @@ -27,7 +27,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +@RunWith(JUnit4.class) public class WriteUserEventTest { private ByteArrayOutputStream bout; diff --git a/retail/interactive-tutorials/src/test/java/resources/user_events.json b/retail/interactive-tutorials/src/test/java/resources/user_events.json new file mode 100644 index 00000000000..76d7c0df262 --- /dev/null +++ b/retail/interactive-tutorials/src/test/java/resources/user_events.json @@ -0,0 +1,4 @@ +{"eventType":"home-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2022-04-13T10:27:42+00:00"} +{"eventType":"search","visitorId":"bjbs_group1_visitor1","eventTime":"2022-04-13T10:27:42+00:00","searchQuery":"RockerJeans teenagers blue jeans"} +{"eventType":"search","visitorId":"bjbs_group1_visitor1","eventTime":"2022-04-13T10:27:42+00:00","searchQuery":"SocksUnlimited teenagers black socks"} +{"eventType":"detail-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2022-04-13T10:27:42+00:00","productDetails":{"product":{"id":"GGCOGAEC100616"},"quantity":3}} \ No newline at end of file diff --git a/retail/interactive-tutorials/src/test/java/resources/user_events_some_invalid.json b/retail/interactive-tutorials/src/test/java/resources/user_events_some_invalid.json new file mode 100644 index 00000000000..27e00a0c878 --- /dev/null +++ b/retail/interactive-tutorials/src/test/java/resources/user_events_some_invalid.json @@ -0,0 +1,4 @@ +{"eventType":"home-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2022-04-13T10:27:42+00:00"} +{"eventType":"invalid","visitorId":"bjbs_group1_visitor1","eventTime":"2022-04-13T10:27:42+00:00","searchQuery":"RockerJeans teenagers blue jeans"} +{"eventType":"search","visitorId":"bjbs_group1_visitor1","eventTime":"2022-04-13T10:27:42+00:00","searchQuery":"SocksUnlimited teenagers black socks"} +{"eventType":"detail-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2022-04-13T10:27:42+00:00","productDetails":{"product":{"id":"GGCOGAEC100616"},"quantity":3}}