feat: add prediction service RPC RawPredict to aiplatform_v1beta1 fea…

…t: add tensorboard service RPCs to aiplatform_v1beta1: BatchCreateTensorboardRuns, BatchCreateTensorboardTimeSeries, WriteTensorboardExperimentData feat: add model_deployment_monitori... (#196)

feat: add tensorboard service RPCs to aiplatform_v1beta1: BatchCreateTensorboardRuns, BatchCreateTensorboardTimeSeries, WriteTensorboardExperimentData
feat: add model_deployment_monitoring_job to Endpoint in aiplatform_v1beta1
feat: add deployment_group to DeployedIndex in aiplatform_v1beta1
feat: add ModelEvaluationExplanationSpec in aiplatform_v1beta1
Committer: @dizcology
PiperOrigin-RevId: 393890669
Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
Co-authored-by: Benjamin E. Coe <bencoe@google.com>

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/annotation.proto

@@ -18,6 +18,7 @@ package google.cloud.aiplatform.v1;
 
 import "google/api/field_behavior.proto";
 import "google/api/resource.proto";
+
 import "google/cloud/aiplatform/v1/user_action_reference.proto";
 import "google/protobuf/struct.proto";
 import "google/protobuf/timestamp.proto";

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/artifact.proto

@@ -87,4 +87,25 @@ message Artifact {
   // clients (such as Vertex Pipelines), and the system does not prescribe
   // or check the validity of state transitions.
   State state = 13;
+
+  // The title of the schema describing the metadata.
+  //
+  // Schema title and version is expected to be registered in earlier Create
+  // Schema calls. And both are used together as unique identifiers to identify
+  // schemas within the local metadata store.
+  string schema_title = 14;
+
+  // The version of the schema in schema_name to use.
+  //
+  // Schema title and version is expected to be registered in earlier Create
+  // Schema calls. And both are used together as unique identifiers to identify
+  // schemas within the local metadata store.
+  string schema_version = 15;
+
+  // Properties of the Artifact.
+  // The size of this field should not exceed 200KB.
+  google.protobuf.Struct metadata = 16;
+
+  // Description of the Artifact
+  string description = 17;
 }

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/batch_prediction_job.proto

@@ -20,10 +20,12 @@ import "google/api/field_behavior.proto";
 import "google/api/resource.proto";
 import "google/cloud/aiplatform/v1/completion_stats.proto";
 import "google/cloud/aiplatform/v1/encryption_spec.proto";
+import "google/cloud/aiplatform/v1/explanation.proto";
 import "google/cloud/aiplatform/v1/io.proto";
 import "google/cloud/aiplatform/v1/job_state.proto";
 import "google/cloud/aiplatform/v1/machine_resources.proto";
 import "google/cloud/aiplatform/v1/manual_batch_tuning_parameters.proto";
+import "google/cloud/aiplatform/v1/model_monitoring.proto";
 import "google/protobuf/struct.proto";
 import "google/protobuf/timestamp.proto";
 import "google/rpc/status.proto";
@@ -199,6 +201,32 @@ message BatchPredictionJob {
   // the tuning itself).
   ManualBatchTuningParameters manual_batch_tuning_parameters = 8 [(google.api.field_behavior) = IMMUTABLE];
 
+  // Generate explanation with the batch prediction results.
+  //
+  // When set to `true`, the batch prediction output changes based on the
+  // `predictions_format` field of the
+  // [BatchPredictionJob.output_config][google.cloud.aiplatform.v1.BatchPredictionJob.output_config] object:
+  //
+  //  * `bigquery`: output includes a column named `explanation`. The value
+  //    is a struct that conforms to the [Explanation][google.cloud.aiplatform.v1.Explanation] object.
+  //  * `jsonl`: The JSON objects on each line include an additional entry
+  //    keyed `explanation`. The value of the entry is a JSON object that
+  //    conforms to the [Explanation][google.cloud.aiplatform.v1.Explanation] object.
+  //  * `csv`: Generating explanations for CSV format is not supported.
+  //
+  // If this field is set to true, either the [Model.explanation_spec][google.cloud.aiplatform.v1.Model.explanation_spec] or
+  // [explanation_spec][google.cloud.aiplatform.v1.BatchPredictionJob.explanation_spec] must be populated.
+  bool generate_explanation = 23;
+
+  // Explanation configuration for this BatchPredictionJob. Can be
+  // specified only if [generate_explanation][google.cloud.aiplatform.v1.BatchPredictionJob.generate_explanation] is set to `true`.
+  //
+  // This value overrides the value of [Model.explanation_spec][google.cloud.aiplatform.v1.Model.explanation_spec]. All fields of
+  // [explanation_spec][google.cloud.aiplatform.v1.BatchPredictionJob.explanation_spec] are optional in the request. If a field of the
+  // [explanation_spec][google.cloud.aiplatform.v1.BatchPredictionJob.explanation_spec] object is not populated, the corresponding field of
+  // the [Model.explanation_spec][google.cloud.aiplatform.v1.Model.explanation_spec] object is inherited.
+  ExplanationSpec explanation_spec = 25;
+
   // Output only. Information further describing the output of this job.
   OutputInfo output_info = 9 [(google.api.field_behavior) = OUTPUT_ONLY];
 

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/context.proto

@@ -72,4 +72,25 @@ message Context {
       type: "aiplatform.googleapis.com/Context"
     }
   ];
+
+  // The title of the schema describing the metadata.
+  //
+  // Schema title and version is expected to be registered in earlier Create
+  // Schema calls. And both are used together as unique identifiers to identify
+  // schemas within the local metadata store.
+  string schema_title = 13;
+
+  // The version of the schema in schema_name to use.
+  //
+  // Schema title and version is expected to be registered in earlier Create
+  // Schema calls. And both are used together as unique identifiers to identify
+  // schemas within the local metadata store.
+  string schema_version = 14;
+
+  // Properties of the Context.
+  // The size of this field should not exceed 200KB.
+  google.protobuf.Struct metadata = 15;
+
+  // Description of the Context
+  string description = 16;
 }

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/custom_job.proto

@@ -91,6 +91,19 @@ message CustomJob {
   // then all resources created by the CustomJob will be encrypted with the
   // provided encryption key.
   EncryptionSpec encryption_spec = 12;
+
+  // Output only. URIs for accessing [interactive
+  // shells](https://cloud.google.com/vertex-ai/docs/training/monitor-debug-interactive-shell)
+  // (one URI for each training node). Only available if
+  // [job_spec.enable_web_access][google.cloud.aiplatform.v1.CustomJobSpec.enable_web_access] is `true`.
+  //
+  // The keys are names of each node in the training job; for example,
+  // `workerpool0-0` for the primary node, `workerpool1-0` for the first node in
+  // the second worker pool, and `workerpool1-1` for the second node in the
+  // second worker pool.
+  //
+  // The values are the URIs for each node's interactive shell.
+  map<string, string> web_access_uris = 16 [(google.api.field_behavior) = OUTPUT_ONLY];
 }
 
 // Represents the spec of a CustomJob.
@@ -105,7 +118,7 @@ message CustomJobSpec {
 
   // Specifies the service account for workload run-as account.
   // Users submitting jobs must have act-as permission on this run-as account.
-  // If unspecified, the [AI Platform Custom Code Service
+  // If unspecified, the [Vertex AI Custom Code Service
   // Agent](https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents)
   // for the CustomJob's project is used.
   string service_account = 4;
@@ -146,6 +159,15 @@ message CustomJobSpec {
   //   * AIP_CHECKPOINT_DIR = `<base_output_directory>/<trial_id>/checkpoints/`
   //   * AIP_TENSORBOARD_LOG_DIR = `<base_output_directory>/<trial_id>/logs/`
   GcsDestination base_output_directory = 6;
+
+  // Optional. Whether you want Vertex AI to enable [interactive shell
+  // access](https://cloud.google.com/vertex-ai/docs/training/monitor-debug-interactive-shell)
+  // to training containers.
+  //
+  // If set to `true`, you can access interactive shells at the URIs given
+  // by [CustomJob.web_access_uris][google.cloud.aiplatform.v1.CustomJob.web_access_uris] or [Trial.web_access_uris][google.cloud.aiplatform.v1.Trial.web_access_uris] (within
+  // [HyperparameterTuningJob.trials][google.cloud.aiplatform.v1.HyperparameterTuningJob.trials]).
+  bool enable_web_access = 10 [(google.api.field_behavior) = OPTIONAL];
 }
 
 // Represents the spec of a worker pool in a job.
@@ -186,6 +208,7 @@ message ContainerSpec {
   repeated string args = 3;
 
   // Environment variables to be passed to the container.
+  // Maximum limit is 100.
   repeated EnvVar env = 4;
 }
 
@@ -211,6 +234,7 @@ message PythonPackageSpec {
   repeated string args = 4;
 
   // Environment variables to be passed to the python module.
+  // Maximum limit is 100.
   repeated EnvVar env = 5;
 }
 

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/data_item.proto

@@ -18,6 +18,7 @@ package google.cloud.aiplatform.v1;
 
 import "google/api/field_behavior.proto";
 import "google/api/resource.proto";
+
 import "google/protobuf/struct.proto";
 import "google/protobuf/timestamp.proto";
 import "google/api/annotations.proto";

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/dataset.proto

@@ -47,6 +47,9 @@ message Dataset {
   // characters.
   string display_name = 2 [(google.api.field_behavior) = REQUIRED];
 
+  // Optional. The description of the Dataset.
+  string description = 16 [(google.api.field_behavior) = OPTIONAL];
+
   // Required. Points to a YAML file stored on Google Cloud Storage describing additional
   // information about the Dataset.
   // The schema is defined as an OpenAPI 3.0.2 Schema Object.

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/dataset_service.proto

@@ -37,6 +37,8 @@ option java_package = "com.google.cloud.aiplatform.v1";
 option php_namespace = "Google\\Cloud\\AIPlatform\\V1";
 option ruby_package = "Google::Cloud::AIPlatform::V1";
 
+// The service that handles the CRUD of Vertex AI Dataset and its child
+// resources.
 service DatasetService {
   option (google.api.default_host) = "aiplatform.googleapis.com";
   option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform";

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/deployed_index_ref.proto

@@ -0,0 +1,43 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.cloud.aiplatform.v1;
+
+import "google/api/field_behavior.proto";
+import "google/api/resource.proto";
+import "google/api/annotations.proto";
+
+option csharp_namespace = "Google.Cloud.AIPlatform.V1";
+option go_package = "google.golang.org/genproto/googleapis/cloud/aiplatform/v1;aiplatform";
+option java_multiple_files = true;
+option java_outer_classname = "DeployedIndexRefProto";
+option java_package = "com.google.cloud.aiplatform.v1";
+option php_namespace = "Google\\Cloud\\AIPlatform\\V1";
+option ruby_package = "Google::Cloud::AIPlatform::V1";
+
+// Points to a DeployedIndex.
+message DeployedIndexRef {
+  // Immutable. A resource name of the IndexEndpoint.
+  string index_endpoint = 1 [
+    (google.api.field_behavior) = IMMUTABLE,
+    (google.api.resource_reference) = {
+      type: "aiplatform.googleapis.com/IndexEndpoint"
+    }
+  ];
+
+  // Immutable. The ID of the DeployedIndex in the above IndexEndpoint.
+  string deployed_index_id = 2 [(google.api.field_behavior) = IMMUTABLE];
+}

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/endpoint.proto

@@ -19,7 +19,10 @@ package google.cloud.aiplatform.v1;
 import "google/api/field_behavior.proto";
 import "google/api/resource.proto";
 import "google/cloud/aiplatform/v1/encryption_spec.proto";
+import "google/cloud/aiplatform/v1/explanation.proto";
 import "google/cloud/aiplatform/v1/machine_resources.proto";
+import "google/cloud/aiplatform/v1/model_deployment_monitoring_job.proto";
+import "google/cloud/aiplatform/v1/model_monitoring.proto";
 import "google/protobuf/timestamp.proto";
 import "google/api/annotations.proto";
 
@@ -88,6 +91,17 @@ message Endpoint {
   // Endpoint and all sub-resources of this Endpoint will be secured by
   // this key.
   EncryptionSpec encryption_spec = 10;
+
+  // Output only. Resource name of the Model Monitoring job associated with this Endpoint
+  // if monitoring is enabled by [CreateModelDeploymentMonitoringJob][].
+  // Format:
+  // `projects/{project}/locations/{location}/modelDeploymentMonitoringJobs/{model_deployment_monitoring_job}`
+  string model_deployment_monitoring_job = 14 [
+    (google.api.field_behavior) = OUTPUT_ONLY,
+    (google.api.resource_reference) = {
+      type: "aiplatform.googleapis.com/ModelDeploymentMonitoringJob"
+    }
+  ];
 }
 
 // A deployment of a Model. Endpoints contain one or more DeployedModels.
@@ -102,8 +116,8 @@ message DeployedModel {
     // that need a higher degree of manual configuration.
     DedicatedResources dedicated_resources = 7;
 
-    // A description of resources that to large degree are decided by AI
-    // Platform, and require only a modest additional configuration.
+    // A description of resources that to large degree are decided by Vertex
+    // AI, and require only a modest additional configuration.
     AutomaticResources automatic_resources = 8;
   }
 
@@ -126,6 +140,17 @@ message DeployedModel {
   // Output only. Timestamp when the DeployedModel was created.
   google.protobuf.Timestamp create_time = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
 
+  // Explanation configuration for this DeployedModel.
+  //
+  // When deploying a Model using [EndpointService.DeployModel][google.cloud.aiplatform.v1.EndpointService.DeployModel], this value
+  // overrides the value of [Model.explanation_spec][google.cloud.aiplatform.v1.Model.explanation_spec]. All fields of
+  // [explanation_spec][google.cloud.aiplatform.v1.DeployedModel.explanation_spec] are optional in the request. If a field of
+  // [explanation_spec][google.cloud.aiplatform.v1.DeployedModel.explanation_spec] is not populated, the value of the same field of
+  // [Model.explanation_spec][google.cloud.aiplatform.v1.Model.explanation_spec] is inherited. If the corresponding
+  // [Model.explanation_spec][google.cloud.aiplatform.v1.Model.explanation_spec] is not populated, all fields of the
+  // [explanation_spec][google.cloud.aiplatform.v1.DeployedModel.explanation_spec] will be used for the explanation configuration.
+  ExplanationSpec explanation_spec = 9;
+
   // The service account that the DeployedModel's container runs as. Specify the
   // email address of the service account. If this service account is not
   // specified, the container runs as a service account that doesn't have access

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/endpoint_service.proto

@@ -33,6 +33,7 @@ option java_package = "com.google.cloud.aiplatform.v1";
 option php_namespace = "Google\\Cloud\\AIPlatform\\V1";
 option ruby_package = "Google::Cloud::AIPlatform::V1";
 
+// A service for managing Vertex AI's Endpoints.
 service EndpointService {
   option (google.api.default_host) = "aiplatform.googleapis.com";
   option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform";

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/execution.proto

@@ -93,4 +93,25 @@ message Execution {
 
   // Output only. Timestamp when this Execution was last updated.
   google.protobuf.Timestamp update_time = 12 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // The title of the schema describing the metadata.
+  //
+  // Schema title and version is expected to be registered in earlier Create
+  // Schema calls. And both are used together as unique identifiers to identify
+  // schemas within the local metadata store.
+  string schema_title = 13;
+
+  // The version of the schema in `schema_title` to use.
+  //
+  // Schema title and version is expected to be registered in earlier Create
+  // Schema calls. And both are used together as unique identifiers to identify
+  // schemas within the local metadata store.
+  string schema_version = 14;
+
+  // Properties of the Execution.
+  // The size of this field should not exceed 200KB.
+  google.protobuf.Struct metadata = 15;
+
+  // Description of the Execution
+  string description = 16;
 }