feat: Tensorboard v1 protos release (#234)

feat: Exposing a field for v1 CustomJob-Tensorboard integration.
PiperOrigin-RevId: 410304645
Source-Link: googleapis/googleapis@bd7db05
Source-Link: googleapis/googleapis-gen@66432ef
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiNjY0MzJlZjhkOTMxNGU1ZDMwNWY1NTc0N2YxYzA3ZjM3NzA4MTUxZSJ9
Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
Co-authored-by: Benjamin E. Coe <bencoe@google.com>

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/custom_job.proto

@@ -160,6 +160,17 @@ message CustomJobSpec {
   //   * AIP_TENSORBOARD_LOG_DIR = `<base_output_directory>/<trial_id>/logs/`
   GcsDestination base_output_directory = 6;
 
+  // Optional. The name of a Vertex AI [Tensorboard][google.cloud.aiplatform.v1.Tensorboard] resource to which this CustomJob
+  // will upload Tensorboard logs.
+  // Format:
+  // `projects/{project}/locations/{location}/tensorboards/{tensorboard}`
+  string tensorboard = 7 [
+    (google.api.field_behavior) = OPTIONAL,
+    (google.api.resource_reference) = {
+      type: "aiplatform.googleapis.com/Tensorboard"
+    }
+  ];
+
   // Optional. Whether you want Vertex AI to enable [interactive shell
   // access](https://cloud.google.com/vertex-ai/docs/training/monitor-debug-interactive-shell)
   // to training containers.

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/tensorboard.proto

@@ -0,0 +1,87 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.cloud.aiplatform.v1;
+
+import "google/api/field_behavior.proto";
+import "google/api/resource.proto";
+import "google/cloud/aiplatform/v1/encryption_spec.proto";
+import "google/protobuf/timestamp.proto";
+import "google/api/annotations.proto";
+
+option csharp_namespace = "Google.Cloud.AIPlatform.V1";
+option go_package = "google.golang.org/genproto/googleapis/cloud/aiplatform/v1;aiplatform";
+option java_multiple_files = true;
+option java_outer_classname = "TensorboardProto";
+option java_package = "com.google.cloud.aiplatform.v1";
+option php_namespace = "Google\\Cloud\\AIPlatform\\V1";
+option ruby_package = "Google::Cloud::AIPlatform::V1";
+
+// Tensorboard is a physical database that stores users' training metrics.
+// A default Tensorboard is provided in each region of a GCP project.
+// If needed users can also create extra Tensorboards in their projects.
+message Tensorboard {
+  option (google.api.resource) = {
+    type: "aiplatform.googleapis.com/Tensorboard"
+    pattern: "projects/{project}/locations/{location}/tensorboards/{tensorboard}"
+  };
+
+  // Output only. Name of the Tensorboard.
+  // Format:
+  // `projects/{project}/locations/{location}/tensorboards/{tensorboard}`
+  string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Required. User provided name of this Tensorboard.
+  string display_name = 2 [(google.api.field_behavior) = REQUIRED];
+
+  // Description of this Tensorboard.
+  string description = 3;
+
+  // Customer-managed encryption key spec for a Tensorboard. If set, this
+  // Tensorboard and all sub-resources of this Tensorboard will be secured by
+  // this key.
+  EncryptionSpec encryption_spec = 11;
+
+  // Output only. Consumer project Cloud Storage path prefix used to store blob data, which
+  // can either be a bucket or directory. Does not end with a '/'.
+  string blob_storage_path_prefix = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. The number of Runs stored in this Tensorboard.
+  int32 run_count = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Timestamp when this Tensorboard was created.
+  google.protobuf.Timestamp create_time = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Timestamp when this Tensorboard was last updated.
+  google.protobuf.Timestamp update_time = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // The labels with user-defined metadata to organize your Tensorboards.
+  //
+  // Label keys and values can be no longer than 64 characters
+  // (Unicode codepoints), can only contain lowercase letters, numeric
+  // characters, underscores and dashes. International characters are allowed.
+  // No more than 64 user labels can be associated with one Tensorboard
+  // (System labels are excluded).
+  //
+  // See https://goo.gl/xmQnxf for more information and examples of labels.
+  // System reserved label keys are prefixed with "aiplatform.googleapis.com/"
+  // and are immutable.
+  map<string, string> labels = 8;
+
+  // Used to perform a consistent read-modify-write updates. If not set, a blind
+  // "overwrite" update happens.
+  string etag = 9;
+}

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/tensorboard_data.proto

@@ -0,0 +1,102 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.cloud.aiplatform.v1;
+
+import "google/api/field_behavior.proto";
+import "google/cloud/aiplatform/v1/tensorboard_time_series.proto";
+import "google/protobuf/timestamp.proto";
+import "google/api/annotations.proto";
+
+option csharp_namespace = "Google.Cloud.AIPlatform.V1";
+option go_package = "google.golang.org/genproto/googleapis/cloud/aiplatform/v1;aiplatform";
+option java_multiple_files = true;
+option java_outer_classname = "TensorboardDataProto";
+option java_package = "com.google.cloud.aiplatform.v1";
+option php_namespace = "Google\\Cloud\\AIPlatform\\V1";
+option ruby_package = "Google::Cloud::AIPlatform::V1";
+
+// All the data stored in a TensorboardTimeSeries.
+message TimeSeriesData {
+  // Required. The ID of the TensorboardTimeSeries, which will become the final component
+  // of the TensorboardTimeSeries' resource name
+  string tensorboard_time_series_id = 1 [(google.api.field_behavior) = REQUIRED];
+
+  // Required. Immutable. The value type of this time series. All the values in this time series data
+  // must match this value type.
+  TensorboardTimeSeries.ValueType value_type = 2 [
+    (google.api.field_behavior) = REQUIRED,
+    (google.api.field_behavior) = IMMUTABLE
+  ];
+
+  // Required. Data points in this time series.
+  repeated TimeSeriesDataPoint values = 3 [(google.api.field_behavior) = REQUIRED];
+}
+
+// A TensorboardTimeSeries data point.
+message TimeSeriesDataPoint {
+  // Value of this time series data point.
+  oneof value {
+    // A scalar value.
+    Scalar scalar = 3;
+
+    // A tensor value.
+    TensorboardTensor tensor = 4;
+
+    // A blob sequence value.
+    TensorboardBlobSequence blobs = 5;
+  }
+
+  // Wall clock timestamp when this data point is generated by the end user.
+  google.protobuf.Timestamp wall_time = 1;
+
+  // Step index of this data point within the run.
+  int64 step = 2;
+}
+
+// One point viewable on a scalar metric plot.
+message Scalar {
+  // Value of the point at this step / timestamp.
+  double value = 1;
+}
+
+// One point viewable on a tensor metric plot.
+message TensorboardTensor {
+  // Required. Serialized form of
+  // https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor.proto
+  bytes value = 1 [(google.api.field_behavior) = REQUIRED];
+
+  // Optional. Version number of TensorProto used to serialize [value][google.cloud.aiplatform.v1.TensorboardTensor.value].
+  int32 version_number = 2 [(google.api.field_behavior) = OPTIONAL];
+}
+
+// One point viewable on a blob metric plot, but mostly just a wrapper message
+// to work around repeated fields can't be used directly within `oneof` fields.
+message TensorboardBlobSequence {
+  // List of blobs contained within the sequence.
+  repeated TensorboardBlob values = 1;
+}
+
+// One blob (e.g, image, graph) viewable on a blob metric plot.
+message TensorboardBlob {
+  // Output only. A URI safe key uniquely identifying a blob. Can be used to locate the blob
+  // stored in the Cloud Storage bucket of the consumer project.
+  string id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Optional. The bytes of the blob is not present unless it's returned by the
+  // ReadTensorboardBlobData endpoint.
+  bytes data = 2 [(google.api.field_behavior) = OPTIONAL];
+}

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/tensorboard_experiment.proto

@@ -0,0 +1,79 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.cloud.aiplatform.v1;
+
+import "google/api/field_behavior.proto";
+import "google/api/resource.proto";
+import "google/protobuf/timestamp.proto";
+import "google/api/annotations.proto";
+
+option csharp_namespace = "Google.Cloud.AIPlatform.V1";
+option go_package = "google.golang.org/genproto/googleapis/cloud/aiplatform/v1;aiplatform";
+option java_multiple_files = true;
+option java_outer_classname = "TensorboardExperimentProto";
+option java_package = "com.google.cloud.aiplatform.v1";
+option php_namespace = "Google\\Cloud\\AIPlatform\\V1";
+option ruby_package = "Google::Cloud::AIPlatform::V1";
+
+// A TensorboardExperiment is a group of TensorboardRuns, that are typically the
+// results of a training job run, in a Tensorboard.
+message TensorboardExperiment {
+  option (google.api.resource) = {
+    type: "aiplatform.googleapis.com/TensorboardExperiment"
+    pattern: "projects/{project}/locations/{location}/tensorboards/{tensorboard}/experiments/{experiment}"
+  };
+
+  // Output only. Name of the TensorboardExperiment.
+  // Format:
+  // `projects/{project}/locations/{location}/tensorboards/{tensorboard}/experiments/{experiment}`
+  string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // User provided name of this TensorboardExperiment.
+  string display_name = 2;
+
+  // Description of this TensorboardExperiment.
+  string description = 3;
+
+  // Output only. Timestamp when this TensorboardExperiment was created.
+  google.protobuf.Timestamp create_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Timestamp when this TensorboardExperiment was last updated.
+  google.protobuf.Timestamp update_time = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // The labels with user-defined metadata to organize your Datasets.
+  //
+  // Label keys and values can be no longer than 64 characters
+  // (Unicode codepoints), can only contain lowercase letters, numeric
+  // characters, underscores and dashes. International characters are allowed.
+  // No more than 64 user labels can be associated with one Dataset (System
+  // labels are excluded).
+  //
+  // See https://goo.gl/xmQnxf for more information and examples of labels.
+  // System reserved label keys are prefixed with "aiplatform.googleapis.com/"
+  // and are immutable. Following system labels exist for each Dataset:
+  // * "aiplatform.googleapis.com/dataset_metadata_schema":
+  //   - output only, its value is the
+  //   [metadata_schema's][metadata_schema_uri] title.
+  map<string, string> labels = 6;
+
+  // Used to perform consistent read-modify-write updates. If not set, a blind
+  // "overwrite" update happens.
+  string etag = 7;
+
+  // Immutable. Source of the TensorboardExperiment. Example: a custom training job.
+  string source = 8 [(google.api.field_behavior) = IMMUTABLE];
+}

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/tensorboard_run.proto

@@ -0,0 +1,82 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.cloud.aiplatform.v1;
+
+import "google/api/field_behavior.proto";
+import "google/api/resource.proto";
+import "google/protobuf/timestamp.proto";
+import "google/api/annotations.proto";
+
+option csharp_namespace = "Google.Cloud.AIPlatform.V1";
+option go_package = "google.golang.org/genproto/googleapis/cloud/aiplatform/v1;aiplatform";
+option java_multiple_files = true;
+option java_outer_classname = "TensorboardRunProto";
+option java_package = "com.google.cloud.aiplatform.v1";
+option php_namespace = "Google\\Cloud\\AIPlatform\\V1";
+option ruby_package = "Google::Cloud::AIPlatform::V1";
+
+// TensorboardRun maps to a specific execution of a training job with a given
+// set of hyperparameter values, model definition, dataset, etc
+message TensorboardRun {
+  option (google.api.resource) = {
+    type: "aiplatform.googleapis.com/TensorboardRun"
+    pattern: "projects/{project}/locations/{location}/tensorboards/{tensorboard}/experiments/{experiment}/runs/{run}"
+  };
+
+  // Output only. Name of the TensorboardRun.
+  // Format:
+  // `projects/{project}/locations/{location}/tensorboards/{tensorboard}/experiments/{experiment}/runs/{run}`
+  string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Required. User provided name of this TensorboardRun.
+  // This value must be unique among all TensorboardRuns
+  // belonging to the same parent TensorboardExperiment.
+  string display_name = 2 [(google.api.field_behavior) = REQUIRED];
+
+  // Description of this TensorboardRun.
+  string description = 3;
+
+  // Output only. Timestamp when this TensorboardRun was created.
+  google.protobuf.Timestamp create_time = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Timestamp when this TensorboardRun was last updated.
+  google.protobuf.Timestamp update_time = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // The labels with user-defined metadata to organize your TensorboardRuns.
+  //
+  // This field will be used to filter and visualize Runs in the Tensorboard UI.
+  // For example, a Vertex AI training job can set a label
+  // aiplatform.googleapis.com/training_job_id=xxxxx to all the runs created
+  // within that job. An end user can set a label experiment_id=xxxxx for all
+  // the runs produced in a Jupyter notebook. These runs can be grouped by a
+  // label value and visualized together in the Tensorboard UI.
+  //
+  // Label keys and values can be no longer than 64 characters
+  // (Unicode codepoints), can only contain lowercase letters, numeric
+  // characters, underscores and dashes. International characters are allowed.
+  // No more than 64 user labels can be associated with one TensorboardRun
+  // (System labels are excluded).
+  //
+  // See https://goo.gl/xmQnxf for more information and examples of labels.
+  // System reserved label keys are prefixed with "aiplatform.googleapis.com/"
+  // and are immutable.
+  map<string, string> labels = 8;
+
+  // Used to perform a consistent read-modify-write updates. If not set, a blind
+  // "overwrite" update happens.
+  string etag = 9;
+}