From 581d78f0e5bbf0237fea25673e4b12b61e2f574c Mon Sep 17 00:00:00 2001 From: iperetz-goo Date: Tue, 22 Nov 2022 15:27:39 +0200 Subject: [PATCH 1/4] Add desired state field to make sure PC is created in CREATED - b/260057913 --- mmv1/products/datastream/terraform.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index a9734e32b22f..23d837b26bb4 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -89,3 +89,10 @@ overrides: !ruby/object:Overrides::ResourceOverrides vars: private_connection_id: "my-connection" network_name: "my-network" + virtual_fields: + - !ruby/object:Api::Type::Enum + name: 'desired_state' + description: | + Desired state of the PrivateConnection. + values: + - :CREATED From b3c543ce6414456b8c3bdc8a287b5ecf5789c308 Mon Sep 17 00:00:00 2001 From: iperetz-goo Date: Wed, 23 Nov 2022 10:05:07 +0200 Subject: [PATCH 2/4] Revert virtual field --- mmv1/products/datastream/terraform.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index 23d837b26bb4..a9734e32b22f 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -89,10 +89,3 @@ overrides: !ruby/object:Overrides::ResourceOverrides vars: private_connection_id: "my-connection" network_name: "my-network" - virtual_fields: - - !ruby/object:Api::Type::Enum - name: 'desired_state' - description: | - Desired state of the PrivateConnection. - values: - - :CREATED From 9e3c0785cf255e1c675cc3d2617ba01d3e087327 Mon Sep 17 00:00:00 2001 From: iperetz-goo Date: Thu, 26 Jan 2023 10:41:17 +0200 Subject: [PATCH 3/4] recreate PR --- mmv1/products/datastream/api.yaml | 1968 +++++++++++------ mmv1/products/datastream/terraform.yaml | 191 +- .../examples/datastream_stream_oracle.tf.erb | 86 + .../datastream_stream_postgresql.tf.erb | 85 + .../post_create/datastream_stream.go.erb | 9 +- .../terraform/utils/datastream_operation.go | 63 +- 6 files changed, 1612 insertions(+), 790 deletions(-) create mode 100644 mmv1/templates/terraform/examples/datastream_stream_oracle.tf.erb create mode 100644 mmv1/templates/terraform/examples/datastream_stream_postgresql.tf.erb diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index cd10ca264b18..df71609f1cb6 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -14,15 +14,15 @@ --- !ruby/object:Api::Product name: Datastream versions: - - !ruby/object:Api::Product::Version - name: ga - base_url: https://datastream.googleapis.com/v1/ +- !ruby/object:Api::Product::Version + name: ga + base_url: https://datastream.googleapis.com/v1/ scopes: - - https://www.googleapis.com/auth/cloud-platform +- https://www.googleapis.com/auth/cloud-platform apis_required: - - !ruby/object:Api::Product::ApiReference - name: Datastream API - url: https://console.cloud.google.com/apis/library/datastream.googleapis.com +- !ruby/object:Api::Product::ApiReference + name: Datastream API + url: https://console.cloud.google.com/apis/library/datastream.googleapis.com async: !ruby/object:Api::OpAsync operation: !ruby/object:Api::OpAsync::Operation path: 'name' @@ -35,787 +35,1355 @@ async: !ruby/object:Api::OpAsync path: 'done' complete: true allowed: - - true - - false + - true + - false error: !ruby/object:Api::OpAsync::Error path: 'error' message: 'message' objects: - - !ruby/object:Api::Resource - name: 'ConnectionProfile' - base_url: "projects/{{project}}/locations/{{location}}/connectionProfiles" - create_url: "projects/{{project}}/locations/{{location}}/connectionProfiles?connectionProfileId={{connection_profile_id}}" - self_link: "projects/{{project}}/locations/{{location}}/connectionProfiles/{{connection_profile_id}}" - update_verb: :PATCH - update_mask: true - references: !ruby/object:Api::Resource::ReferenceLinks - guides: - 'Official Documentation': 'https://cloud.google.com/datastream/docs/create-connection-profiles' - api: 'https://cloud.google.com/datastream/docs/reference/rest/v1/projects.locations.connectionProfiles' +- !ruby/object:Api::Resource + name: 'ConnectionProfile' + base_url: "projects/{{project}}/locations/{{location}}/connectionProfiles" + create_url: "projects/{{project}}/locations/{{location}}/connectionProfiles?connectionProfileId={{connection_profile_id}}" + self_link: "projects/{{project}}/locations/{{location}}/connectionProfiles/{{connection_profile_id}}" + update_verb: :PATCH + update_mask: true + references: !ruby/object:Api::Resource::ReferenceLinks + guides: + 'Official Documentation': 'https://cloud.google.com/datastream/docs/create-connection-profiles' + api: 'https://cloud.google.com/datastream/docs/reference/rest/v1/projects.locations.connectionProfiles' + description: | + A set of reusable connection configurations to be used as a source or destination for a stream. + parameters: + - !ruby/object:Api::Type::String + name: connectionProfileId + description: |- + The connection profile identifier. + required: true + input: true + url_param_only: true + - !ruby/object:Api::Type::String + name: 'location' + description: | + The name of the location this connection profile is located in. + required: true + input: true + url_param_only: true + properties: + - !ruby/object:Api::Type::String + name: 'name' + output: true + description: The resource's name. + - !ruby/object:Api::Type::KeyValuePairs + name: 'labels' + description: Labels. + - !ruby/object:Api::Type::String + name: 'displayName' + required: true + description: Display name. + - !ruby/object:Api::Type::NestedObject + name: 'oracleProfile' + exactly_one_of: + - oracle_profile + - gcs_profile + - mysql_profile + - bigquery_profile + - postgresql_profile + description: | + Oracle database profile. + properties: + - !ruby/object:Api::Type::String + name: 'hostname' + required: true + description: | + Hostname for the Oracle connection. + - !ruby/object:Api::Type::Integer + name: 'port' + default_value: 1521 + description: | + Port for the Oracle connection. + - !ruby/object:Api::Type::String + name: 'username' + required: true + description: | + Username for the Oracle connection. + - !ruby/object:Api::Type::String + name: 'password' + required: true + description: | + Password for the Oracle connection. + - !ruby/object:Api::Type::String + name: 'databaseService' + required: true + description: | + Database for the Oracle connection. + - !ruby/object:Api::Type::KeyValuePairs + name: 'connectionAttributes' + description: Connection string attributes + - !ruby/object:Api::Type::NestedObject + name: 'gcsProfile' + exactly_one_of: + - oracle_profile + - gcs_profile + - mysql_profile + - bigquery_profile + - postgresql_profile + description: | + Cloud Storage bucket profile. + properties: + - !ruby/object:Api::Type::String + name: 'bucket' + required: true + description: | + The Cloud Storage bucket name. + - !ruby/object:Api::Type::String + name: 'rootPath' + description: | + The root path inside the Cloud Storage bucket. + - !ruby/object:Api::Type::NestedObject + name: 'mysqlProfile' + exactly_one_of: + - oracle_profile + - gcs_profile + - mysql_profile + - bigquery_profile + - postgresql_profile description: | - A set of reusable connection configurations to be used as a source or destination for a stream. - parameters: + MySQL database profile. + properties: + - !ruby/object:Api::Type::String + name: 'hostname' + required: true + description: | + Hostname for the MySQL connection. + - !ruby/object:Api::Type::Integer + name: 'port' + default_value: 3306 + description: | + Port for the MySQL connection. + - !ruby/object:Api::Type::String + name: 'username' + required: true + description: | + Username for the MySQL connection. + - !ruby/object:Api::Type::String + name: 'password' + required: true + input: true + description: | + Password for the MySQL connection. + - !ruby/object:Api::Type::NestedObject + name: 'sslConfig' + description: | + SSL configuration for the MySQL connection. + properties: - !ruby/object:Api::Type::String - name: connectionProfileId - description: |- - The connection profile identifier. - required: true + name: 'clientKey' input: true - url_param_only: true - - !ruby/object:Api::Type::String - name: 'location' description: | - The name of the location this connection profile is located in. - required: true - input: true - url_param_only: true - properties: + PEM-encoded private key associated with the Client Certificate. + If this field is used then the 'client_certificate' and the + 'ca_certificate' fields are mandatory. + - !ruby/object:Api::Type::Boolean + name: 'clientKeySet' + output: true + description: | + Indicates whether the clientKey field is set. - !ruby/object:Api::Type::String - name: 'name' + name: 'clientCertificate' + input: true + description: | + PEM-encoded certificate that will be used by the replica to + authenticate against the source database server. If this field + is used then the 'clientKey' and the 'caCertificate' fields are + mandatory. + - !ruby/object:Api::Type::Boolean + name: 'clientCertificateSet' output: true - description: The resource's name. - - !ruby/object:Api::Type::KeyValuePairs - name: 'labels' - description: Labels. + description: | + Indicates whether the clientCertificate field is set. - !ruby/object:Api::Type::String - name: 'displayName' - required: true - description: Display name. + name: 'caCertificate' + input: true + description: | + PEM-encoded certificate of the CA that signed the source database + server's certificate. + - !ruby/object:Api::Type::Boolean + name: 'caCertificateSet' + output: true + description: | + Indicates whether the clientKey field is set. + - !ruby/object:Api::Type::NestedObject + name: 'bigqueryProfile' + send_empty_value: true + allow_empty_object: true + exactly_one_of: + - oracle_profile + - gcs_profile + - mysql_profile + - bigquery_profile + - postgresql_profile + description: | + BigQuery warehouse profile. + properties: [] + - !ruby/object:Api::Type::NestedObject + name: 'postgresqlProfile' + exactly_one_of: + - oracle_profile + - gcs_profile + - mysql_profile + - bigquery_profile + - postgresql_profile + description: | + PostgreSQL database profile. + properties: + - !ruby/object:Api::Type::String + name: 'hostname' + required: true + description: | + Hostname for the PostgreSQL connection. + - !ruby/object:Api::Type::Integer + name: 'port' + default_value: 5432 + description: | + Port for the PostgreSQL connection. + - !ruby/object:Api::Type::String + name: 'username' + required: true + description: | + Username for the PostgreSQL connection. + - !ruby/object:Api::Type::String + name: 'password' + required: true + description: | + Password for the PostgreSQL connection. + - !ruby/object:Api::Type::String + name: 'database' + required: true + description: | + Database for the PostgreSQL connection. + - !ruby/object:Api::Type::NestedObject + name: 'forwardSshConnectivity' + description: | + Forward SSH tunnel connectivity. + conflicts: + - private_connectivity + properties: + - !ruby/object:Api::Type::String + name: 'hostname' + required: true + description: | + Hostname for the SSH tunnel. + - !ruby/object:Api::Type::String + name: 'username' + required: true + description: | + Username for the SSH tunnel. + - !ruby/object:Api::Type::Integer + name: 'port' + default_value: 22 + description: | + Port for the SSH tunnel. + - !ruby/object:Api::Type::String + name: 'password' + input: true + description: | + SSH password. + conflicts: + - forward_ssh_connectivity.0.private_key + - !ruby/object:Api::Type::String + name: 'privateKey' + input: true + description: | + SSH private key. + conflicts: + - forward_ssh_connectivity.0.password + - !ruby/object:Api::Type::NestedObject + name: 'privateConnectivity' + description: | + Private connectivity. + conflicts: + - forward_ssh_connectivity + properties: + - !ruby/object:Api::Type::String + name: 'privateConnection' + required: true + description: | + A reference to a private connection resource. Format: `projects/{project}/locations/{location}/privateConnections/{name}` +- !ruby/object:Api::Resource + name: 'PrivateConnection' + base_url: "projects/{{project}}/locations/{{location}}/privateConnections" + create_url: "projects/{{project}}/locations/{{location}}/privateConnections?privateConnectionId={{private_connection_id}}" + self_link: "projects/{{project}}/locations/{{location}}/privateConnections/{{private_connection_id}}" + references: !ruby/object:Api::Resource::ReferenceLinks + guides: + 'Official Documentation': 'https://cloud.google.com/datastream/docs/create-a-private-connectivity-configuration' + api: 'https://cloud.google.com/datastream/docs/reference/rest/v1/projects.locations.privateConnections' + description: | + The PrivateConnection resource is used to establish private connectivity between Datastream and a customer's network. + input: true + parameters: + - !ruby/object:Api::Type::String + name: privateConnectionId + description: |- + The private connectivity identifier. + required: true + input: true + url_param_only: true + - !ruby/object:Api::Type::String + name: 'location' + description: | + The name of the location this private connection is located in. + required: true + input: true + url_param_only: true + properties: + - !ruby/object:Api::Type::String + name: 'name' + output: true + description: The resource's name. + - !ruby/object:Api::Type::KeyValuePairs + name: 'labels' + description: Labels. + - !ruby/object:Api::Type::String + name: 'displayName' + required: true + description: Display name. + - !ruby/object:Api::Type::Enum + name: 'state' + description: | + State of the PrivateConnection. + output: true + values: + - :CREATING + - :CREATED + - :FAILED + - :DELETING + - :FAILED_TO_DELETE + - !ruby/object:Api::Type::NestedObject + name: 'error' + output: true + description: | + The PrivateConnection error in case of failure. + properties: + - !ruby/object:Api::Type::String + name: 'message' + description: | + A message containing more information about the error that occurred. + - !ruby/object:Api::Type::KeyValuePairs + name: 'details' + description: | + A list of messages that carry the error details. + - !ruby/object:Api::Type::NestedObject + name: 'vpcPeeringConfig' + required: true + description: | + The VPC Peering configuration is used to create VPC peering + between Datastream and the consumer's VPC. + properties: + - !ruby/object:Api::Type::String + name: 'vpc' + required: true + description: | + Fully qualified name of the VPC that Datastream will peer to. + Format: projects/{project}/global/{networks}/{name} + - !ruby/object:Api::Type::String + name: 'subnet' + required: true + description: | + A free subnet for peering. (CIDR of /29) +- !ruby/object:Api::Resource + name: 'Stream' + base_url: "projects/{{project}}/locations/{{location}}/streams" + create_url: "projects/{{project}}/locations/{{location}}/streams?streamId={{stream_id}}" + self_link: "projects/{{project}}/locations/{{location}}/streams/{{stream_id}}" + update_verb: :PATCH + update_mask: true + references: !ruby/object:Api::Resource::ReferenceLinks + guides: + 'Official Documentation': 'https://cloud.google.com/datastream/docs/create-a-stream' + api: 'https://cloud.google.com/datastream/docs/reference/rest/v1/projects.locations.streams' + description: | + A resource representing streaming data from a source to a destination. + parameters: + - !ruby/object:Api::Type::String + name: streamId + description: |- + The stream identifier. + required: true + input: true + url_param_only: true + - !ruby/object:Api::Type::String + name: 'location' + description: | + The name of the location this stream is located in. + required: true + input: true + url_param_only: true + properties: + - !ruby/object:Api::Type::String + name: 'name' + output: true + description: The stream's name. + - !ruby/object:Api::Type::KeyValuePairs + name: 'labels' + description: Labels. + - !ruby/object:Api::Type::String + name: 'displayName' + required: true + description: Display name. + - !ruby/object:Api::Type::NestedObject + name: 'sourceConfig' + required: true + description: | + Source connection profile configuration. + properties: + - !ruby/object:Api::Type::String + name: 'sourceConnectionProfile' + input: true + required: true + description: | + Source connection profile resource. Format: projects/{project}/locations/{location}/connectionProfiles/{name} + - !ruby/object:Api::Type::NestedObject + name: 'mysqlSourceConfig' + allow_empty_object: true + send_empty_value: true + exactly_one_of: + - source_config.0.mysql_source_config + - source_config.0.oracle_source_config + - source_config.0.postgresql_source_config + description: | + MySQL data source configuration. + properties: - !ruby/object:Api::Type::NestedObject - name: 'oracleProfile' - exactly_one_of: - - oracle_profile - - gcs_profile - - mysql_profile - - bigquery_profile - - postgresql_profile + name: 'includeObjects' description: | - Oracle database profile. + MySQL objects to retrieve from the source. properties: - - !ruby/object:Api::Type::String - name: 'hostname' - required: true + - !ruby/object:Api::Type::Array + name: 'mysqlDatabases' + required: true + min_size: 1 + description: | + MySQL databases on the server + item_type: !ruby/object:Api::Type::NestedObject description: | - Hostname for the Oracle connection. - - !ruby/object:Api::Type::Integer - name: 'port' - default_value: 1521 - description: | - Port for the Oracle connection. - - !ruby/object:Api::Type::String - name: 'username' - required: true - description: | - Username for the Oracle connection. - - !ruby/object:Api::Type::String - name: 'password' - required: true - description: | - Password for the Oracle connection. - - !ruby/object:Api::Type::String - name: 'databaseService' - required: true - description: | - Database for the Oracle connection. - - !ruby/object:Api::Type::KeyValuePairs - name: 'connectionAttributes' - description: Connection string attributes + MySQL database. + properties: + - !ruby/object:Api::Type::String + name: 'database' + required: true + description: | + Database name. + - !ruby/object:Api::Type::Array + name: 'mysqlTables' + min_size: 1 + description: | + Tables in the database. + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'mysqlColumns' + min_size: 1 + description: | + MySQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The MySQL data type. Full data types list can be found here: + https://dev.mysql.com/doc/refman/8.0/en/data-types.html + - !ruby/object:Api::Type::Integer + name: 'length' + output: true + description: | + Column length. + - !ruby/object:Api::Type::String + name: 'collation' + description: | + Column collation. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. - !ruby/object:Api::Type::NestedObject - name: 'gcsProfile' - exactly_one_of: - - oracle_profile - - gcs_profile - - mysql_profile - - bigquery_profile - - postgresql_profile + name: 'excludeObjects' description: | - Cloud Storage bucket profile. + MySQL objects to exclude from the stream. properties: - - !ruby/object:Api::Type::String - name: 'bucket' - required: true - description: | - The Cloud Storage bucket name. - - !ruby/object:Api::Type::String - name: 'rootPath' + - !ruby/object:Api::Type::Array + name: 'mysqlDatabases' + required: true + min_size: 1 + description: | + MySQL databases on the server + item_type: !ruby/object:Api::Type::NestedObject description: | - The root path inside the Cloud Storage bucket. + MySQL database. + properties: + - !ruby/object:Api::Type::String + name: 'database' + required: true + description: | + Database name. + - !ruby/object:Api::Type::Array + name: 'mysqlTables' + min_size: 1 + description: | + Tables in the database. + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'mysqlColumns' + min_size: 1 + description: | + MySQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The MySQL data type. Full data types list can be found here: + https://dev.mysql.com/doc/refman/8.0/en/data-types.html + - !ruby/object:Api::Type::Integer + name: 'length' + output: true + description: | + Column length. + - !ruby/object:Api::Type::String + name: 'collation' + description: | + Column collation. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::Integer + name: 'maxConcurrentCdcTasks' + send_empty_value: true + description: | + Maximum number of concurrent CDC tasks. The number should be non negative. + If not set (or set to 0), the system's default value will be used. + - !ruby/object:Api::Type::NestedObject + name: 'oracleSourceConfig' + allow_empty_object: true + send_empty_value: true + exactly_one_of: + - source_config.0.mysql_source_config + - source_config.0.oracle_source_config + - source_config.0.postgresql_source_config + description: | + MySQL data source configuration. + properties: - !ruby/object:Api::Type::NestedObject - name: 'mysqlProfile' - exactly_one_of: - - oracle_profile - - gcs_profile - - mysql_profile - - bigquery_profile - - postgresql_profile + name: 'includeObjects' description: | - MySQL database profile. + Oracle objects to retrieve from the source. properties: - - !ruby/object:Api::Type::String - name: 'hostname' - required: true + - !ruby/object:Api::Type::Array + name: 'oracleSchemas' + required: true + min_size: 1 + description: | + Oracle schemas/databases in the database server + item_type: !ruby/object:Api::Type::NestedObject description: | - Hostname for the MySQL connection. - - !ruby/object:Api::Type::Integer - name: 'port' - default_value: 3306 - description: | - Port for the MySQL connection. - - !ruby/object:Api::Type::String - name: 'username' - required: true - description: | - Username for the MySQL connection. - - !ruby/object:Api::Type::String - name: 'password' - required: true - input: true - description: | - Password for the MySQL connection. - - !ruby/object:Api::Type::NestedObject - name: 'sslConfig' - description: | - SSL configuration for the MySQL connection. + MySQL database. properties: - - !ruby/object:Api::Type::String - name: 'clientKey' - input: true - description: | - PEM-encoded private key associated with the Client Certificate. - If this field is used then the 'client_certificate' and the - 'ca_certificate' fields are mandatory. - - !ruby/object:Api::Type::Boolean - name: 'clientKeySet' - output: true - description: | - Indicates whether the clientKey field is set. - - !ruby/object:Api::Type::String - name: 'clientCertificate' - input: true - description: | - PEM-encoded certificate that will be used by the replica to - authenticate against the source database server. If this field - is used then the 'clientKey' and the 'caCertificate' fields are - mandatory. - - !ruby/object:Api::Type::Boolean - name: 'clientCertificateSet' - output: true - description: | - Indicates whether the clientCertificate field is set. - - !ruby/object:Api::Type::String - name: 'caCertificate' - input: true - description: | - PEM-encoded certificate of the CA that signed the source database - server's certificate. - - !ruby/object:Api::Type::Boolean - name: 'caCertificateSet' - output: true + - !ruby/object:Api::Type::String + name: 'schema' + required: true + description: | + Schema name. + - !ruby/object:Api::Type::Array + name: 'oracleTables' + min_size: 1 + description: | + Tables in the database. + item_type: !ruby/object:Api::Type::NestedObject description: | - Indicates whether the clientKey field is set. + Oracle table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'oracleColumns' + min_size: 1 + description: | + Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The Oracle data type. Full data types list can be found here: + https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/Data-Types.html + - !ruby/object:Api::Type::Integer + name: 'length' + output: true + description: | + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + output: true + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + output: true + description: | + Column scale. + - !ruby/object:Api::Type::String + name: 'encoding' + output: true + description: | + Column encoding. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + output: true + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + output: true + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + output: true + description: | + The ordinal position of the column in the table. - !ruby/object:Api::Type::NestedObject - name: 'bigqueryProfile' + name: 'excludeObjects' + description: | + Oracle objects to exclude from the stream. + properties: + - !ruby/object:Api::Type::Array + name: 'oracleSchemas' + required: true + min_size: 1 + description: | + Oracle schemas/databases in the database server + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL database. + properties: + - !ruby/object:Api::Type::String + name: 'schema' + required: true + description: | + Schema name. + - !ruby/object:Api::Type::Array + name: 'oracleTables' + min_size: 1 + description: | + Tables in the database. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'oracleColumns' + min_size: 1 + description: | + Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The Oracle data type. Full data types list can be found here: + https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/Data-Types.html + - !ruby/object:Api::Type::Integer + name: 'length' + output: true + description: | + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + output: true + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + output: true + description: | + Column scale. + - !ruby/object:Api::Type::String + name: 'encoding' + output: true + description: | + Column encoding. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + output: true + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + output: true + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + output: true + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::Integer + name: 'maxConcurrentCdcTasks' send_empty_value: true + description: | + Maximum number of concurrent CDC tasks. The number should be non negative. + If not set (or set to 0), the system's default value will be used. + - !ruby/object:Api::Type::Integer + name: 'maxConcurrentBackfillTasks' + send_empty_value: true + description: | + Maximum number of concurrent backfill tasks. The number should be non negative. + If not set (or set to 0), the system's default value will be used. + - !ruby/object:Api::Type::NestedObject + name: 'dropLargeObjects' allow_empty_object: true - exactly_one_of: - - oracle_profile - - gcs_profile - - mysql_profile - - bigquery_profile - - postgresql_profile + send_empty_value: true description: | - BigQuery warehouse profile. + Configuration to drop large object values. properties: [] - !ruby/object:Api::Type::NestedObject - name: 'postgresqlProfile' - exactly_one_of: - - oracle_profile - - gcs_profile - - mysql_profile - - bigquery_profile - - postgresql_profile + name: 'streamLargeObjects' + allow_empty_object: true + send_empty_value: true description: | - PostgreSQL database profile. - properties: - - !ruby/object:Api::Type::String - name: 'hostname' - required: true - description: | - Hostname for the PostgreSQL connection. - - !ruby/object:Api::Type::Integer - name: 'port' - default_value: 5432 - description: | - Port for the PostgreSQL connection. - - !ruby/object:Api::Type::String - name: 'username' - required: true - description: | - Username for the PostgreSQL connection. - - !ruby/object:Api::Type::String - name: 'password' - required: true - description: | - Password for the PostgreSQL connection. - - !ruby/object:Api::Type::String - name: 'database' - required: true - description: | - Database for the PostgreSQL connection. + Configuration to drop large object values. + properties: [ ] + - !ruby/object:Api::Type::NestedObject + name: 'postgresqlSourceConfig' + allow_empty_object: true + send_empty_value: true + exactly_one_of: + - source_config.0.mysql_source_config + - source_config.0.oracle_source_config + - source_config.0.postgresql_source_config + description: | + PostgreSQL data source configuration. + properties: - !ruby/object:Api::Type::NestedObject - name: 'forwardSshConnectivity' + name: 'includeObjects' description: | - Forward SSH tunnel connectivity. - conflicts: - - private_connectivity + PostgreSQL objects to retrieve from the source. properties: - - !ruby/object:Api::Type::String - name: 'hostname' - required: true + - !ruby/object:Api::Type::Array + name: 'postgresqlSchemas' + required: true + min_size: 1 + description: | + PostgreSQL schemas on the server + item_type: !ruby/object:Api::Type::NestedObject description: | - Hostname for the SSH tunnel. - - !ruby/object:Api::Type::String - name: 'username' - required: true - description: | - Username for the SSH tunnel. - - !ruby/object:Api::Type::Integer - name: 'port' - default_value: 22 - description: | - Port for the SSH tunnel. - - !ruby/object:Api::Type::String - name: 'password' - input: true - description: | - SSH password. - conflicts: - - forward_ssh_connectivity.0.private_key - - !ruby/object:Api::Type::String - name: 'privateKey' - input: true - description: | - SSH private key. - conflicts: - - forward_ssh_connectivity.0.password + PostgreSQL schema. + properties: + - !ruby/object:Api::Type::String + name: 'schema' + required: true + description: | + Database name. + - !ruby/object:Api::Type::Array + name: 'postgresqlTables' + min_size: 1 + description: | + Tables in the schema. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'postgresqlColumns' + min_size: 1 + description: | + PostgreSQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The PostgreSQL data type. Full data types list can be found here: + https://www.postgresql.org/docs/current/datatype.html + - !ruby/object:Api::Type::Integer + name: 'length' + output: true + description: | + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + output: true + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + output: true + description: | + Column scale. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. - !ruby/object:Api::Type::NestedObject - name: 'privateConnectivity' + name: 'excludeObjects' description: | - Private connectivity. - conflicts: - - forward_ssh_connectivity + PostgreSQL objects to exclude from the stream. properties: - - !ruby/object:Api::Type::String - name: 'privateConnection' - required: true + - !ruby/object:Api::Type::Array + name: 'postgresqlSchemas' + required: true + min_size: 1 + description: | + PostgreSQL schemas on the server + item_type: !ruby/object:Api::Type::NestedObject description: | - A reference to a private connection resource. Format: `projects/{project}/locations/{location}/privateConnections/{name}` - - !ruby/object:Api::Resource - name: 'PrivateConnection' - base_url: "projects/{{project}}/locations/{{location}}/privateConnections" - create_url: "projects/{{project}}/locations/{{location}}/privateConnections?privateConnectionId={{private_connection_id}}" - self_link: "projects/{{project}}/locations/{{location}}/privateConnections/{{private_connection_id}}" - references: !ruby/object:Api::Resource::ReferenceLinks - guides: - 'Official Documentation': 'https://cloud.google.com/datastream/docs/create-a-private-connectivity-configuration' - api: 'https://cloud.google.com/datastream/docs/reference/rest/v1/projects.locations.privateConnections' - description: | - The PrivateConnection resource is used to establish private connectivity between Datastream and a customer's network. - input: true - parameters: + PostgreSQL schema. + properties: + - !ruby/object:Api::Type::String + name: 'schema' + required: true + description: | + Database name. + - !ruby/object:Api::Type::Array + name: 'postgresqlTables' + min_size: 1 + description: | + Tables in the schema. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'postgresqlColumns' + min_size: 1 + description: | + PostgreSQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The PostgreSQL data type. Full data types list can be found here: + https://www.postgresql.org/docs/current/datatype.html + - !ruby/object:Api::Type::Integer + name: 'length' + output: true + description: | + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + output: true + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + output: true + description: | + Column scale. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. - !ruby/object:Api::Type::String - name: privateConnectionId - description: |- - The private connectivity identifier. + name: 'replicationSlot' required: true - input: true - url_param_only: true - - !ruby/object:Api::Type::String - name: 'location' description: | - The name of the location this private connection is located in. + The name of the logical replication slot that's configured with + the pgoutput plugin. + - !ruby/object:Api::Type::String + name: 'publication' required: true - input: true - url_param_only: true + description: | + The name of the publication that includes the set of all tables + that are defined in the stream's include_objects. + - !ruby/object:Api::Type::Integer + name: 'maxConcurrentBackfillTasks' + send_empty_value: true + description: | + Maximum number of concurrent backfill tasks. The number should be non + negative. If not set (or set to 0), the system's default value will be used. + + - !ruby/object:Api::Type::NestedObject + name: 'destinationConfig' + required: true + description: | + Destination connection profile configuration. properties: + - !ruby/object:Api::Type::String + name: 'destinationConnectionProfile' + input: true + required: true + description: | + Destination connection profile resource. Format: projects/{project}/locations/{location}/connectionProfiles/{name} + - !ruby/object:Api::Type::NestedObject + name: 'gcsDestinationConfig' + exactly_one_of: + - destination_config.0.gcs_destination_config + - destination_config.0.bigquery_destination_config + description: | + A configuration for how data should be loaded to Cloud Storage. + properties: - !ruby/object:Api::Type::String - name: 'name' - output: true - description: The resource's name. - - !ruby/object:Api::Type::KeyValuePairs - name: 'labels' - description: Labels. + name: 'path' + description: | + Path inside the Cloud Storage bucket to write data to. + - !ruby/object:Api::Type::Integer + name: 'fileRotationMb' + description: | + The maximum file size to be saved in the bucket. - !ruby/object:Api::Type::String - name: 'displayName' - required: true - description: Display name. - - !ruby/object:Api::Type::Enum - name: 'state' + name: 'fileRotationInterval' description: | - State of the PrivateConnection. - output: true - values: - - :CREATING - - :CREATED - - :FAILED - - :DELETING - - :FAILED_TO_DELETE + The maximum duration for which new events are added before a file is closed and a new file is created. + A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". Defaults to 900s. - !ruby/object:Api::Type::NestedObject - name: 'error' - output: true + name: 'avroFileFormat' + exactly_one_of: + - destination_config.0.gcs_destination_config.0.avro_file_format + - destination_config.0.gcs_destination_config.0.json_file_format + allow_empty_object: true + send_empty_value: true description: | - The PrivateConnection error in case of failure. + AVRO file format configuration. + properties: [] + - !ruby/object:Api::Type::NestedObject + name: 'jsonFileFormat' + exactly_one_of: + - destination_config.0.gcs_destination_config.0.avro_file_format + - destination_config.0.gcs_destination_config.0.json_file_format + description: | + JSON file format configuration. properties: - - !ruby/object:Api::Type::String - name: 'message' - description: | - A message containing more information about the error that occurred. - - !ruby/object:Api::Type::KeyValuePairs - name: 'details' - description: | - A list of messages that carry the error details. + - !ruby/object:Api::Type::Enum + name: 'schemaFileFormat' + description: | + The schema file format along JSON data files. + values: + - NO_SCHEMA_FILE + - AVRO_SCHEMA_FILE + - !ruby/object:Api::Type::Enum + name: 'compression' + description: | + Compression of the loaded JSON file. + values: + - NO_COMPRESSION + - GZIP + - !ruby/object:Api::Type::NestedObject + name: 'bigqueryDestinationConfig' + exactly_one_of: + - destination_config.0.gcs_destination_config + - destination_config.0.bigquery_destination_config + description: | + A configuration for how data should be loaded to Cloud Storage. + properties: + - !ruby/object:Api::Type::String + name: 'dataFreshness' + description: | + The guaranteed data freshness (in seconds) when querying tables created by the stream. + Editing this field will only affect new tables created in the future, but existing tables + will not be impacted. Lower values mean that queries will return fresher data, but may result in higher cost. + A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". Defaults to 900s. - !ruby/object:Api::Type::NestedObject - name: 'vpcPeeringConfig' - required: true + name: 'singleTargetDataset' + exactly_one_of: + - destination_config.0.bigquery_destination_config.0.single_target_dataset + - destination_config.0.bigquery_destination_config.0.source_hierarchy_datasets description: | - The VPC Peering configuration is used to create VPC peering - between Datastream and the consumer's VPC. + A single target dataset to which all data will be streamed. properties: + - !ruby/object:Api::Type::String + name: 'datasetId' + required: true + description: | + Dataset ID in the format projects/{project}/datasets/{dataset_id} + - !ruby/object:Api::Type::NestedObject + name: 'sourceHierarchyDatasets' + exactly_one_of: + - destination_config.0.bigquery_destination_config.0.single_target_dataset + - destination_config.0.bigquery_destination_config.0.source_hierarchy_datasets + description: | + Destination datasets are created so that hierarchy of the destination data objects matches the source hierarchy. + properties: + - !ruby/object:Api::Type::NestedObject + name: 'datasetTemplate' + required: true + description: | + Dataset template used for dynamic dataset creation. + properties: - !ruby/object:Api::Type::String - name: 'vpc' + name: 'location' required: true description: | - Fully qualified name of the VPC that Datastream will peer to. - Format: projects/{project}/global/{networks}/{name} + The geographic location where the dataset should reside. + See https://cloud.google.com/bigquery/docs/locations for supported locations. - !ruby/object:Api::Type::String - name: 'subnet' - required: true + name: 'datasetIdPrefix' description: | - A free subnet for peering. (CIDR of /29) - - !ruby/object:Api::Resource - name: 'Stream' - base_url: "projects/{{project}}/locations/{{location}}/streams" - create_url: "projects/{{project}}/locations/{{location}}/streams?streamId={{stream_id}}" - self_link: "projects/{{project}}/locations/{{location}}/streams/{{stream_id}}" - update_verb: :PATCH - update_mask: true - references: !ruby/object:Api::Resource::ReferenceLinks - guides: - 'Official Documentation': 'https://cloud.google.com/datastream/docs/create-a-stream' - api: 'https://cloud.google.com/datastream/docs/reference/rest/v1/projects.locations.streams' + If supplied, every created dataset will have its name prefixed by the provided value. + The prefix and name will be separated by an underscore. i.e. _. + - !ruby/object:Api::Type::String + name: 'state' + description: The state of the stream. + output: true + - !ruby/object:Api::Type::NestedObject + name: 'backfillAll' + exactly_one_of: + - backfill_all + - backfill_none + allow_empty_object: true + send_empty_value: true description: | - A resource representing streaming data from a source to a destination. - parameters: - - !ruby/object:Api::Type::String - name: streamId - description: |- - The stream identifier. - required: true - input: true - url_param_only: true - - !ruby/object:Api::Type::String - name: 'location' - description: | - The name of the location this stream is located in. - required: true - input: true - url_param_only: true + Backfill strategy to automatically backfill the Stream's objects. Specific objects can be excluded. properties: - - !ruby/object:Api::Type::String - name: 'name' - output: true - description: The stream's name. - - !ruby/object:Api::Type::KeyValuePairs - name: 'labels' - description: Labels. - - !ruby/object:Api::Type::String - name: 'displayName' - required: true - description: Display name. - - !ruby/object:Api::Type::NestedObject - name: 'sourceConfig' + - !ruby/object:Api::Type::NestedObject + name: 'mysqlExcludedObjects' + description: | + MySQL data source objects to avoid backfilling. + properties: + - !ruby/object:Api::Type::Array + name: 'mysqlDatabases' required: true + min_size: 1 description: | - Source connection profile configuration. - properties: + MySQL databases on the server + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL database. + properties: - !ruby/object:Api::Type::String - name: 'sourceConnectionProfile' - input: true + name: 'database' required: true description: | - Source connection profile resource. Format: projects/{project}/locations/{location}/connectionProfiles/{name} - - !ruby/object:Api::Type::NestedObject - name: 'mysqlSourceConfig' - allow_empty_object: true - send_empty_value: true - required: true + Database name. + - !ruby/object:Api::Type::Array + name: 'mysqlTables' + min_size: 1 description: | - MySQL data source configuration. - properties: - - !ruby/object:Api::Type::NestedObject - name: 'includeObjects' + Tables in the database. + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true description: | - MySQL objects to retrieve from the source. - properties: - - !ruby/object:Api::Type::Array - name: 'mysqlDatabases' - required: true - min_size: 1 - description: | - MySQL databases on the server - item_type: !ruby/object:Api::Type::NestedObject - description: | - MySQL database. - properties: - - !ruby/object:Api::Type::String - name: 'database' - required: true - description: | - Database name. - - !ruby/object:Api::Type::Array - name: 'mysqlTables' - min_size: 1 - description: | - Tables in the database. - item_type: !ruby/object:Api::Type::NestedObject - description: | - MySQL table. - properties: - - !ruby/object:Api::Type::String - name: 'table' - required: true - description: | - Table name. - - !ruby/object:Api::Type::Array - name: 'mysqlColumns' - min_size: 1 - description: | - MySQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. - item_type: !ruby/object:Api::Type::NestedObject - description: | - MySQL Column. - properties: - - !ruby/object:Api::Type::String - name: 'column' - description: | - Column name. - - !ruby/object:Api::Type::String - name: 'dataType' - description: | - The MySQL data type. Full data types list can be found here: - https://dev.mysql.com/doc/refman/8.0/en/data-types.html - - !ruby/object:Api::Type::Integer - name: 'length' - output: true - description: | - Column length. - - !ruby/object:Api::Type::String - name: 'collation' - description: | - Column collation. - - !ruby/object:Api::Type::Boolean - name: 'primaryKey' - description: | - Whether or not the column represents a primary key. - - !ruby/object:Api::Type::Boolean - name: 'nullable' - description: | - Whether or not the column can accept a null value. - - !ruby/object:Api::Type::Integer - name: 'ordinalPosition' - description: | - The ordinal position of the column in the table. - - !ruby/object:Api::Type::NestedObject - name: 'excludeObjects' + Table name. + - !ruby/object:Api::Type::Array + name: 'mysqlColumns' + min_size: 1 description: | - MySQL objects to exclude from the stream. - properties: - - !ruby/object:Api::Type::Array - name: 'mysqlDatabases' - required: true - min_size: 1 + MySQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' description: | - MySQL databases on the server - item_type: !ruby/object:Api::Type::NestedObject - description: | - MySQL database. - properties: - - !ruby/object:Api::Type::String - name: 'database' - required: true - description: | - Database name. - - !ruby/object:Api::Type::Array - name: 'mysqlTables' - min_size: 1 - description: | - Tables in the database. - item_type: !ruby/object:Api::Type::NestedObject - description: | - MySQL table. - properties: - - !ruby/object:Api::Type::String - name: 'table' - required: true - description: | - Table name. - - !ruby/object:Api::Type::Array - name: 'mysqlColumns' - min_size: 1 - description: | - MySQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. - item_type: !ruby/object:Api::Type::NestedObject - description: | - MySQL Column. - properties: - - !ruby/object:Api::Type::String - name: 'column' - description: | - Column name. - - !ruby/object:Api::Type::String - name: 'dataType' - description: | - The MySQL data type. Full data types list can be found here: - https://dev.mysql.com/doc/refman/8.0/en/data-types.html - - !ruby/object:Api::Type::Integer - name: 'length' - output: true - description: | - Column length. - - !ruby/object:Api::Type::String - name: 'collation' - description: | - Column collation. - - !ruby/object:Api::Type::Boolean - name: 'primaryKey' - description: | - Whether or not the column represents a primary key. - - !ruby/object:Api::Type::Boolean - name: 'nullable' - description: | - Whether or not the column can accept a null value. - - !ruby/object:Api::Type::Integer - name: 'ordinalPosition' - description: | - The ordinal position of the column in the table. - - !ruby/object:Api::Type::Integer - name: 'maxConcurrentCdcTasks' - send_empty_value: true - description: | - Maximum number of concurrent CDC tasks. The number should be non negative. - If not set (or set to 0), the system's default value will be used. - - !ruby/object:Api::Type::NestedObject - name: 'destinationConfig' + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The MySQL data type. Full data types list can be found here: + https://dev.mysql.com/doc/refman/8.0/en/data-types.html + - !ruby/object:Api::Type::Integer + name: 'length' + output: true + description: | + Column length. + - !ruby/object:Api::Type::String + name: 'collation' + description: | + Column collation. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::NestedObject + name: 'postgresqlExcludedObjects' + description: | + PostgreSQL data source objects to avoid backfilling. + properties: + - !ruby/object:Api::Type::Array + name: 'postgresqlSchemas' required: true + min_size: 1 description: | - Destination connection profile configuration. - properties: + PostgreSQL schemas on the server + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL schema. + properties: - !ruby/object:Api::Type::String - name: 'destinationConnectionProfile' - input: true + name: 'schema' required: true description: | - Destination connection profile resource. Format: projects/{project}/locations/{location}/connectionProfiles/{name} - - !ruby/object:Api::Type::NestedObject - name: 'gcsDestinationConfig' - exactly_one_of: - - destination_config.0.gcs_destination_config - - destination_config.0.bigquery_destination_config + Database name. + - !ruby/object:Api::Type::Array + name: 'postgresqlTables' + min_size: 1 description: | - A configuration for how data should be loaded to Cloud Storage. - properties: + Tables in the schema. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL table. + properties: - !ruby/object:Api::Type::String - name: 'path' - description: | - Path inside the Cloud Storage bucket to write data to. - - !ruby/object:Api::Type::Integer - name: 'fileRotationMb' - description: | - The maximum file size to be saved in the bucket. - - !ruby/object:Api::Type::String - name: 'fileRotationInterval' - description: | - The maximum duration for which new events are added before a file is closed and a new file is created. - A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". Defaults to 900s. - - !ruby/object:Api::Type::NestedObject - name: 'avroFileFormat' - exactly_one_of: - - destination_config.0.gcs_destination_config.0.avro_file_format - - destination_config.0.gcs_destination_config.0.json_file_format - allow_empty_object: true - send_empty_value: true + name: 'table' + required: true description: | - AVRO file format configuration. - properties: [] - - !ruby/object:Api::Type::NestedObject - name: 'jsonFileFormat' - exactly_one_of: - - destination_config.0.gcs_destination_config.0.avro_file_format - - destination_config.0.gcs_destination_config.0.json_file_format + Table name. + - !ruby/object:Api::Type::Array + name: 'postgresqlColumns' + min_size: 1 description: | - JSON file format configuration. - properties: - - !ruby/object:Api::Type::Enum - name: 'schemaFileFormat' - description: | - The schema file format along JSON data files. - values: - - NO_SCHEMA_FILE - - AVRO_SCHEMA_FILE - - !ruby/object:Api::Type::Enum - name: 'compression' + PostgreSQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' description: | - Compression of the loaded JSON file. - values: - - NO_COMPRESSION - - GZIP - - !ruby/object:Api::Type::NestedObject - name: 'bigqueryDestinationConfig' - exactly_one_of: - - destination_config.0.gcs_destination_config - - destination_config.0.bigquery_destination_config - description: | - A configuration for how data should be loaded to Cloud Storage. - properties: - - !ruby/object:Api::Type::String - name: 'dataFreshness' - description: | - The guaranteed data freshness (in seconds) when querying tables created by the stream. - Editing this field will only affect new tables created in the future, but existing tables - will not be impacted. Lower values mean that queries will return fresher data, but may result in higher cost. - A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". Defaults to 900s. - - !ruby/object:Api::Type::NestedObject - name: 'singleTargetDataset' - exactly_one_of: - - destination_config.0.bigquery_destination_config.0.single_target_dataset - - destination_config.0.bigquery_destination_config.0.source_hierarchy_datasets - description: | - A single target dataset to which all data will be streamed. - properties: + Column name. - !ruby/object:Api::Type::String - name: 'datasetId' - required: true + name: 'dataType' description: | - Dataset ID in the format projects/{project}/datasets/{dataset_id} - - !ruby/object:Api::Type::NestedObject - name: 'sourceHierarchyDatasets' - exactly_one_of: - - destination_config.0.bigquery_destination_config.0.single_target_dataset - - destination_config.0.bigquery_destination_config.0.source_hierarchy_datasets - description: | - Destination datasets are created so that hierarchy of the destination data objects matches the source hierarchy. - properties: - - !ruby/object:Api::Type::NestedObject - name: 'datasetTemplate' - required: true + The PostgreSQL data type. Full data types list can be found here: + https://www.postgresql.org/docs/current/datatype.html + - !ruby/object:Api::Type::Integer + name: 'length' + output: true description: | - Dataset template used for dynamic dataset creation. - properties: - - !ruby/object:Api::Type::String - name: 'location' - required: true - description: | - The geographic location where the dataset should reside. - See https://cloud.google.com/bigquery/docs/locations for supported locations. - - !ruby/object:Api::Type::String - name: 'datasetIdPrefix' - description: | - If supplied, every created dataset will have its name prefixed by the provided value. - The prefix and name will be separated by an underscore. i.e. _. - - !ruby/object:Api::Type::String - name: 'state' - description: The state of the stream. - output: true - - !ruby/object:Api::Type::NestedObject - name: 'backfillAll' - exactly_one_of: - - backfill_all - - backfill_none - allow_empty_object: true - send_empty_value: true + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + output: true + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + output: true + description: | + Column scale. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::NestedObject + name: 'oracleExcludedObjects' + description: | + PostgreSQL data source objects to avoid backfilling. + properties: + - !ruby/object:Api::Type::Array + name: 'oracleSchemas' + required: true + min_size: 1 description: | - Backfill strategy to automatically backfill the Stream's objects. Specific objects can be excluded. - properties: - - !ruby/object:Api::Type::NestedObject - name: 'mysqlExcludedObjects' + Oracle schemas/databases in the database server + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL database. + properties: + - !ruby/object:Api::Type::String + name: 'schema' + required: true description: | - MySQL data source objects to avoid backfilling. - properties: - - !ruby/object:Api::Type::Array - name: 'mysqlDatabases' + Schema name. + - !ruby/object:Api::Type::Array + name: 'oracleTables' + min_size: 1 + description: | + Tables in the database. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle table. + properties: + - !ruby/object:Api::Type::String + name: 'table' required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'oracleColumns' min_size: 1 description: | - MySQL databases on the server + Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. item_type: !ruby/object:Api::Type::NestedObject description: | - MySQL database. + Oracle Column. properties: - - !ruby/object:Api::Type::String - name: 'database' - required: true - description: | - Database name. - - !ruby/object:Api::Type::Array - name: 'mysqlTables' - min_size: 1 - description: | - Tables in the database. - item_type: !ruby/object:Api::Type::NestedObject - description: | - MySQL table. - properties: - - !ruby/object:Api::Type::String - name: 'table' - required: true - description: | - Table name. - - !ruby/object:Api::Type::Array - name: 'mysqlColumns' - min_size: 1 - description: | - MySQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. - item_type: !ruby/object:Api::Type::NestedObject - description: | - MySQL Column. - properties: - - !ruby/object:Api::Type::String - name: 'column' - description: | - Column name. - - !ruby/object:Api::Type::String - name: 'dataType' - description: | - The MySQL data type. Full data types list can be found here: - https://dev.mysql.com/doc/refman/8.0/en/data-types.html - - !ruby/object:Api::Type::Integer - name: 'length' - output: true - description: | - Column length. - - !ruby/object:Api::Type::String - name: 'collation' - description: | - Column collation. - - !ruby/object:Api::Type::Boolean - name: 'primaryKey' - description: | - Whether or not the column represents a primary key. - - !ruby/object:Api::Type::Boolean - name: 'nullable' - description: | - Whether or not the column can accept a null value. - - !ruby/object:Api::Type::Integer - name: 'ordinalPosition' - description: | - The ordinal position of the column in the table. - - !ruby/object:Api::Type::NestedObject - name: 'backfillNone' - exactly_one_of: - - backfill_all - - backfill_none - allow_empty_object: true - send_empty_value: true - description: | - Backfill strategy to disable automatic backfill for the Stream's objects. - properties: [] + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The Oracle data type. Full data types list can be found here: + https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/Data-Types.html + - !ruby/object:Api::Type::Integer + name: 'length' + output: true + description: | + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + output: true + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + output: true + description: | + Column scale. + - !ruby/object:Api::Type::String + name: 'encoding' + output: true + description: | + Column encoding. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + output: true + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + output: true + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + output: true + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::NestedObject + name: 'backfillNone' + exactly_one_of: + - backfill_all + - backfill_none + allow_empty_object: true + send_empty_value: true + description: | + Backfill strategy to disable automatic backfill for the Stream's objects. + properties: [] \ No newline at end of file diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index 9792971faf8c..10f643959b5f 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -50,45 +50,45 @@ overrides: !ruby/object:Overrides::ResourceOverrides # Workaround for https://github.com/hashicorp/terraform-provider-google/issues/12410 custom_flatten: templates/terraform/custom_flatten/datastream_connection_profile_forward_ssh_connectivity_private_key.go.erb examples: - - !ruby/object:Provider::Terraform::Examples - name: "datastream_connection_profile_basic" - primary_resource_id: "default" - vars: - connection_profile_id: "my-profile" - - !ruby/object:Provider::Terraform::Examples - name: "datastream_connection_profile_bigquery_private_connection" - primary_resource_id: "default" - vars: - private_connection_id: "my-connection" - connection_profile_id: "my-profile" - network_name: "my-network" - - !ruby/object:Provider::Terraform::Examples - name: "datastream_connection_profile_full" - primary_resource_id: "default" - # Workaround for https://github.com/hashicorp/terraform-provider-google/issues/12410 - ignore_read_extra: - - "forward_ssh_connectivity.0.password" - vars: - connection_profile_id: "my-profile" - - !ruby/object:Provider::Terraform::Examples - name: "datastream_connection_profile_postgres" - primary_resource_id: "default" - vars: - connection_profile_id: "my-profile" - database_instance_name: "my-instance" - deletion_protection: "true" - # This is covered by the update tests. - skip_test: true + - !ruby/object:Provider::Terraform::Examples + name: "datastream_connection_profile_basic" + primary_resource_id: "default" + vars: + connection_profile_id: "my-profile" + - !ruby/object:Provider::Terraform::Examples + name: "datastream_connection_profile_bigquery_private_connection" + primary_resource_id: "default" + vars: + private_connection_id: "my-connection" + connection_profile_id: "my-profile" + network_name: "my-network" + - !ruby/object:Provider::Terraform::Examples + name: "datastream_connection_profile_full" + primary_resource_id: "default" + # Workaround for https://github.com/hashicorp/terraform-provider-google/issues/12410 + ignore_read_extra: + - "forward_ssh_connectivity.0.password" + vars: + connection_profile_id: "my-profile" + - !ruby/object:Provider::Terraform::Examples + name: "datastream_connection_profile_postgres" + primary_resource_id: "default" + vars: + connection_profile_id: "my-profile" + database_instance_name: "my-instance" + deletion_protection: "true" + # This is covered by the update tests. + skip_test: true PrivateConnection: !ruby/object:Overrides::Terraform::ResourceOverride id_format: projects/{{project}}/locations/{{location}}/privateConnections/{{private_connection_id}} import_format: ["projects/{{project}}/locations/{{location}}/privateConnections/{{private_connection_id}}"] examples: - - !ruby/object:Provider::Terraform::Examples - name: "datastream_private_connection_full" - primary_resource_id: "default" - vars: - private_connection_id: "my-connection" - network_name: "my-network" + - !ruby/object:Provider::Terraform::Examples + name: "datastream_private_connection_full" + primary_resource_id: "default" + vars: + private_connection_id: "my-connection" + network_name: "my-network" custom_code: !ruby/object:Provider::Terraform::CustomCode constants: templates/terraform/constants/private_connection.go.erb post_create: templates/terraform/post_create/private_connection.go.erb @@ -109,16 +109,28 @@ overrides: !ruby/object:Overrides::ResourceOverrides default_from_api: true validation: !ruby/object:Provider::Terraform::Validation function: 'validation.IntAtLeast(0)' + sourceConfig.postgresqlSourceConfig.maxConcurrentBackfillTasks: !ruby/object:Overrides::Terraform::PropertyOverride + default_from_api: true + validation: !ruby/object:Provider::Terraform::Validation + function: 'validation.IntAtLeast(0)' + sourceConfig.oracleSourceConfig.maxConcurrentCdcTasks: !ruby/object:Overrides::Terraform::PropertyOverride + default_from_api: true + validation: !ruby/object:Provider::Terraform::Validation + function: 'validation.IntAtLeast(0)' + sourceConfig.oracleSourceConfig.maxConcurrentBackfillTasks: !ruby/object:Overrides::Terraform::PropertyOverride + default_from_api: true + validation: !ruby/object:Provider::Terraform::Validation + function: 'validation.IntAtLeast(0)' virtual_fields: - - !ruby/object:Api::Type::Enum - name: 'desired_state' - description: | - Desired state of the Stream. Set this field to `RUNNING` to start the stream, and `PAUSED` to pause the stream. - values: - - :NOT_STARTED - - :RUNNING - - :PAUSED - default_value: :NOT_STARTED + - !ruby/object:Api::Type::Enum + name: 'desired_state' + description: | + Desired state of the Stream. Set this field to `RUNNING` to start the stream, and `PAUSED` to pause the stream. + values: + - :NOT_STARTED + - :RUNNING + - :PAUSED + default_value: :NOT_STARTED custom_code: !ruby/object:Provider::Terraform::CustomCode constants: 'templates/terraform/constants/datastream_stream.go.erb' post_create: 'templates/terraform/post_create/datastream_stream.go.erb' @@ -128,45 +140,64 @@ overrides: !ruby/object:Overrides::ResourceOverrides encoder: 'templates/terraform/encoders/datastream_stream.go.erb' resource_definition: 'templates/terraform/resource_definition/datastream_stream.go.erb' examples: - - !ruby/object:Provider::Terraform::Examples - name: "datastream_stream_basic" - pull_external: true - primary_resource_id: "default" - skip_docs: true - # Random provider - skip_vcr: true - vars: - stream_id: "my-stream" - private_connection_id: "my-connection" - network_name: "my-network" - source_connection_profile_id: "source-profile" - database_instance_name: "my-instance" - deletion_protection: "true" - bucket_name: "my-bucket" - destination_connection_profile_id: "destination-profile" - test_vars_overrides: - deletion_protection: "false" - - !ruby/object:Provider::Terraform::Examples - name: "datastream_stream_full" - pull_external: true - primary_resource_id: "default" - # Random provider - skip_vcr: true - vars: - stream_id: "my-stream" - private_connection_id: "my-connection" - network_name: "my-network" - source_connection_profile_id: "source-profile" - database_instance_name: "my-instance" - deletion_protection: "true" - bucket_name: "my-bucket" - destination_connection_profile_id: "destination-profile" - test_vars_overrides: - deletion_protection: "false" + - !ruby/object:Provider::Terraform::Examples + name: "datastream_stream_basic" + pull_external: true + primary_resource_id: "default" + skip_docs: true + # Random provider + skip_vcr: true + vars: + stream_id: "my-stream" + private_connection_id: "my-connection" + network_name: "my-network" + source_connection_profile_id: "source-profile" + database_instance_name: "my-instance" + deletion_protection: "true" + bucket_name: "my-bucket" + destination_connection_profile_id: "destination-profile" + test_vars_overrides: + deletion_protection: "false" + - !ruby/object:Provider::Terraform::Examples + name: "datastream_stream_full" + pull_external: true + primary_resource_id: "default" + # Random provider + skip_vcr: true + vars: + stream_id: "my-stream" + private_connection_id: "my-connection" + network_name: "my-network" + source_connection_profile_id: "source-profile" + database_instance_name: "my-instance" + deletion_protection: "true" + bucket_name: "my-bucket" + destination_connection_profile_id: "destination-profile" + test_vars_overrides: + deletion_protection: "false" + - !ruby/object:Provider::Terraform::Examples + name: "datastream_stream_postgresql" + pull_external: true + primary_resource_id: "default" + skip_vcr: true + vars: + stream_id: "my-stream" + source_connection_profile_id: "source-profile" + destination_connection_profile_id: "destination-profile" + - !ruby/object:Provider::Terraform::Examples + name: "datastream_stream_oracle" + pull_external: true + primary_resource_id: "default" + skip_vcr: true + vars: + stream_id: "my-stream" + source_connection_profile_id: "source-profile" + destination_connection_profile_id: "destination-profile" + # This is for copying files over files: !ruby/object:Provider::Config::Files # These files have templating (ERB) code that will be run. # This is usually to add licensing info, autogeneration notices, etc. compile: - <%= lines(indent(compile('provider/terraform/product~compile.yaml'), 4)) -%> + <%= lines(indent(compile('provider/terraform/product~compile.yaml'), 4)) -%> \ No newline at end of file diff --git a/mmv1/templates/terraform/examples/datastream_stream_oracle.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_oracle.tf.erb new file mode 100644 index 000000000000..effe02d89cc6 --- /dev/null +++ b/mmv1/templates/terraform/examples/datastream_stream_oracle.tf.erb @@ -0,0 +1,86 @@ +resource "google_datastream_connection_profile" "source" { + display_name = "Oracle Source" + location = "us-central1" + connection_profile_id = "<%= ctx[:vars]['source_connection_profile_id'] %>" + + oracle_profile { + hostname = "hostname" + port = 1521 + username = "user" + password = "pass" + database_service = "ORCL" + } +} + +resource "google_datastream_connection_profile" "destination" { + display_name = "BigQuery Destination" + location = "us-central1" + connection_profile_id = "<%= ctx[:vars]['destination_connection_profile_id'] %>" + + bigquery_profile {} +} + +resource "google_datastream_stream" "stream5" { + display_name = "Oracle to BigQuery" + location = "us-central1" + stream_id = "<%= ctx[:vars]['stream_id'] %>" + desired_state = "RUNNING" + + source_config { + source_connection_profile = google_datastream_connection_profile.source.id + oracle_source_config { + max_concurrent_cdc_tasks = 8 + max_concurrent_backfill_tasks = 12 + include_objects { + oracle_schemas { + schema = "schema" + oracle_tables { + table = "table" + oracle_columns { + column = "column" + } + } + } + } + exclude_objects { + oracle_schemas { + schema = "schema" + oracle_tables { + table = "table" + oracle_columns { + column = "column" + } + } + } + } + drop_large_objects {} + } + } + + destination_config { + destination_connection_profile = google_datastream_connection_profile.destination.id + bigquery_destination_config { + data_freshness = "900s" + source_hierarchy_datasets { + dataset_template { + location = "us-central1" + } + } + } + } + + backfill_all { + oracle_excluded_objects { + oracle_schemas { + schema = "schema" + oracle_tables { + table = "table" + oracle_columns { + column = "column" + } + } + } + } + } + } +} diff --git a/mmv1/templates/terraform/examples/datastream_stream_postgresql.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_postgresql.tf.erb new file mode 100644 index 000000000000..7df075b05e1d --- /dev/null +++ b/mmv1/templates/terraform/examples/datastream_stream_postgresql.tf.erb @@ -0,0 +1,85 @@ +resource "google_datastream_connection_profile" "source" { + display_name = "Postgresql Source" + location = "us-central1" + connection_profile_id = "<%= ctx[:vars]['source_connection_profile_id'] %>" + + postgresql_profile { + hostname = "hostname" + port = 3306 + username = "user" + password = "pass" + database = "postgres" + } +} + +resource "google_datastream_connection_profile" "destination" { + display_name = "BigQuery Destination" + location = "us-central1" + connection_profile_id = "<%= ctx[:vars]['destination_connection_profile_id'] %>" + + bigquery_profile {} +} + +resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { + display_name = "Postgres to BigQuery" + location = "us-central1" + stream_id = "<%= ctx[:vars]['stream_id'] %>" + desired_state = "RUNNING" + + source_config { + source_connection_profile = google_datastream_connection_profile.source.id + postgresql_source_config { + max_concurrent_backfill_tasks = 12 + publication = "publication" + replication_slot = "replication_slot" + include_objects { + postgresql_schemas { + schema = "schema" + postgresql_tables { + table = "table" + postgresql_columns { + column = "column" + } + } + } + } + exclude_objects { + postgresql_schemas { + schema = "schema" + postgresql_tables { + table = "table" + postgresql_columns { + column = "column" + } + } + } + } + } + } + + destination_config { + destination_connection_profile = google_datastream_connection_profile.destination.id + bigquery_destination_config { + data_freshness = "900s" + source_hierarchy_datasets { + dataset_template { + location = "us-central1" + } + } + } + } + + backfill_all { + postgresql_excluded_objects { + postgresql_schemas { + schema = "schema" + postgresql_tables { + table = "table" + postgresql_columns { + column = "column" + } + } + } + } + } +} diff --git a/mmv1/templates/terraform/post_create/datastream_stream.go.erb b/mmv1/templates/terraform/post_create/datastream_stream.go.erb index e45d6054f670..1d916bd6e1d8 100644 --- a/mmv1/templates/terraform/post_create/datastream_stream.go.erb +++ b/mmv1/templates/terraform/post_create/datastream_stream.go.erb @@ -14,4 +14,11 @@ -%> if err := waitForDatastreamStreamReady(d, config, d.Timeout(schema.TimeoutCreate) - time.Minute); err != nil { return fmt.Errorf("Error waiting for Stream %q to be NOT_STARTED or RUNNING during creation: %q", d.Get("name").(string), err) -} \ No newline at end of file +} + +if d.Get("state") != d.Get("desired_state") { + log.Printf("[DEBUG] Desired state %s not equal to state = %s, updating stream %q", d.Get("desired_state"), d.Get("state"), d.Id()) + if err = resourceDatastreamStreamUpdate(d, meta); err != nil { + return fmt.Errorf("Error updating Stream %q during creation: %q", d.Get("name").(string), err) + } +} diff --git a/mmv1/third_party/terraform/utils/datastream_operation.go b/mmv1/third_party/terraform/utils/datastream_operation.go index 0078bd8fb191..7f18cb3c3fd7 100644 --- a/mmv1/third_party/terraform/utils/datastream_operation.go +++ b/mmv1/third_party/terraform/utils/datastream_operation.go @@ -4,14 +4,16 @@ import ( "bytes" "encoding/json" "fmt" - datastream "google.golang.org/api/datastream/v1" "time" + + datastream "google.golang.org/api/datastream/v1" ) type DatastreamOperationWaiter struct { Config *Config UserAgent string Project string + Op datastream.Operation CommonOperationWaiter } @@ -20,14 +22,22 @@ func (w *DatastreamOperationWaiter) QueryOp() (interface{}, error) { return nil, fmt.Errorf("Cannot query operation, it's unset or nil.") } // Returns the proper get. - url := fmt.Sprintf("%s%s", w.Config.DatastreamBasePath, w.CommonOperationWaiter.Op.Name) + url := fmt.Sprintf("%s%s", w.Config.DatastreamBasePath, w.Op.Name) return sendRequest(w.Config, "GET", w.Project, url, w.UserAgent, nil) } func (w *DatastreamOperationWaiter) Error() error { if w != nil && w.Op.Error != nil { - return DatastreamError(*w.Op.Error) + return &DatastreamOperationError{Op: w.Op} + } + return nil +} + +func (w *DatastreamOperationWaiter) SetOp(op interface{}) error { + w.CommonOperationWaiter.SetOp(op) + if err := Convert(op, &w.Op); err != nil { + return err } return nil } @@ -38,7 +48,7 @@ func createDatastreamWaiter(config *Config, op map[string]interface{}, project, UserAgent: userAgent, Project: project, } - if err := w.CommonOperationWaiter.SetOp(op); err != nil { + if err := w.SetOp(op); err != nil { return nil, err } return w, nil @@ -53,7 +63,7 @@ func datastreamOperationWaitTimeWithResponse(config *Config, op map[string]inter if err := OperationWait(w, activity, timeout, config.PollInterval); err != nil { return err } - return json.Unmarshal([]byte(w.CommonOperationWaiter.Op.Response), response) + return json.Unmarshal([]byte(w.Op.Response), response) } func datastreamOperationWaitTime(config *Config, op map[string]interface{}, project, activity, userAgent string, timeout time.Duration) error { @@ -69,17 +79,52 @@ func datastreamOperationWaitTime(config *Config, op map[string]interface{}, proj return OperationWait(w, activity, timeout, config.PollInterval) } -// DatastreamError wraps datastream.Status and implements the +// DatastreamOperationError wraps datastream.Status and implements the // error interface so it can be returned. -type DatastreamError datastream.Status +type DatastreamOperationError struct { + Op datastream.Operation +} -func (e DatastreamError) Error() string { +func (e DatastreamOperationError) Error() string { var buf bytes.Buffer - for _, err := range e.Details { + for _, err := range e.Op.Error.Details { buf.Write(err) buf.WriteString("\n") } + if validations := e.extractFailedValidationResult(); validations != nil { + buf.Write(validations) + buf.WriteString("\n") + } return buf.String() } + +// extractFailedValidationResult extracts the internal failed validations +// if there are any. +func (e DatastreamOperationError) extractFailedValidationResult() []byte { + var metadata datastream.OperationMetadata + data, err := e.Op.Metadata.MarshalJSON() + if err != nil { + return nil + } + err = json.Unmarshal(data, &metadata) + if err != nil { + return nil + } + if metadata.ValidationResult == nil { + return nil + } + var res []byte + for _, v := range metadata.ValidationResult.Validations { + if v.State == "FAILED" { + data, err := v.MarshalJSON() + if err != nil { + return nil + } + res = append(res, data...) + res = append(res, []byte("\n")...) + } + } + return res +} From 0101fbdc9125d3533490a1797e2eb35490263d6b Mon Sep 17 00:00:00 2001 From: iperetz-goo Date: Thu, 26 Jan 2023 10:48:49 +0200 Subject: [PATCH 4/4] fix indent --- mmv1/products/datastream/api.yaml | 2120 +++++++++++------------ mmv1/products/datastream/terraform.yaml | 194 +-- 2 files changed, 1157 insertions(+), 1157 deletions(-) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index df71609f1cb6..27c21e8bbb36 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -14,15 +14,15 @@ --- !ruby/object:Api::Product name: Datastream versions: -- !ruby/object:Api::Product::Version - name: ga - base_url: https://datastream.googleapis.com/v1/ + - !ruby/object:Api::Product::Version + name: ga + base_url: https://datastream.googleapis.com/v1/ scopes: -- https://www.googleapis.com/auth/cloud-platform + - https://www.googleapis.com/auth/cloud-platform apis_required: -- !ruby/object:Api::Product::ApiReference - name: Datastream API - url: https://console.cloud.google.com/apis/library/datastream.googleapis.com + - !ruby/object:Api::Product::ApiReference + name: Datastream API + url: https://console.cloud.google.com/apis/library/datastream.googleapis.com async: !ruby/object:Api::OpAsync operation: !ruby/object:Api::OpAsync::Operation path: 'name' @@ -41,698 +41,1131 @@ async: !ruby/object:Api::OpAsync path: 'error' message: 'message' objects: -- !ruby/object:Api::Resource - name: 'ConnectionProfile' - base_url: "projects/{{project}}/locations/{{location}}/connectionProfiles" - create_url: "projects/{{project}}/locations/{{location}}/connectionProfiles?connectionProfileId={{connection_profile_id}}" - self_link: "projects/{{project}}/locations/{{location}}/connectionProfiles/{{connection_profile_id}}" - update_verb: :PATCH - update_mask: true - references: !ruby/object:Api::Resource::ReferenceLinks - guides: - 'Official Documentation': 'https://cloud.google.com/datastream/docs/create-connection-profiles' - api: 'https://cloud.google.com/datastream/docs/reference/rest/v1/projects.locations.connectionProfiles' - description: | - A set of reusable connection configurations to be used as a source or destination for a stream. - parameters: - - !ruby/object:Api::Type::String - name: connectionProfileId - description: |- - The connection profile identifier. - required: true - input: true - url_param_only: true - - !ruby/object:Api::Type::String - name: 'location' - description: | - The name of the location this connection profile is located in. - required: true - input: true - url_param_only: true - properties: - - !ruby/object:Api::Type::String - name: 'name' - output: true - description: The resource's name. - - !ruby/object:Api::Type::KeyValuePairs - name: 'labels' - description: Labels. - - !ruby/object:Api::Type::String - name: 'displayName' - required: true - description: Display name. - - !ruby/object:Api::Type::NestedObject - name: 'oracleProfile' - exactly_one_of: - - oracle_profile - - gcs_profile - - mysql_profile - - bigquery_profile - - postgresql_profile + - !ruby/object:Api::Resource + name: 'ConnectionProfile' + base_url: "projects/{{project}}/locations/{{location}}/connectionProfiles" + create_url: "projects/{{project}}/locations/{{location}}/connectionProfiles?connectionProfileId={{connection_profile_id}}" + self_link: "projects/{{project}}/locations/{{location}}/connectionProfiles/{{connection_profile_id}}" + update_verb: :PATCH + update_mask: true + references: !ruby/object:Api::Resource::ReferenceLinks + guides: + 'Official Documentation': 'https://cloud.google.com/datastream/docs/create-connection-profiles' + api: 'https://cloud.google.com/datastream/docs/reference/rest/v1/projects.locations.connectionProfiles' description: | - Oracle database profile. - properties: + A set of reusable connection configurations to be used as a source or destination for a stream. + parameters: - !ruby/object:Api::Type::String - name: 'hostname' + name: connectionProfileId + description: |- + The connection profile identifier. required: true - description: | - Hostname for the Oracle connection. - - !ruby/object:Api::Type::Integer - name: 'port' - default_value: 1521 - description: | - Port for the Oracle connection. + input: true + url_param_only: true - !ruby/object:Api::Type::String - name: 'username' - required: true + name: 'location' description: | - Username for the Oracle connection. - - !ruby/object:Api::Type::String - name: 'password' + The name of the location this connection profile is located in. required: true - description: | - Password for the Oracle connection. + input: true + url_param_only: true + properties: - !ruby/object:Api::Type::String - name: 'databaseService' - required: true - description: | - Database for the Oracle connection. + name: 'name' + output: true + description: The resource's name. - !ruby/object:Api::Type::KeyValuePairs - name: 'connectionAttributes' - description: Connection string attributes - - !ruby/object:Api::Type::NestedObject - name: 'gcsProfile' - exactly_one_of: - - oracle_profile - - gcs_profile - - mysql_profile - - bigquery_profile - - postgresql_profile - description: | - Cloud Storage bucket profile. - properties: + name: 'labels' + description: Labels. - !ruby/object:Api::Type::String - name: 'bucket' + name: 'displayName' required: true + description: Display name. + - !ruby/object:Api::Type::NestedObject + name: 'oracleProfile' + exactly_one_of: + - oracle_profile + - gcs_profile + - mysql_profile + - bigquery_profile + - postgresql_profile description: | - The Cloud Storage bucket name. - - !ruby/object:Api::Type::String - name: 'rootPath' - description: | - The root path inside the Cloud Storage bucket. - - !ruby/object:Api::Type::NestedObject - name: 'mysqlProfile' - exactly_one_of: - - oracle_profile - - gcs_profile - - mysql_profile - - bigquery_profile - - postgresql_profile - description: | - MySQL database profile. - properties: - - !ruby/object:Api::Type::String - name: 'hostname' - required: true + Oracle database profile. + properties: + - !ruby/object:Api::Type::String + name: 'hostname' + required: true + description: | + Hostname for the Oracle connection. + - !ruby/object:Api::Type::Integer + name: 'port' + default_value: 1521 + description: | + Port for the Oracle connection. + - !ruby/object:Api::Type::String + name: 'username' + required: true + description: | + Username for the Oracle connection. + - !ruby/object:Api::Type::String + name: 'password' + required: true + description: | + Password for the Oracle connection. + - !ruby/object:Api::Type::String + name: 'databaseService' + required: true + description: | + Database for the Oracle connection. + - !ruby/object:Api::Type::KeyValuePairs + name: 'connectionAttributes' + description: Connection string attributes + - !ruby/object:Api::Type::NestedObject + name: 'gcsProfile' + exactly_one_of: + - oracle_profile + - gcs_profile + - mysql_profile + - bigquery_profile + - postgresql_profile description: | - Hostname for the MySQL connection. - - !ruby/object:Api::Type::Integer - name: 'port' - default_value: 3306 + Cloud Storage bucket profile. + properties: + - !ruby/object:Api::Type::String + name: 'bucket' + required: true + description: | + The Cloud Storage bucket name. + - !ruby/object:Api::Type::String + name: 'rootPath' + description: | + The root path inside the Cloud Storage bucket. + - !ruby/object:Api::Type::NestedObject + name: 'mysqlProfile' + exactly_one_of: + - oracle_profile + - gcs_profile + - mysql_profile + - bigquery_profile + - postgresql_profile description: | - Port for the MySQL connection. - - !ruby/object:Api::Type::String - name: 'username' - required: true + MySQL database profile. + properties: + - !ruby/object:Api::Type::String + name: 'hostname' + required: true + description: | + Hostname for the MySQL connection. + - !ruby/object:Api::Type::Integer + name: 'port' + default_value: 3306 + description: | + Port for the MySQL connection. + - !ruby/object:Api::Type::String + name: 'username' + required: true + description: | + Username for the MySQL connection. + - !ruby/object:Api::Type::String + name: 'password' + required: true + input: true + description: | + Password for the MySQL connection. + - !ruby/object:Api::Type::NestedObject + name: 'sslConfig' + description: | + SSL configuration for the MySQL connection. + properties: + - !ruby/object:Api::Type::String + name: 'clientKey' + input: true + description: | + PEM-encoded private key associated with the Client Certificate. + If this field is used then the 'client_certificate' and the + 'ca_certificate' fields are mandatory. + - !ruby/object:Api::Type::Boolean + name: 'clientKeySet' + output: true + description: | + Indicates whether the clientKey field is set. + - !ruby/object:Api::Type::String + name: 'clientCertificate' + input: true + description: | + PEM-encoded certificate that will be used by the replica to + authenticate against the source database server. If this field + is used then the 'clientKey' and the 'caCertificate' fields are + mandatory. + - !ruby/object:Api::Type::Boolean + name: 'clientCertificateSet' + output: true + description: | + Indicates whether the clientCertificate field is set. + - !ruby/object:Api::Type::String + name: 'caCertificate' + input: true + description: | + PEM-encoded certificate of the CA that signed the source database + server's certificate. + - !ruby/object:Api::Type::Boolean + name: 'caCertificateSet' + output: true + description: | + Indicates whether the clientKey field is set. + - !ruby/object:Api::Type::NestedObject + name: 'bigqueryProfile' + send_empty_value: true + allow_empty_object: true + exactly_one_of: + - oracle_profile + - gcs_profile + - mysql_profile + - bigquery_profile + - postgresql_profile description: | - Username for the MySQL connection. - - !ruby/object:Api::Type::String - name: 'password' - required: true - input: true + BigQuery warehouse profile. + properties: [] + - !ruby/object:Api::Type::NestedObject + name: 'postgresqlProfile' + exactly_one_of: + - oracle_profile + - gcs_profile + - mysql_profile + - bigquery_profile + - postgresql_profile description: | - Password for the MySQL connection. + PostgreSQL database profile. + properties: + - !ruby/object:Api::Type::String + name: 'hostname' + required: true + description: | + Hostname for the PostgreSQL connection. + - !ruby/object:Api::Type::Integer + name: 'port' + default_value: 5432 + description: | + Port for the PostgreSQL connection. + - !ruby/object:Api::Type::String + name: 'username' + required: true + description: | + Username for the PostgreSQL connection. + - !ruby/object:Api::Type::String + name: 'password' + required: true + description: | + Password for the PostgreSQL connection. + - !ruby/object:Api::Type::String + name: 'database' + required: true + description: | + Database for the PostgreSQL connection. - !ruby/object:Api::Type::NestedObject - name: 'sslConfig' + name: 'forwardSshConnectivity' description: | - SSL configuration for the MySQL connection. + Forward SSH tunnel connectivity. + conflicts: + - private_connectivity properties: - !ruby/object:Api::Type::String - name: 'clientKey' - input: true + name: 'hostname' + required: true + description: | + Hostname for the SSH tunnel. + - !ruby/object:Api::Type::String + name: 'username' + required: true description: | - PEM-encoded private key associated with the Client Certificate. - If this field is used then the 'client_certificate' and the - 'ca_certificate' fields are mandatory. - - !ruby/object:Api::Type::Boolean - name: 'clientKeySet' - output: true + Username for the SSH tunnel. + - !ruby/object:Api::Type::Integer + name: 'port' + default_value: 22 description: | - Indicates whether the clientKey field is set. + Port for the SSH tunnel. - !ruby/object:Api::Type::String - name: 'clientCertificate' + name: 'password' input: true description: | - PEM-encoded certificate that will be used by the replica to - authenticate against the source database server. If this field - is used then the 'clientKey' and the 'caCertificate' fields are - mandatory. - - !ruby/object:Api::Type::Boolean - name: 'clientCertificateSet' - output: true - description: | - Indicates whether the clientCertificate field is set. + SSH password. + conflicts: + - forward_ssh_connectivity.0.private_key - !ruby/object:Api::Type::String - name: 'caCertificate' + name: 'privateKey' input: true description: | - PEM-encoded certificate of the CA that signed the source database - server's certificate. - - !ruby/object:Api::Type::Boolean - name: 'caCertificateSet' - output: true + SSH private key. + conflicts: + - forward_ssh_connectivity.0.password + - !ruby/object:Api::Type::NestedObject + name: 'privateConnectivity' + description: | + Private connectivity. + conflicts: + - forward_ssh_connectivity + properties: + - !ruby/object:Api::Type::String + name: 'privateConnection' + required: true description: | - Indicates whether the clientKey field is set. - - !ruby/object:Api::Type::NestedObject - name: 'bigqueryProfile' - send_empty_value: true - allow_empty_object: true - exactly_one_of: - - oracle_profile - - gcs_profile - - mysql_profile - - bigquery_profile - - postgresql_profile + A reference to a private connection resource. Format: `projects/{project}/locations/{location}/privateConnections/{name}` + - !ruby/object:Api::Resource + name: 'PrivateConnection' + base_url: "projects/{{project}}/locations/{{location}}/privateConnections" + create_url: "projects/{{project}}/locations/{{location}}/privateConnections?privateConnectionId={{private_connection_id}}" + self_link: "projects/{{project}}/locations/{{location}}/privateConnections/{{private_connection_id}}" + references: !ruby/object:Api::Resource::ReferenceLinks + guides: + 'Official Documentation': 'https://cloud.google.com/datastream/docs/create-a-private-connectivity-configuration' + api: 'https://cloud.google.com/datastream/docs/reference/rest/v1/projects.locations.privateConnections' description: | - BigQuery warehouse profile. - properties: [] - - !ruby/object:Api::Type::NestedObject - name: 'postgresqlProfile' - exactly_one_of: - - oracle_profile - - gcs_profile - - mysql_profile - - bigquery_profile - - postgresql_profile - description: | - PostgreSQL database profile. - properties: - - !ruby/object:Api::Type::String - name: 'hostname' - required: true - description: | - Hostname for the PostgreSQL connection. - - !ruby/object:Api::Type::Integer - name: 'port' - default_value: 5432 - description: | - Port for the PostgreSQL connection. + The PrivateConnection resource is used to establish private connectivity between Datastream and a customer's network. + input: true + parameters: - !ruby/object:Api::Type::String - name: 'username' + name: privateConnectionId + description: |- + The private connectivity identifier. required: true - description: | - Username for the PostgreSQL connection. + input: true + url_param_only: true - !ruby/object:Api::Type::String - name: 'password' - required: true + name: 'location' description: | - Password for the PostgreSQL connection. - - !ruby/object:Api::Type::String - name: 'database' + The name of the location this private connection is located in. required: true - description: | - Database for the PostgreSQL connection. - - !ruby/object:Api::Type::NestedObject - name: 'forwardSshConnectivity' - description: | - Forward SSH tunnel connectivity. - conflicts: - - private_connectivity + input: true + url_param_only: true properties: - !ruby/object:Api::Type::String - name: 'hostname' - required: true - description: | - Hostname for the SSH tunnel. + name: 'name' + output: true + description: The resource's name. + - !ruby/object:Api::Type::KeyValuePairs + name: 'labels' + description: Labels. - !ruby/object:Api::Type::String - name: 'username' + name: 'displayName' required: true + description: Display name. + - !ruby/object:Api::Type::Enum + name: 'state' description: | - Username for the SSH tunnel. - - !ruby/object:Api::Type::Integer - name: 'port' - default_value: 22 - description: | - Port for the SSH tunnel. - - !ruby/object:Api::Type::String - name: 'password' - input: true - description: | - SSH password. - conflicts: - - forward_ssh_connectivity.0.private_key - - !ruby/object:Api::Type::String - name: 'privateKey' - input: true + State of the PrivateConnection. + output: true + values: + - :CREATING + - :CREATED + - :FAILED + - :DELETING + - :FAILED_TO_DELETE + - !ruby/object:Api::Type::NestedObject + name: 'error' + output: true description: | - SSH private key. - conflicts: - - forward_ssh_connectivity.0.password - - !ruby/object:Api::Type::NestedObject - name: 'privateConnectivity' - description: | - Private connectivity. - conflicts: - - forward_ssh_connectivity - properties: - - !ruby/object:Api::Type::String - name: 'privateConnection' + The PrivateConnection error in case of failure. + properties: + - !ruby/object:Api::Type::String + name: 'message' + description: | + A message containing more information about the error that occurred. + - !ruby/object:Api::Type::KeyValuePairs + name: 'details' + description: | + A list of messages that carry the error details. + - !ruby/object:Api::Type::NestedObject + name: 'vpcPeeringConfig' required: true description: | - A reference to a private connection resource. Format: `projects/{project}/locations/{location}/privateConnections/{name}` -- !ruby/object:Api::Resource - name: 'PrivateConnection' - base_url: "projects/{{project}}/locations/{{location}}/privateConnections" - create_url: "projects/{{project}}/locations/{{location}}/privateConnections?privateConnectionId={{private_connection_id}}" - self_link: "projects/{{project}}/locations/{{location}}/privateConnections/{{private_connection_id}}" - references: !ruby/object:Api::Resource::ReferenceLinks - guides: - 'Official Documentation': 'https://cloud.google.com/datastream/docs/create-a-private-connectivity-configuration' - api: 'https://cloud.google.com/datastream/docs/reference/rest/v1/projects.locations.privateConnections' - description: | - The PrivateConnection resource is used to establish private connectivity between Datastream and a customer's network. - input: true - parameters: - - !ruby/object:Api::Type::String - name: privateConnectionId - description: |- - The private connectivity identifier. - required: true - input: true - url_param_only: true - - !ruby/object:Api::Type::String - name: 'location' - description: | - The name of the location this private connection is located in. - required: true - input: true - url_param_only: true - properties: - - !ruby/object:Api::Type::String - name: 'name' - output: true - description: The resource's name. - - !ruby/object:Api::Type::KeyValuePairs - name: 'labels' - description: Labels. - - !ruby/object:Api::Type::String - name: 'displayName' - required: true - description: Display name. - - !ruby/object:Api::Type::Enum - name: 'state' - description: | - State of the PrivateConnection. - output: true - values: - - :CREATING - - :CREATED - - :FAILED - - :DELETING - - :FAILED_TO_DELETE - - !ruby/object:Api::Type::NestedObject - name: 'error' - output: true - description: | - The PrivateConnection error in case of failure. - properties: - - !ruby/object:Api::Type::String - name: 'message' - description: | - A message containing more information about the error that occurred. - - !ruby/object:Api::Type::KeyValuePairs - name: 'details' - description: | - A list of messages that carry the error details. - - !ruby/object:Api::Type::NestedObject - name: 'vpcPeeringConfig' - required: true + The VPC Peering configuration is used to create VPC peering + between Datastream and the consumer's VPC. + properties: + - !ruby/object:Api::Type::String + name: 'vpc' + required: true + description: | + Fully qualified name of the VPC that Datastream will peer to. + Format: projects/{project}/global/{networks}/{name} + - !ruby/object:Api::Type::String + name: 'subnet' + required: true + description: | + A free subnet for peering. (CIDR of /29) + - !ruby/object:Api::Resource + name: 'Stream' + base_url: "projects/{{project}}/locations/{{location}}/streams" + create_url: "projects/{{project}}/locations/{{location}}/streams?streamId={{stream_id}}" + self_link: "projects/{{project}}/locations/{{location}}/streams/{{stream_id}}" + update_verb: :PATCH + update_mask: true + references: !ruby/object:Api::Resource::ReferenceLinks + guides: + 'Official Documentation': 'https://cloud.google.com/datastream/docs/create-a-stream' + api: 'https://cloud.google.com/datastream/docs/reference/rest/v1/projects.locations.streams' description: | - The VPC Peering configuration is used to create VPC peering - between Datastream and the consumer's VPC. - properties: + A resource representing streaming data from a source to a destination. + parameters: - !ruby/object:Api::Type::String - name: 'vpc' + name: streamId + description: |- + The stream identifier. required: true - description: | - Fully qualified name of the VPC that Datastream will peer to. - Format: projects/{project}/global/{networks}/{name} + input: true + url_param_only: true - !ruby/object:Api::Type::String - name: 'subnet' - required: true + name: 'location' description: | - A free subnet for peering. (CIDR of /29) -- !ruby/object:Api::Resource - name: 'Stream' - base_url: "projects/{{project}}/locations/{{location}}/streams" - create_url: "projects/{{project}}/locations/{{location}}/streams?streamId={{stream_id}}" - self_link: "projects/{{project}}/locations/{{location}}/streams/{{stream_id}}" - update_verb: :PATCH - update_mask: true - references: !ruby/object:Api::Resource::ReferenceLinks - guides: - 'Official Documentation': 'https://cloud.google.com/datastream/docs/create-a-stream' - api: 'https://cloud.google.com/datastream/docs/reference/rest/v1/projects.locations.streams' - description: | - A resource representing streaming data from a source to a destination. - parameters: - - !ruby/object:Api::Type::String - name: streamId - description: |- - The stream identifier. - required: true - input: true - url_param_only: true - - !ruby/object:Api::Type::String - name: 'location' - description: | - The name of the location this stream is located in. - required: true - input: true - url_param_only: true - properties: - - !ruby/object:Api::Type::String - name: 'name' - output: true - description: The stream's name. - - !ruby/object:Api::Type::KeyValuePairs - name: 'labels' - description: Labels. - - !ruby/object:Api::Type::String - name: 'displayName' - required: true - description: Display name. - - !ruby/object:Api::Type::NestedObject - name: 'sourceConfig' - required: true - description: | - Source connection profile configuration. + The name of the location this stream is located in. + required: true + input: true + url_param_only: true properties: - !ruby/object:Api::Type::String - name: 'sourceConnectionProfile' - input: true + name: 'name' + output: true + description: The stream's name. + - !ruby/object:Api::Type::KeyValuePairs + name: 'labels' + description: Labels. + - !ruby/object:Api::Type::String + name: 'displayName' required: true - description: | - Source connection profile resource. Format: projects/{project}/locations/{location}/connectionProfiles/{name} + description: Display name. - !ruby/object:Api::Type::NestedObject - name: 'mysqlSourceConfig' - allow_empty_object: true - send_empty_value: true - exactly_one_of: - - source_config.0.mysql_source_config - - source_config.0.oracle_source_config - - source_config.0.postgresql_source_config + name: 'sourceConfig' + required: true description: | - MySQL data source configuration. + Source connection profile configuration. properties: + - !ruby/object:Api::Type::String + name: 'sourceConnectionProfile' + input: true + required: true + description: | + Source connection profile resource. Format: projects/{project}/locations/{location}/connectionProfiles/{name} - !ruby/object:Api::Type::NestedObject - name: 'includeObjects' + name: 'mysqlSourceConfig' + allow_empty_object: true + send_empty_value: true + exactly_one_of: + - source_config.0.mysql_source_config + - source_config.0.oracle_source_config + - source_config.0.postgresql_source_config description: | - MySQL objects to retrieve from the source. + MySQL data source configuration. properties: - - !ruby/object:Api::Type::Array - name: 'mysqlDatabases' - required: true - min_size: 1 + - !ruby/object:Api::Type::NestedObject + name: 'includeObjects' description: | - MySQL databases on the server - item_type: !ruby/object:Api::Type::NestedObject + MySQL objects to retrieve from the source. + properties: + - !ruby/object:Api::Type::Array + name: 'mysqlDatabases' + required: true + min_size: 1 description: | - MySQL database. - properties: - - !ruby/object:Api::Type::String - name: 'database' - required: true - description: | - Database name. - - !ruby/object:Api::Type::Array - name: 'mysqlTables' - min_size: 1 + MySQL databases on the server + item_type: !ruby/object:Api::Type::NestedObject description: | - Tables in the database. - item_type: !ruby/object:Api::Type::NestedObject + MySQL database. + properties: + - !ruby/object:Api::Type::String + name: 'database' + required: true description: | - MySQL table. - properties: - - !ruby/object:Api::Type::String - name: 'table' - required: true - description: | - Table name. - - !ruby/object:Api::Type::Array - name: 'mysqlColumns' - min_size: 1 + Database name. + - !ruby/object:Api::Type::Array + name: 'mysqlTables' + min_size: 1 + description: | + Tables in the database. + item_type: !ruby/object:Api::Type::NestedObject description: | - MySQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. - item_type: !ruby/object:Api::Type::NestedObject + MySQL table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true description: | - MySQL Column. - properties: - - !ruby/object:Api::Type::String - name: 'column' - description: | - Column name. - - !ruby/object:Api::Type::String - name: 'dataType' - description: | - The MySQL data type. Full data types list can be found here: - https://dev.mysql.com/doc/refman/8.0/en/data-types.html - - !ruby/object:Api::Type::Integer - name: 'length' - output: true - description: | - Column length. - - !ruby/object:Api::Type::String - name: 'collation' + Table name. + - !ruby/object:Api::Type::Array + name: 'mysqlColumns' + min_size: 1 + description: | + MySQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject description: | - Column collation. - - !ruby/object:Api::Type::Boolean - name: 'primaryKey' - description: | - Whether or not the column represents a primary key. - - !ruby/object:Api::Type::Boolean - name: 'nullable' - description: | - Whether or not the column can accept a null value. - - !ruby/object:Api::Type::Integer - name: 'ordinalPosition' + MySQL Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The MySQL data type. Full data types list can be found here: + https://dev.mysql.com/doc/refman/8.0/en/data-types.html + - !ruby/object:Api::Type::Integer + name: 'length' + output: true + description: | + Column length. + - !ruby/object:Api::Type::String + name: 'collation' + description: | + Column collation. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::NestedObject + name: 'excludeObjects' + description: | + MySQL objects to exclude from the stream. + properties: + - !ruby/object:Api::Type::Array + name: 'mysqlDatabases' + required: true + min_size: 1 + description: | + MySQL databases on the server + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL database. + properties: + - !ruby/object:Api::Type::String + name: 'database' + required: true + description: | + Database name. + - !ruby/object:Api::Type::Array + name: 'mysqlTables' + min_size: 1 + description: | + Tables in the database. + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'mysqlColumns' + min_size: 1 + description: | + MySQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject description: | - The ordinal position of the column in the table. + MySQL Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The MySQL data type. Full data types list can be found here: + https://dev.mysql.com/doc/refman/8.0/en/data-types.html + - !ruby/object:Api::Type::Integer + name: 'length' + output: true + description: | + Column length. + - !ruby/object:Api::Type::String + name: 'collation' + description: | + Column collation. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::Integer + name: 'maxConcurrentCdcTasks' + send_empty_value: true + description: | + Maximum number of concurrent CDC tasks. The number should be non negative. + If not set (or set to 0), the system's default value will be used. - !ruby/object:Api::Type::NestedObject - name: 'excludeObjects' + name: 'oracleSourceConfig' + allow_empty_object: true + send_empty_value: true + exactly_one_of: + - source_config.0.mysql_source_config + - source_config.0.oracle_source_config + - source_config.0.postgresql_source_config description: | - MySQL objects to exclude from the stream. + MySQL data source configuration. properties: - - !ruby/object:Api::Type::Array - name: 'mysqlDatabases' - required: true - min_size: 1 + - !ruby/object:Api::Type::NestedObject + name: 'includeObjects' description: | - MySQL databases on the server - item_type: !ruby/object:Api::Type::NestedObject + Oracle objects to retrieve from the source. + properties: + - !ruby/object:Api::Type::Array + name: 'oracleSchemas' + required: true + min_size: 1 description: | - MySQL database. - properties: - - !ruby/object:Api::Type::String - name: 'database' - required: true - description: | - Database name. - - !ruby/object:Api::Type::Array - name: 'mysqlTables' - min_size: 1 + Oracle schemas/databases in the database server + item_type: !ruby/object:Api::Type::NestedObject description: | - Tables in the database. - item_type: !ruby/object:Api::Type::NestedObject + MySQL database. + properties: + - !ruby/object:Api::Type::String + name: 'schema' + required: true description: | - MySQL table. - properties: - - !ruby/object:Api::Type::String - name: 'table' - required: true - description: | - Table name. - - !ruby/object:Api::Type::Array - name: 'mysqlColumns' - min_size: 1 + Schema name. + - !ruby/object:Api::Type::Array + name: 'oracleTables' + min_size: 1 + description: | + Tables in the database. + item_type: !ruby/object:Api::Type::NestedObject description: | - MySQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. - item_type: !ruby/object:Api::Type::NestedObject + Oracle table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true description: | - MySQL Column. - properties: - - !ruby/object:Api::Type::String - name: 'column' - description: | - Column name. - - !ruby/object:Api::Type::String - name: 'dataType' - description: | - The MySQL data type. Full data types list can be found here: - https://dev.mysql.com/doc/refman/8.0/en/data-types.html - - !ruby/object:Api::Type::Integer - name: 'length' - output: true - description: | - Column length. - - !ruby/object:Api::Type::String - name: 'collation' - description: | - Column collation. - - !ruby/object:Api::Type::Boolean - name: 'primaryKey' - description: | - Whether or not the column represents a primary key. - - !ruby/object:Api::Type::Boolean - name: 'nullable' + Table name. + - !ruby/object:Api::Type::Array + name: 'oracleColumns' + min_size: 1 + description: | + Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject description: | - Whether or not the column can accept a null value. - - !ruby/object:Api::Type::Integer - name: 'ordinalPosition' + Oracle Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The Oracle data type. Full data types list can be found here: + https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/Data-Types.html + - !ruby/object:Api::Type::Integer + name: 'length' + output: true + description: | + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + output: true + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + output: true + description: | + Column scale. + - !ruby/object:Api::Type::String + name: 'encoding' + output: true + description: | + Column encoding. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + output: true + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + output: true + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + output: true + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::NestedObject + name: 'excludeObjects' + description: | + Oracle objects to exclude from the stream. + properties: + - !ruby/object:Api::Type::Array + name: 'oracleSchemas' + required: true + min_size: 1 + description: | + Oracle schemas/databases in the database server + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL database. + properties: + - !ruby/object:Api::Type::String + name: 'schema' + required: true + description: | + Schema name. + - !ruby/object:Api::Type::Array + name: 'oracleTables' + min_size: 1 + description: | + Tables in the database. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'oracleColumns' + min_size: 1 + description: | + Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject description: | - The ordinal position of the column in the table. - - !ruby/object:Api::Type::Integer - name: 'maxConcurrentCdcTasks' + Oracle Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The Oracle data type. Full data types list can be found here: + https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/Data-Types.html + - !ruby/object:Api::Type::Integer + name: 'length' + output: true + description: | + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + output: true + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + output: true + description: | + Column scale. + - !ruby/object:Api::Type::String + name: 'encoding' + output: true + description: | + Column encoding. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + output: true + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + output: true + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + output: true + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::Integer + name: 'maxConcurrentCdcTasks' + send_empty_value: true + description: | + Maximum number of concurrent CDC tasks. The number should be non negative. + If not set (or set to 0), the system's default value will be used. + - !ruby/object:Api::Type::Integer + name: 'maxConcurrentBackfillTasks' + send_empty_value: true + description: | + Maximum number of concurrent backfill tasks. The number should be non negative. + If not set (or set to 0), the system's default value will be used. + - !ruby/object:Api::Type::NestedObject + name: 'dropLargeObjects' + allow_empty_object: true + send_empty_value: true + description: | + Configuration to drop large object values. + properties: [] + - !ruby/object:Api::Type::NestedObject + name: 'streamLargeObjects' + allow_empty_object: true + send_empty_value: true + description: | + Configuration to drop large object values. + properties: [ ] + - !ruby/object:Api::Type::NestedObject + name: 'postgresqlSourceConfig' + allow_empty_object: true send_empty_value: true + exactly_one_of: + - source_config.0.mysql_source_config + - source_config.0.oracle_source_config + - source_config.0.postgresql_source_config description: | - Maximum number of concurrent CDC tasks. The number should be non negative. - If not set (or set to 0), the system's default value will be used. + PostgreSQL data source configuration. + properties: + - !ruby/object:Api::Type::NestedObject + name: 'includeObjects' + description: | + PostgreSQL objects to retrieve from the source. + properties: + - !ruby/object:Api::Type::Array + name: 'postgresqlSchemas' + required: true + min_size: 1 + description: | + PostgreSQL schemas on the server + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL schema. + properties: + - !ruby/object:Api::Type::String + name: 'schema' + required: true + description: | + Database name. + - !ruby/object:Api::Type::Array + name: 'postgresqlTables' + min_size: 1 + description: | + Tables in the schema. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'postgresqlColumns' + min_size: 1 + description: | + PostgreSQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The PostgreSQL data type. Full data types list can be found here: + https://www.postgresql.org/docs/current/datatype.html + - !ruby/object:Api::Type::Integer + name: 'length' + output: true + description: | + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + output: true + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + output: true + description: | + Column scale. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::NestedObject + name: 'excludeObjects' + description: | + PostgreSQL objects to exclude from the stream. + properties: + - !ruby/object:Api::Type::Array + name: 'postgresqlSchemas' + required: true + min_size: 1 + description: | + PostgreSQL schemas on the server + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL schema. + properties: + - !ruby/object:Api::Type::String + name: 'schema' + required: true + description: | + Database name. + - !ruby/object:Api::Type::Array + name: 'postgresqlTables' + min_size: 1 + description: | + Tables in the schema. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'postgresqlColumns' + min_size: 1 + description: | + PostgreSQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The PostgreSQL data type. Full data types list can be found here: + https://www.postgresql.org/docs/current/datatype.html + - !ruby/object:Api::Type::Integer + name: 'length' + output: true + description: | + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + output: true + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + output: true + description: | + Column scale. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::String + name: 'replicationSlot' + required: true + description: | + The name of the logical replication slot that's configured with + the pgoutput plugin. + - !ruby/object:Api::Type::String + name: 'publication' + required: true + description: | + The name of the publication that includes the set of all tables + that are defined in the stream's include_objects. + - !ruby/object:Api::Type::Integer + name: 'maxConcurrentBackfillTasks' + send_empty_value: true + description: | + Maximum number of concurrent backfill tasks. The number should be non + negative. If not set (or set to 0), the system's default value will be used. + - !ruby/object:Api::Type::NestedObject - name: 'oracleSourceConfig' - allow_empty_object: true - send_empty_value: true - exactly_one_of: - - source_config.0.mysql_source_config - - source_config.0.oracle_source_config - - source_config.0.postgresql_source_config + name: 'destinationConfig' + required: true description: | - MySQL data source configuration. + Destination connection profile configuration. properties: + - !ruby/object:Api::Type::String + name: 'destinationConnectionProfile' + input: true + required: true + description: | + Destination connection profile resource. Format: projects/{project}/locations/{location}/connectionProfiles/{name} - !ruby/object:Api::Type::NestedObject - name: 'includeObjects' + name: 'gcsDestinationConfig' + exactly_one_of: + - destination_config.0.gcs_destination_config + - destination_config.0.bigquery_destination_config description: | - Oracle objects to retrieve from the source. + A configuration for how data should be loaded to Cloud Storage. properties: - - !ruby/object:Api::Type::Array - name: 'oracleSchemas' - required: true - min_size: 1 + - !ruby/object:Api::Type::String + name: 'path' description: | - Oracle schemas/databases in the database server - item_type: !ruby/object:Api::Type::NestedObject + Path inside the Cloud Storage bucket to write data to. + - !ruby/object:Api::Type::Integer + name: 'fileRotationMb' + description: | + The maximum file size to be saved in the bucket. + - !ruby/object:Api::Type::String + name: 'fileRotationInterval' + description: | + The maximum duration for which new events are added before a file is closed and a new file is created. + A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". Defaults to 900s. + - !ruby/object:Api::Type::NestedObject + name: 'avroFileFormat' + exactly_one_of: + - destination_config.0.gcs_destination_config.0.avro_file_format + - destination_config.0.gcs_destination_config.0.json_file_format + allow_empty_object: true + send_empty_value: true + description: | + AVRO file format configuration. + properties: [] + - !ruby/object:Api::Type::NestedObject + name: 'jsonFileFormat' + exactly_one_of: + - destination_config.0.gcs_destination_config.0.avro_file_format + - destination_config.0.gcs_destination_config.0.json_file_format + description: | + JSON file format configuration. + properties: + - !ruby/object:Api::Type::Enum + name: 'schemaFileFormat' description: | - MySQL database. + The schema file format along JSON data files. + values: + - NO_SCHEMA_FILE + - AVRO_SCHEMA_FILE + - !ruby/object:Api::Type::Enum + name: 'compression' + description: | + Compression of the loaded JSON file. + values: + - NO_COMPRESSION + - GZIP + - !ruby/object:Api::Type::NestedObject + name: 'bigqueryDestinationConfig' + exactly_one_of: + - destination_config.0.gcs_destination_config + - destination_config.0.bigquery_destination_config + description: | + A configuration for how data should be loaded to Cloud Storage. + properties: + - !ruby/object:Api::Type::String + name: 'dataFreshness' + description: | + The guaranteed data freshness (in seconds) when querying tables created by the stream. + Editing this field will only affect new tables created in the future, but existing tables + will not be impacted. Lower values mean that queries will return fresher data, but may result in higher cost. + A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". Defaults to 900s. + - !ruby/object:Api::Type::NestedObject + name: 'singleTargetDataset' + exactly_one_of: + - destination_config.0.bigquery_destination_config.0.single_target_dataset + - destination_config.0.bigquery_destination_config.0.source_hierarchy_datasets + description: | + A single target dataset to which all data will be streamed. + properties: + - !ruby/object:Api::Type::String + name: 'datasetId' + required: true + description: | + Dataset ID in the format projects/{project}/datasets/{dataset_id} + - !ruby/object:Api::Type::NestedObject + name: 'sourceHierarchyDatasets' + exactly_one_of: + - destination_config.0.bigquery_destination_config.0.single_target_dataset + - destination_config.0.bigquery_destination_config.0.source_hierarchy_datasets + description: | + Destination datasets are created so that hierarchy of the destination data objects matches the source hierarchy. + properties: + - !ruby/object:Api::Type::NestedObject + name: 'datasetTemplate' + required: true + description: | + Dataset template used for dynamic dataset creation. properties: - !ruby/object:Api::Type::String - name: 'schema' + name: 'location' required: true description: | - Schema name. - - !ruby/object:Api::Type::Array - name: 'oracleTables' - min_size: 1 + The geographic location where the dataset should reside. + See https://cloud.google.com/bigquery/docs/locations for supported locations. + - !ruby/object:Api::Type::String + name: 'datasetIdPrefix' description: | - Tables in the database. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle table. - properties: - - !ruby/object:Api::Type::String - name: 'table' - required: true - description: | - Table name. - - !ruby/object:Api::Type::Array - name: 'oracleColumns' - min_size: 1 - description: | - Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle Column. - properties: - - !ruby/object:Api::Type::String - name: 'column' - description: | - Column name. - - !ruby/object:Api::Type::String - name: 'dataType' - description: | - The Oracle data type. Full data types list can be found here: - https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/Data-Types.html - - !ruby/object:Api::Type::Integer - name: 'length' - output: true - description: | - Column length. - - !ruby/object:Api::Type::Integer - name: 'precision' - output: true - description: | - Column precision. - - !ruby/object:Api::Type::Integer - name: 'scale' - output: true - description: | - Column scale. - - !ruby/object:Api::Type::String - name: 'encoding' - output: true - description: | - Column encoding. - - !ruby/object:Api::Type::Boolean - name: 'primaryKey' - output: true - description: | - Whether or not the column represents a primary key. - - !ruby/object:Api::Type::Boolean - name: 'nullable' - output: true - description: | - Whether or not the column can accept a null value. - - !ruby/object:Api::Type::Integer - name: 'ordinalPosition' - output: true - description: | - The ordinal position of the column in the table. + If supplied, every created dataset will have its name prefixed by the provided value. + The prefix and name will be separated by an underscore. i.e. _. + - !ruby/object:Api::Type::String + name: 'state' + description: The state of the stream. + output: true + - !ruby/object:Api::Type::NestedObject + name: 'backfillAll' + exactly_one_of: + - backfill_all + - backfill_none + allow_empty_object: true + send_empty_value: true + description: | + Backfill strategy to automatically backfill the Stream's objects. Specific objects can be excluded. + properties: - !ruby/object:Api::Type::NestedObject - name: 'excludeObjects' + name: 'mysqlExcludedObjects' description: | - Oracle objects to exclude from the stream. + MySQL data source objects to avoid backfilling. properties: - !ruby/object:Api::Type::Array - name: 'oracleSchemas' + name: 'mysqlDatabases' required: true min_size: 1 description: | - Oracle schemas/databases in the database server + MySQL databases on the server item_type: !ruby/object:Api::Type::NestedObject description: | MySQL database. properties: - !ruby/object:Api::Type::String - name: 'schema' + name: 'database' required: true description: | - Schema name. + Database name. - !ruby/object:Api::Type::Array - name: 'oracleTables' + name: 'mysqlTables' min_size: 1 description: | Tables in the database. item_type: !ruby/object:Api::Type::NestedObject description: | - Oracle table. + MySQL table. properties: - !ruby/object:Api::Type::String name: 'table' @@ -740,13 +1173,13 @@ objects: description: | Table name. - !ruby/object:Api::Type::Array - name: 'oracleColumns' + name: 'mysqlColumns' min_size: 1 description: | - Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + MySQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. item_type: !ruby/object:Api::Type::NestedObject description: | - Oracle Column. + MySQL Column. properties: - !ruby/object:Api::Type::String name: 'column' @@ -755,84 +1188,33 @@ objects: - !ruby/object:Api::Type::String name: 'dataType' description: | - The Oracle data type. Full data types list can be found here: - https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/Data-Types.html + The MySQL data type. Full data types list can be found here: + https://dev.mysql.com/doc/refman/8.0/en/data-types.html - !ruby/object:Api::Type::Integer name: 'length' output: true description: | Column length. - - !ruby/object:Api::Type::Integer - name: 'precision' - output: true - description: | - Column precision. - - !ruby/object:Api::Type::Integer - name: 'scale' - output: true - description: | - Column scale. - !ruby/object:Api::Type::String - name: 'encoding' - output: true + name: 'collation' description: | - Column encoding. + Column collation. - !ruby/object:Api::Type::Boolean name: 'primaryKey' - output: true description: | Whether or not the column represents a primary key. - !ruby/object:Api::Type::Boolean name: 'nullable' - output: true description: | Whether or not the column can accept a null value. - !ruby/object:Api::Type::Integer name: 'ordinalPosition' - output: true description: | The ordinal position of the column in the table. - - !ruby/object:Api::Type::Integer - name: 'maxConcurrentCdcTasks' - send_empty_value: true - description: | - Maximum number of concurrent CDC tasks. The number should be non negative. - If not set (or set to 0), the system's default value will be used. - - !ruby/object:Api::Type::Integer - name: 'maxConcurrentBackfillTasks' - send_empty_value: true - description: | - Maximum number of concurrent backfill tasks. The number should be non negative. - If not set (or set to 0), the system's default value will be used. - - !ruby/object:Api::Type::NestedObject - name: 'dropLargeObjects' - allow_empty_object: true - send_empty_value: true - description: | - Configuration to drop large object values. - properties: [] - - !ruby/object:Api::Type::NestedObject - name: 'streamLargeObjects' - allow_empty_object: true - send_empty_value: true - description: | - Configuration to drop large object values. - properties: [ ] - - !ruby/object:Api::Type::NestedObject - name: 'postgresqlSourceConfig' - allow_empty_object: true - send_empty_value: true - exactly_one_of: - - source_config.0.mysql_source_config - - source_config.0.oracle_source_config - - source_config.0.postgresql_source_config - description: | - PostgreSQL data source configuration. - properties: - !ruby/object:Api::Type::NestedObject - name: 'includeObjects' + name: 'postgresqlExcludedObjects' description: | - PostgreSQL objects to retrieve from the source. + PostgreSQL data source objects to avoid backfilling. properties: - !ruby/object:Api::Type::Array name: 'postgresqlSchemas' @@ -909,33 +1291,33 @@ objects: description: | The ordinal position of the column in the table. - !ruby/object:Api::Type::NestedObject - name: 'excludeObjects' + name: 'oracleExcludedObjects' description: | - PostgreSQL objects to exclude from the stream. + PostgreSQL data source objects to avoid backfilling. properties: - !ruby/object:Api::Type::Array - name: 'postgresqlSchemas' + name: 'oracleSchemas' required: true min_size: 1 description: | - PostgreSQL schemas on the server + Oracle schemas/databases in the database server item_type: !ruby/object:Api::Type::NestedObject description: | - PostgreSQL schema. + MySQL database. properties: - !ruby/object:Api::Type::String name: 'schema' required: true description: | - Database name. + Schema name. - !ruby/object:Api::Type::Array - name: 'postgresqlTables' + name: 'oracleTables' min_size: 1 description: | - Tables in the schema. + Tables in the database. item_type: !ruby/object:Api::Type::NestedObject description: | - PostgreSQL table. + Oracle table. properties: - !ruby/object:Api::Type::String name: 'table' @@ -943,13 +1325,13 @@ objects: description: | Table name. - !ruby/object:Api::Type::Array - name: 'postgresqlColumns' + name: 'oracleColumns' min_size: 1 description: | - PostgreSQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. item_type: !ruby/object:Api::Type::NestedObject description: | - PostgreSQL Column. + Oracle Column. properties: - !ruby/object:Api::Type::String name: 'column' @@ -958,8 +1340,8 @@ objects: - !ruby/object:Api::Type::String name: 'dataType' description: | - The PostgreSQL data type. Full data types list can be found here: - https://www.postgresql.org/docs/current/datatype.html + The Oracle data type. Full data types list can be found here: + https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/Data-Types.html - !ruby/object:Api::Type::Integer name: 'length' output: true @@ -975,415 +1357,33 @@ objects: output: true description: | Column scale. + - !ruby/object:Api::Type::String + name: 'encoding' + output: true + description: | + Column encoding. - !ruby/object:Api::Type::Boolean name: 'primaryKey' + output: true description: | Whether or not the column represents a primary key. - !ruby/object:Api::Type::Boolean name: 'nullable' + output: true description: | Whether or not the column can accept a null value. - !ruby/object:Api::Type::Integer name: 'ordinalPosition' + output: true description: | The ordinal position of the column in the table. - - !ruby/object:Api::Type::String - name: 'replicationSlot' - required: true - description: | - The name of the logical replication slot that's configured with - the pgoutput plugin. - - !ruby/object:Api::Type::String - name: 'publication' - required: true - description: | - The name of the publication that includes the set of all tables - that are defined in the stream's include_objects. - - !ruby/object:Api::Type::Integer - name: 'maxConcurrentBackfillTasks' - send_empty_value: true - description: | - Maximum number of concurrent backfill tasks. The number should be non - negative. If not set (or set to 0), the system's default value will be used. - - - !ruby/object:Api::Type::NestedObject - name: 'destinationConfig' - required: true - description: | - Destination connection profile configuration. - properties: - - !ruby/object:Api::Type::String - name: 'destinationConnectionProfile' - input: true - required: true - description: | - Destination connection profile resource. Format: projects/{project}/locations/{location}/connectionProfiles/{name} - - !ruby/object:Api::Type::NestedObject - name: 'gcsDestinationConfig' - exactly_one_of: - - destination_config.0.gcs_destination_config - - destination_config.0.bigquery_destination_config - description: | - A configuration for how data should be loaded to Cloud Storage. - properties: - - !ruby/object:Api::Type::String - name: 'path' - description: | - Path inside the Cloud Storage bucket to write data to. - - !ruby/object:Api::Type::Integer - name: 'fileRotationMb' - description: | - The maximum file size to be saved in the bucket. - - !ruby/object:Api::Type::String - name: 'fileRotationInterval' - description: | - The maximum duration for which new events are added before a file is closed and a new file is created. - A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". Defaults to 900s. - - !ruby/object:Api::Type::NestedObject - name: 'avroFileFormat' - exactly_one_of: - - destination_config.0.gcs_destination_config.0.avro_file_format - - destination_config.0.gcs_destination_config.0.json_file_format - allow_empty_object: true - send_empty_value: true - description: | - AVRO file format configuration. - properties: [] - - !ruby/object:Api::Type::NestedObject - name: 'jsonFileFormat' - exactly_one_of: - - destination_config.0.gcs_destination_config.0.avro_file_format - - destination_config.0.gcs_destination_config.0.json_file_format - description: | - JSON file format configuration. - properties: - - !ruby/object:Api::Type::Enum - name: 'schemaFileFormat' - description: | - The schema file format along JSON data files. - values: - - NO_SCHEMA_FILE - - AVRO_SCHEMA_FILE - - !ruby/object:Api::Type::Enum - name: 'compression' - description: | - Compression of the loaded JSON file. - values: - - NO_COMPRESSION - - GZIP - !ruby/object:Api::Type::NestedObject - name: 'bigqueryDestinationConfig' + name: 'backfillNone' exactly_one_of: - - destination_config.0.gcs_destination_config - - destination_config.0.bigquery_destination_config - description: | - A configuration for how data should be loaded to Cloud Storage. - properties: - - !ruby/object:Api::Type::String - name: 'dataFreshness' - description: | - The guaranteed data freshness (in seconds) when querying tables created by the stream. - Editing this field will only affect new tables created in the future, but existing tables - will not be impacted. Lower values mean that queries will return fresher data, but may result in higher cost. - A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". Defaults to 900s. - - !ruby/object:Api::Type::NestedObject - name: 'singleTargetDataset' - exactly_one_of: - - destination_config.0.bigquery_destination_config.0.single_target_dataset - - destination_config.0.bigquery_destination_config.0.source_hierarchy_datasets - description: | - A single target dataset to which all data will be streamed. - properties: - - !ruby/object:Api::Type::String - name: 'datasetId' - required: true - description: | - Dataset ID in the format projects/{project}/datasets/{dataset_id} - - !ruby/object:Api::Type::NestedObject - name: 'sourceHierarchyDatasets' - exactly_one_of: - - destination_config.0.bigquery_destination_config.0.single_target_dataset - - destination_config.0.bigquery_destination_config.0.source_hierarchy_datasets - description: | - Destination datasets are created so that hierarchy of the destination data objects matches the source hierarchy. - properties: - - !ruby/object:Api::Type::NestedObject - name: 'datasetTemplate' - required: true - description: | - Dataset template used for dynamic dataset creation. - properties: - - !ruby/object:Api::Type::String - name: 'location' - required: true - description: | - The geographic location where the dataset should reside. - See https://cloud.google.com/bigquery/docs/locations for supported locations. - - !ruby/object:Api::Type::String - name: 'datasetIdPrefix' - description: | - If supplied, every created dataset will have its name prefixed by the provided value. - The prefix and name will be separated by an underscore. i.e. _. - - !ruby/object:Api::Type::String - name: 'state' - description: The state of the stream. - output: true - - !ruby/object:Api::Type::NestedObject - name: 'backfillAll' - exactly_one_of: - - backfill_all - - backfill_none - allow_empty_object: true - send_empty_value: true - description: | - Backfill strategy to automatically backfill the Stream's objects. Specific objects can be excluded. - properties: - - !ruby/object:Api::Type::NestedObject - name: 'mysqlExcludedObjects' - description: | - MySQL data source objects to avoid backfilling. - properties: - - !ruby/object:Api::Type::Array - name: 'mysqlDatabases' - required: true - min_size: 1 - description: | - MySQL databases on the server - item_type: !ruby/object:Api::Type::NestedObject - description: | - MySQL database. - properties: - - !ruby/object:Api::Type::String - name: 'database' - required: true - description: | - Database name. - - !ruby/object:Api::Type::Array - name: 'mysqlTables' - min_size: 1 - description: | - Tables in the database. - item_type: !ruby/object:Api::Type::NestedObject - description: | - MySQL table. - properties: - - !ruby/object:Api::Type::String - name: 'table' - required: true - description: | - Table name. - - !ruby/object:Api::Type::Array - name: 'mysqlColumns' - min_size: 1 - description: | - MySQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. - item_type: !ruby/object:Api::Type::NestedObject - description: | - MySQL Column. - properties: - - !ruby/object:Api::Type::String - name: 'column' - description: | - Column name. - - !ruby/object:Api::Type::String - name: 'dataType' - description: | - The MySQL data type. Full data types list can be found here: - https://dev.mysql.com/doc/refman/8.0/en/data-types.html - - !ruby/object:Api::Type::Integer - name: 'length' - output: true - description: | - Column length. - - !ruby/object:Api::Type::String - name: 'collation' - description: | - Column collation. - - !ruby/object:Api::Type::Boolean - name: 'primaryKey' - description: | - Whether or not the column represents a primary key. - - !ruby/object:Api::Type::Boolean - name: 'nullable' - description: | - Whether or not the column can accept a null value. - - !ruby/object:Api::Type::Integer - name: 'ordinalPosition' - description: | - The ordinal position of the column in the table. - - !ruby/object:Api::Type::NestedObject - name: 'postgresqlExcludedObjects' - description: | - PostgreSQL data source objects to avoid backfilling. - properties: - - !ruby/object:Api::Type::Array - name: 'postgresqlSchemas' - required: true - min_size: 1 - description: | - PostgreSQL schemas on the server - item_type: !ruby/object:Api::Type::NestedObject - description: | - PostgreSQL schema. - properties: - - !ruby/object:Api::Type::String - name: 'schema' - required: true - description: | - Database name. - - !ruby/object:Api::Type::Array - name: 'postgresqlTables' - min_size: 1 - description: | - Tables in the schema. - item_type: !ruby/object:Api::Type::NestedObject - description: | - PostgreSQL table. - properties: - - !ruby/object:Api::Type::String - name: 'table' - required: true - description: | - Table name. - - !ruby/object:Api::Type::Array - name: 'postgresqlColumns' - min_size: 1 - description: | - PostgreSQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. - item_type: !ruby/object:Api::Type::NestedObject - description: | - PostgreSQL Column. - properties: - - !ruby/object:Api::Type::String - name: 'column' - description: | - Column name. - - !ruby/object:Api::Type::String - name: 'dataType' - description: | - The PostgreSQL data type. Full data types list can be found here: - https://www.postgresql.org/docs/current/datatype.html - - !ruby/object:Api::Type::Integer - name: 'length' - output: true - description: | - Column length. - - !ruby/object:Api::Type::Integer - name: 'precision' - output: true - description: | - Column precision. - - !ruby/object:Api::Type::Integer - name: 'scale' - output: true - description: | - Column scale. - - !ruby/object:Api::Type::Boolean - name: 'primaryKey' - description: | - Whether or not the column represents a primary key. - - !ruby/object:Api::Type::Boolean - name: 'nullable' - description: | - Whether or not the column can accept a null value. - - !ruby/object:Api::Type::Integer - name: 'ordinalPosition' - description: | - The ordinal position of the column in the table. - - !ruby/object:Api::Type::NestedObject - name: 'oracleExcludedObjects' + - backfill_all + - backfill_none + allow_empty_object: true + send_empty_value: true description: | - PostgreSQL data source objects to avoid backfilling. - properties: - - !ruby/object:Api::Type::Array - name: 'oracleSchemas' - required: true - min_size: 1 - description: | - Oracle schemas/databases in the database server - item_type: !ruby/object:Api::Type::NestedObject - description: | - MySQL database. - properties: - - !ruby/object:Api::Type::String - name: 'schema' - required: true - description: | - Schema name. - - !ruby/object:Api::Type::Array - name: 'oracleTables' - min_size: 1 - description: | - Tables in the database. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle table. - properties: - - !ruby/object:Api::Type::String - name: 'table' - required: true - description: | - Table name. - - !ruby/object:Api::Type::Array - name: 'oracleColumns' - min_size: 1 - description: | - Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle Column. - properties: - - !ruby/object:Api::Type::String - name: 'column' - description: | - Column name. - - !ruby/object:Api::Type::String - name: 'dataType' - description: | - The Oracle data type. Full data types list can be found here: - https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/Data-Types.html - - !ruby/object:Api::Type::Integer - name: 'length' - output: true - description: | - Column length. - - !ruby/object:Api::Type::Integer - name: 'precision' - output: true - description: | - Column precision. - - !ruby/object:Api::Type::Integer - name: 'scale' - output: true - description: | - Column scale. - - !ruby/object:Api::Type::String - name: 'encoding' - output: true - description: | - Column encoding. - - !ruby/object:Api::Type::Boolean - name: 'primaryKey' - output: true - description: | - Whether or not the column represents a primary key. - - !ruby/object:Api::Type::Boolean - name: 'nullable' - output: true - description: | - Whether or not the column can accept a null value. - - !ruby/object:Api::Type::Integer - name: 'ordinalPosition' - output: true - description: | - The ordinal position of the column in the table. - - !ruby/object:Api::Type::NestedObject - name: 'backfillNone' - exactly_one_of: - - backfill_all - - backfill_none - allow_empty_object: true - send_empty_value: true - description: | - Backfill strategy to disable automatic backfill for the Stream's objects. - properties: [] \ No newline at end of file + Backfill strategy to disable automatic backfill for the Stream's objects. + properties: [] diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index 10f643959b5f..ae17882eefa1 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -50,45 +50,45 @@ overrides: !ruby/object:Overrides::ResourceOverrides # Workaround for https://github.com/hashicorp/terraform-provider-google/issues/12410 custom_flatten: templates/terraform/custom_flatten/datastream_connection_profile_forward_ssh_connectivity_private_key.go.erb examples: - - !ruby/object:Provider::Terraform::Examples - name: "datastream_connection_profile_basic" - primary_resource_id: "default" - vars: - connection_profile_id: "my-profile" - - !ruby/object:Provider::Terraform::Examples - name: "datastream_connection_profile_bigquery_private_connection" - primary_resource_id: "default" - vars: - private_connection_id: "my-connection" - connection_profile_id: "my-profile" - network_name: "my-network" - - !ruby/object:Provider::Terraform::Examples - name: "datastream_connection_profile_full" - primary_resource_id: "default" - # Workaround for https://github.com/hashicorp/terraform-provider-google/issues/12410 - ignore_read_extra: - - "forward_ssh_connectivity.0.password" - vars: - connection_profile_id: "my-profile" - - !ruby/object:Provider::Terraform::Examples - name: "datastream_connection_profile_postgres" - primary_resource_id: "default" - vars: - connection_profile_id: "my-profile" - database_instance_name: "my-instance" - deletion_protection: "true" - # This is covered by the update tests. - skip_test: true + - !ruby/object:Provider::Terraform::Examples + name: "datastream_connection_profile_basic" + primary_resource_id: "default" + vars: + connection_profile_id: "my-profile" + - !ruby/object:Provider::Terraform::Examples + name: "datastream_connection_profile_bigquery_private_connection" + primary_resource_id: "default" + vars: + private_connection_id: "my-connection" + connection_profile_id: "my-profile" + network_name: "my-network" + - !ruby/object:Provider::Terraform::Examples + name: "datastream_connection_profile_full" + primary_resource_id: "default" + # Workaround for https://github.com/hashicorp/terraform-provider-google/issues/12410 + ignore_read_extra: + - "forward_ssh_connectivity.0.password" + vars: + connection_profile_id: "my-profile" + - !ruby/object:Provider::Terraform::Examples + name: "datastream_connection_profile_postgres" + primary_resource_id: "default" + vars: + connection_profile_id: "my-profile" + database_instance_name: "my-instance" + deletion_protection: "true" + # This is covered by the update tests. + skip_test: true PrivateConnection: !ruby/object:Overrides::Terraform::ResourceOverride id_format: projects/{{project}}/locations/{{location}}/privateConnections/{{private_connection_id}} import_format: ["projects/{{project}}/locations/{{location}}/privateConnections/{{private_connection_id}}"] examples: - - !ruby/object:Provider::Terraform::Examples - name: "datastream_private_connection_full" - primary_resource_id: "default" - vars: - private_connection_id: "my-connection" - network_name: "my-network" + - !ruby/object:Provider::Terraform::Examples + name: "datastream_private_connection_full" + primary_resource_id: "default" + vars: + private_connection_id: "my-connection" + network_name: "my-network" custom_code: !ruby/object:Provider::Terraform::CustomCode constants: templates/terraform/constants/private_connection.go.erb post_create: templates/terraform/post_create/private_connection.go.erb @@ -122,15 +122,15 @@ overrides: !ruby/object:Overrides::ResourceOverrides validation: !ruby/object:Provider::Terraform::Validation function: 'validation.IntAtLeast(0)' virtual_fields: - - !ruby/object:Api::Type::Enum - name: 'desired_state' - description: | - Desired state of the Stream. Set this field to `RUNNING` to start the stream, and `PAUSED` to pause the stream. - values: - - :NOT_STARTED - - :RUNNING - - :PAUSED - default_value: :NOT_STARTED + - !ruby/object:Api::Type::Enum + name: 'desired_state' + description: | + Desired state of the Stream. Set this field to `RUNNING` to start the stream, and `PAUSED` to pause the stream. + values: + - :NOT_STARTED + - :RUNNING + - :PAUSED + default_value: :NOT_STARTED custom_code: !ruby/object:Provider::Terraform::CustomCode constants: 'templates/terraform/constants/datastream_stream.go.erb' post_create: 'templates/terraform/post_create/datastream_stream.go.erb' @@ -140,59 +140,59 @@ overrides: !ruby/object:Overrides::ResourceOverrides encoder: 'templates/terraform/encoders/datastream_stream.go.erb' resource_definition: 'templates/terraform/resource_definition/datastream_stream.go.erb' examples: - - !ruby/object:Provider::Terraform::Examples - name: "datastream_stream_basic" - pull_external: true - primary_resource_id: "default" - skip_docs: true - # Random provider - skip_vcr: true - vars: - stream_id: "my-stream" - private_connection_id: "my-connection" - network_name: "my-network" - source_connection_profile_id: "source-profile" - database_instance_name: "my-instance" - deletion_protection: "true" - bucket_name: "my-bucket" - destination_connection_profile_id: "destination-profile" - test_vars_overrides: - deletion_protection: "false" - - !ruby/object:Provider::Terraform::Examples - name: "datastream_stream_full" - pull_external: true - primary_resource_id: "default" - # Random provider - skip_vcr: true - vars: - stream_id: "my-stream" - private_connection_id: "my-connection" - network_name: "my-network" - source_connection_profile_id: "source-profile" - database_instance_name: "my-instance" - deletion_protection: "true" - bucket_name: "my-bucket" - destination_connection_profile_id: "destination-profile" - test_vars_overrides: - deletion_protection: "false" - - !ruby/object:Provider::Terraform::Examples - name: "datastream_stream_postgresql" - pull_external: true - primary_resource_id: "default" - skip_vcr: true - vars: - stream_id: "my-stream" - source_connection_profile_id: "source-profile" - destination_connection_profile_id: "destination-profile" - - !ruby/object:Provider::Terraform::Examples - name: "datastream_stream_oracle" - pull_external: true - primary_resource_id: "default" - skip_vcr: true - vars: - stream_id: "my-stream" - source_connection_profile_id: "source-profile" - destination_connection_profile_id: "destination-profile" + - !ruby/object:Provider::Terraform::Examples + name: "datastream_stream_basic" + pull_external: true + primary_resource_id: "default" + skip_docs: true + # Random provider + skip_vcr: true + vars: + stream_id: "my-stream" + private_connection_id: "my-connection" + network_name: "my-network" + source_connection_profile_id: "source-profile" + database_instance_name: "my-instance" + deletion_protection: "true" + bucket_name: "my-bucket" + destination_connection_profile_id: "destination-profile" + test_vars_overrides: + deletion_protection: "false" + - !ruby/object:Provider::Terraform::Examples + name: "datastream_stream_full" + pull_external: true + primary_resource_id: "default" + # Random provider + skip_vcr: true + vars: + stream_id: "my-stream" + private_connection_id: "my-connection" + network_name: "my-network" + source_connection_profile_id: "source-profile" + database_instance_name: "my-instance" + deletion_protection: "true" + bucket_name: "my-bucket" + destination_connection_profile_id: "destination-profile" + test_vars_overrides: + deletion_protection: "false" + - !ruby/object:Provider::Terraform::Examples + name: "datastream_stream_postgresql" + pull_external: true + primary_resource_id: "default" + skip_vcr: true + vars: + stream_id: "my-stream" + source_connection_profile_id: "source-profile" + destination_connection_profile_id: "destination-profile" + - !ruby/object:Provider::Terraform::Examples + name: "datastream_stream_oracle" + pull_external: true + primary_resource_id: "default" + skip_vcr: true + vars: + stream_id: "my-stream" + source_connection_profile_id: "source-profile" + destination_connection_profile_id: "destination-profile" # This is for copying files over