From 4c6d7b477bf0932d87980b7c312011da1b5734ec Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Fri, 9 Aug 2019 15:11:45 +0300 Subject: [PATCH 01/68] Move every snippet to it's own file. Create test templates. --- bigquery/docs/snippets.py | 2659 ----------------- bigquery/docs/usage/datasets.rst | 27 + bigquery/docs/usage/tables.rst | 450 ++- bigquery/samples/add_empty_column.py | 46 + bigquery/samples/browse_table_data.py | 56 + bigquery/samples/client_list_jobs.py | 51 + bigquery/samples/client_query.py | 40 + bigquery/samples/client_query_add_column.py | 72 + bigquery/samples/client_query_batch.py | 43 + .../samples/client_query_destination_table.py | 53 + .../client_query_destination_table_cmek.py | 59 + .../client_query_destination_table_legacy.py | 54 + bigquery/samples/client_query_dry_run.py | 47 + bigquery/samples/client_query_legacy_sql.py | 44 + bigquery/samples/client_query_relax_column.py | 76 + bigquery/samples/client_query_total_rows.py | 40 + .../samples/client_query_w_array_params.py | 54 + .../samples/client_query_w_named_params.py | 53 + .../client_query_w_positional_params.py | 55 + .../samples/client_query_w_struct_params.py | 50 + .../client_query_w_timestamp_params.py | 53 + bigquery/samples/copy_table.py | 47 + bigquery/samples/copy_table_cmek.py | 57 + .../samples/copy_table_multiple_source.py | 73 + .../create_client_default_credentials.py | 29 + bigquery/samples/create_partitioned_table.py | 53 + bigquery/samples/create_table_cmek.py | 44 + .../create_table_nested_repeated_schema.py | 54 + bigquery/samples/dataset_exists.py | 29 + bigquery/samples/ddl_create_view.py | 72 + bigquery/samples/extract_table.py | 52 + bigquery/samples/extract_table_compressed.py | 48 + bigquery/samples/extract_table_json.py | 48 + bigquery/samples/list_datasets_by_label.py | 42 + bigquery/samples/list_rows_as_dataframe.py | 31 + .../load_and_query_partitioned_table.py | 87 + bigquery/samples/load_table_add_column.py | 83 + bigquery/samples/load_table_from_dataframe.py | 60 + bigquery/samples/load_table_from_file.py | 61 + .../samples/load_table_from_uri_autodetect.py | 80 + bigquery/samples/load_table_from_uri_avro.py | 47 + bigquery/samples/load_table_from_uri_cmek.py | 58 + bigquery/samples/load_table_from_uri_csv.py | 53 + bigquery/samples/load_table_from_uri_json.py | 55 + bigquery/samples/load_table_from_uri_orc.py | 47 + .../samples/load_table_from_uri_parquet.py | 47 + .../samples/load_table_from_uri_truncate.py | 117 + bigquery/samples/load_table_relax_column.py | 92 + bigquery/samples/manage_dataset_labels.py | 72 + bigquery/samples/manage_job.py | 54 + bigquery/samples/manage_table_labels.py | 78 + bigquery/samples/manage_views.py | 134 + .../query_external_gcs_permanent_table.py | 55 + .../query_external_gcs_temporary_table.py | 45 + .../query_external_sheets_permanent_table.py | 70 + .../query_external_sheets_temporary_table.py | 62 + bigquery/samples/query_no_cache.py | 40 + .../samples/query_results_as_dataframe.py | 35 + bigquery/samples/relax_column.py | 51 + bigquery/samples/table_exists.py | 34 + bigquery/samples/table_insert_rows.py | 45 + .../samples/tests/test_add_empty_column.py | 27 + .../samples/tests/test_browse_table_data.py | 22 + .../samples/tests/test_client_list_jobs.py | 22 + bigquery/samples/tests/test_client_query.py | 22 + .../tests/test_client_query_add_column.py | 22 + .../samples/tests/test_client_query_batch.py | 22 + .../test_client_query_destination_table.py | 22 + ...est_client_query_destination_table_cmek.py | 22 + ...t_client_query_destination_table_legacy.py | 22 + .../tests/test_client_query_dry_run.py | 22 + .../tests/test_client_query_legacy_sql.py | 22 + .../tests/test_client_query_relax_column.py | 22 + .../tests/test_client_query_total_rows.py | 22 + .../tests/test_client_query_w_array_params.py | 22 + .../tests/test_client_query_w_named_params.py | 22 + .../test_client_query_w_positional_params.py | 22 + .../test_client_query_w_struct_params.py | 22 + .../test_client_query_w_timestamp_params.py | 22 + bigquery/samples/tests/test_copy_table.py | 22 + .../samples/tests/test_copy_table_cmek.py | 22 + .../tests/test_copy_table_multiple_source.py | 24 + .../test_create_client_default_credentials.py | 22 + .../tests/test_create_partitioned_table.py | 22 + .../samples/tests/test_create_table_cmek.py | 22 + ...est_create_table_nested_repeated_schema.py | 22 + bigquery/samples/tests/test_dataset_exists.py | 22 + .../samples/tests/test_ddl_create_view.py | 22 + bigquery/samples/tests/test_extract_table.py | 30 + .../tests/test_extract_table_compressed.py | 30 + .../samples/tests/test_extract_table_json.py | 30 + .../tests/test_list_datasets_by_label.py | 22 + .../tests/test_list_rows_as_dataframe.py | 22 + .../test_load_and_query_partitioned_table.py | 22 + .../tests/test_load_table_add_column.py | 24 + .../tests/test_load_table_from_dataframe.py | 22 + .../tests/test_load_table_from_file.py | 24 + .../test_load_table_from_uri_autodetect.py | 22 + .../tests/test_load_table_from_uri_avro.py | 22 + .../tests/test_load_table_from_uri_cmek.py | 22 + .../tests/test_load_table_from_uri_csv.py | 22 + .../tests/test_load_table_from_uri_json.py | 22 + .../tests/test_load_table_from_uri_orc.py | 22 + .../tests/test_load_table_from_uri_parquet.py | 22 + .../test_load_table_from_uri_truncate.py | 24 + .../tests/test_load_table_relax_column.py | 24 + .../tests/test_manage_dataset_labels.py | 22 + bigquery/samples/tests/test_manage_job.py | 22 + .../samples/tests/test_manage_table_labels.py | 27 + bigquery/samples/tests/test_manage_views.py | 22 + ...test_query_external_gcs_permanent_table.py | 22 + ...test_query_external_gcs_temporary_table.py | 22 + ...t_query_external_sheets_permanent_table.py | 22 + ...t_query_external_sheets_temporary_table.py | 22 + bigquery/samples/tests/test_query_no_cache.py | 22 + .../tests/test_query_results_as_dataframe.py | 22 + bigquery/samples/tests/test_relax_column.py | 22 + bigquery/samples/tests/test_table_exists.py | 27 + .../samples/tests/test_table_insert_rows.py | 27 + bigquery/samples/tests/test_undelete_table.py | 27 + .../samples/tests/test_update_table_cmek.py | 22 + .../tests/test_update_table_description.py | 27 + .../tests/test_update_table_expiration.py | 27 + bigquery/samples/undelete_table.py | 77 + bigquery/samples/update_table_cmek.py | 55 + bigquery/samples/update_table_description.py | 42 + bigquery/samples/update_table_expiration.py | 48 + 127 files changed, 5410 insertions(+), 2668 deletions(-) create mode 100644 bigquery/samples/add_empty_column.py create mode 100644 bigquery/samples/browse_table_data.py create mode 100644 bigquery/samples/client_list_jobs.py create mode 100644 bigquery/samples/client_query.py create mode 100644 bigquery/samples/client_query_add_column.py create mode 100644 bigquery/samples/client_query_batch.py create mode 100644 bigquery/samples/client_query_destination_table.py create mode 100644 bigquery/samples/client_query_destination_table_cmek.py create mode 100644 bigquery/samples/client_query_destination_table_legacy.py create mode 100644 bigquery/samples/client_query_dry_run.py create mode 100644 bigquery/samples/client_query_legacy_sql.py create mode 100644 bigquery/samples/client_query_relax_column.py create mode 100644 bigquery/samples/client_query_total_rows.py create mode 100644 bigquery/samples/client_query_w_array_params.py create mode 100644 bigquery/samples/client_query_w_named_params.py create mode 100644 bigquery/samples/client_query_w_positional_params.py create mode 100644 bigquery/samples/client_query_w_struct_params.py create mode 100644 bigquery/samples/client_query_w_timestamp_params.py create mode 100644 bigquery/samples/copy_table.py create mode 100644 bigquery/samples/copy_table_cmek.py create mode 100644 bigquery/samples/copy_table_multiple_source.py create mode 100644 bigquery/samples/create_client_default_credentials.py create mode 100644 bigquery/samples/create_partitioned_table.py create mode 100644 bigquery/samples/create_table_cmek.py create mode 100644 bigquery/samples/create_table_nested_repeated_schema.py create mode 100644 bigquery/samples/dataset_exists.py create mode 100644 bigquery/samples/ddl_create_view.py create mode 100644 bigquery/samples/extract_table.py create mode 100644 bigquery/samples/extract_table_compressed.py create mode 100644 bigquery/samples/extract_table_json.py create mode 100644 bigquery/samples/list_datasets_by_label.py create mode 100644 bigquery/samples/list_rows_as_dataframe.py create mode 100644 bigquery/samples/load_and_query_partitioned_table.py create mode 100644 bigquery/samples/load_table_add_column.py create mode 100644 bigquery/samples/load_table_from_dataframe.py create mode 100644 bigquery/samples/load_table_from_file.py create mode 100644 bigquery/samples/load_table_from_uri_autodetect.py create mode 100644 bigquery/samples/load_table_from_uri_avro.py create mode 100644 bigquery/samples/load_table_from_uri_cmek.py create mode 100644 bigquery/samples/load_table_from_uri_csv.py create mode 100644 bigquery/samples/load_table_from_uri_json.py create mode 100644 bigquery/samples/load_table_from_uri_orc.py create mode 100644 bigquery/samples/load_table_from_uri_parquet.py create mode 100644 bigquery/samples/load_table_from_uri_truncate.py create mode 100644 bigquery/samples/load_table_relax_column.py create mode 100644 bigquery/samples/manage_dataset_labels.py create mode 100644 bigquery/samples/manage_job.py create mode 100644 bigquery/samples/manage_table_labels.py create mode 100644 bigquery/samples/manage_views.py create mode 100644 bigquery/samples/query_external_gcs_permanent_table.py create mode 100644 bigquery/samples/query_external_gcs_temporary_table.py create mode 100644 bigquery/samples/query_external_sheets_permanent_table.py create mode 100644 bigquery/samples/query_external_sheets_temporary_table.py create mode 100644 bigquery/samples/query_no_cache.py create mode 100644 bigquery/samples/query_results_as_dataframe.py create mode 100644 bigquery/samples/relax_column.py create mode 100644 bigquery/samples/table_exists.py create mode 100644 bigquery/samples/table_insert_rows.py create mode 100644 bigquery/samples/tests/test_add_empty_column.py create mode 100644 bigquery/samples/tests/test_browse_table_data.py create mode 100644 bigquery/samples/tests/test_client_list_jobs.py create mode 100644 bigquery/samples/tests/test_client_query.py create mode 100644 bigquery/samples/tests/test_client_query_add_column.py create mode 100644 bigquery/samples/tests/test_client_query_batch.py create mode 100644 bigquery/samples/tests/test_client_query_destination_table.py create mode 100644 bigquery/samples/tests/test_client_query_destination_table_cmek.py create mode 100644 bigquery/samples/tests/test_client_query_destination_table_legacy.py create mode 100644 bigquery/samples/tests/test_client_query_dry_run.py create mode 100644 bigquery/samples/tests/test_client_query_legacy_sql.py create mode 100644 bigquery/samples/tests/test_client_query_relax_column.py create mode 100644 bigquery/samples/tests/test_client_query_total_rows.py create mode 100644 bigquery/samples/tests/test_client_query_w_array_params.py create mode 100644 bigquery/samples/tests/test_client_query_w_named_params.py create mode 100644 bigquery/samples/tests/test_client_query_w_positional_params.py create mode 100644 bigquery/samples/tests/test_client_query_w_struct_params.py create mode 100644 bigquery/samples/tests/test_client_query_w_timestamp_params.py create mode 100644 bigquery/samples/tests/test_copy_table.py create mode 100644 bigquery/samples/tests/test_copy_table_cmek.py create mode 100644 bigquery/samples/tests/test_copy_table_multiple_source.py create mode 100644 bigquery/samples/tests/test_create_client_default_credentials.py create mode 100644 bigquery/samples/tests/test_create_partitioned_table.py create mode 100644 bigquery/samples/tests/test_create_table_cmek.py create mode 100644 bigquery/samples/tests/test_create_table_nested_repeated_schema.py create mode 100644 bigquery/samples/tests/test_dataset_exists.py create mode 100644 bigquery/samples/tests/test_ddl_create_view.py create mode 100644 bigquery/samples/tests/test_extract_table.py create mode 100644 bigquery/samples/tests/test_extract_table_compressed.py create mode 100644 bigquery/samples/tests/test_extract_table_json.py create mode 100644 bigquery/samples/tests/test_list_datasets_by_label.py create mode 100644 bigquery/samples/tests/test_list_rows_as_dataframe.py create mode 100644 bigquery/samples/tests/test_load_and_query_partitioned_table.py create mode 100644 bigquery/samples/tests/test_load_table_add_column.py create mode 100644 bigquery/samples/tests/test_load_table_from_dataframe.py create mode 100644 bigquery/samples/tests/test_load_table_from_file.py create mode 100644 bigquery/samples/tests/test_load_table_from_uri_autodetect.py create mode 100644 bigquery/samples/tests/test_load_table_from_uri_avro.py create mode 100644 bigquery/samples/tests/test_load_table_from_uri_cmek.py create mode 100644 bigquery/samples/tests/test_load_table_from_uri_csv.py create mode 100644 bigquery/samples/tests/test_load_table_from_uri_json.py create mode 100644 bigquery/samples/tests/test_load_table_from_uri_orc.py create mode 100644 bigquery/samples/tests/test_load_table_from_uri_parquet.py create mode 100644 bigquery/samples/tests/test_load_table_from_uri_truncate.py create mode 100644 bigquery/samples/tests/test_load_table_relax_column.py create mode 100644 bigquery/samples/tests/test_manage_dataset_labels.py create mode 100644 bigquery/samples/tests/test_manage_job.py create mode 100644 bigquery/samples/tests/test_manage_table_labels.py create mode 100644 bigquery/samples/tests/test_manage_views.py create mode 100644 bigquery/samples/tests/test_query_external_gcs_permanent_table.py create mode 100644 bigquery/samples/tests/test_query_external_gcs_temporary_table.py create mode 100644 bigquery/samples/tests/test_query_external_sheets_permanent_table.py create mode 100644 bigquery/samples/tests/test_query_external_sheets_temporary_table.py create mode 100644 bigquery/samples/tests/test_query_no_cache.py create mode 100644 bigquery/samples/tests/test_query_results_as_dataframe.py create mode 100644 bigquery/samples/tests/test_relax_column.py create mode 100644 bigquery/samples/tests/test_table_exists.py create mode 100644 bigquery/samples/tests/test_table_insert_rows.py create mode 100644 bigquery/samples/tests/test_undelete_table.py create mode 100644 bigquery/samples/tests/test_update_table_cmek.py create mode 100644 bigquery/samples/tests/test_update_table_description.py create mode 100644 bigquery/samples/tests/test_update_table_expiration.py create mode 100644 bigquery/samples/undelete_table.py create mode 100644 bigquery/samples/update_table_cmek.py create mode 100644 bigquery/samples/update_table_description.py create mode 100644 bigquery/samples/update_table_expiration.py diff --git a/bigquery/docs/snippets.py b/bigquery/docs/snippets.py index 249fbf7baafa..b104e79ea789 100644 --- a/bigquery/docs/snippets.py +++ b/bigquery/docs/snippets.py @@ -50,30 +50,6 @@ from google.cloud import storage from test_utils.retry import RetryErrors -ORIGINAL_FRIENDLY_NAME = "Original friendly name" -ORIGINAL_DESCRIPTION = "Original description" -LOCALLY_CHANGED_FRIENDLY_NAME = "Locally-changed friendly name" -LOCALLY_CHANGED_DESCRIPTION = "Locally-changed description" -UPDATED_FRIENDLY_NAME = "Updated friendly name" -UPDATED_DESCRIPTION = "Updated description" - -SCHEMA = [ - bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), -] - -ROWS = [ - ("Phred Phlyntstone", 32), - ("Bharney Rhubble", 33), - ("Wylma Phlyntstone", 29), - ("Bhettye Rhubble", 27), -] - -QUERY = ( - "SELECT name FROM `bigquery-public-data.usa_names.usa_1910_2013` " - 'WHERE state = "TX"' -) - retry_429 = RetryErrors(TooManyRequests) retry_storage_errors = RetryErrors( @@ -111,48 +87,6 @@ def delete(self): self._wrapped.close() -def test_create_client_default_credentials(): - """Create a BigQuery client with Application Default Credentials""" - - # [START bigquery_client_default_credentials] - from google.cloud import bigquery - - # If you don't specify credentials when constructing the client, the - # client library will look for credentials in the environment. - client = bigquery.Client() - # [END bigquery_client_default_credentials] - - assert client is not None - - -def test_list_datasets_by_label(client, to_delete): - dataset_id = "list_datasets_by_label_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - dataset.labels = {"color": "green"} - dataset = client.create_dataset(dataset) # API request - to_delete.append(dataset) - - # [START bigquery_list_datasets_by_label] - # from google.cloud import bigquery - # client = bigquery.Client() - - # The following label filter example will find datasets with an - # arbitrary 'color' label set to 'green' - label_filter = "labels.color:green" - datasets = list(client.list_datasets(filter=label_filter)) - - if datasets: - print("Datasets filtered by {}:".format(label_filter)) - for dataset in datasets: # API request(s) - print("\t{}".format(dataset.dataset_id)) - else: - print("No datasets found with this filter.") - # [END bigquery_list_datasets_by_label] - found = set([dataset.dataset_id for dataset in datasets]) - assert dataset_id in found - - -# [START bigquery_dataset_exists] def dataset_exists(client, dataset_reference): """Return if a dataset exists. @@ -174,263 +108,6 @@ def dataset_exists(client, dataset_reference): return False -# [END bigquery_dataset_exists] - - -def test_dataset_exists(client, to_delete): - """Determine if a dataset exists.""" - DATASET_ID = "get_table_dataset_{}".format(_millis()) - dataset_ref = client.dataset(DATASET_ID) - dataset = bigquery.Dataset(dataset_ref) - dataset = client.create_dataset(dataset) - to_delete.append(dataset) - - assert dataset_exists(client, dataset_ref) - assert not dataset_exists(client, client.dataset("i_dont_exist")) - - -@pytest.mark.skip( - reason=( - "update_dataset() is flaky " - "https://github.com/GoogleCloudPlatform/google-cloud-python/issues/5588" - ) -) -def test_manage_dataset_labels(client, to_delete): - dataset_id = "label_dataset_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - dataset = client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_label_dataset] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_ref = client.dataset('my_dataset') - # dataset = client.get_dataset(dataset_ref) # API request - - assert dataset.labels == {} - labels = {"color": "green"} - dataset.labels = labels - - dataset = client.update_dataset(dataset, ["labels"]) # API request - - assert dataset.labels == labels - # [END bigquery_label_dataset] - - # [START bigquery_get_dataset_labels] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' - - dataset_ref = client.dataset(dataset_id) - dataset = client.get_dataset(dataset_ref) # API request - - # View dataset labels - print("Dataset ID: {}".format(dataset_id)) - print("Labels:") - if dataset.labels: - for label, value in dataset.labels.items(): - print("\t{}: {}".format(label, value)) - else: - print("\tDataset has no labels defined.") - # [END bigquery_get_dataset_labels] - assert dataset.labels == labels - - # [START bigquery_delete_label_dataset] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_ref = client.dataset('my_dataset') - # dataset = client.get_dataset(dataset_ref) # API request - - # This example dataset starts with one label - assert dataset.labels == {"color": "green"} - # To delete a label from a dataset, set its value to None - dataset.labels["color"] = None - - dataset = client.update_dataset(dataset, ["labels"]) # API request - - assert dataset.labels == {} - # [END bigquery_delete_label_dataset] - - -def test_create_table_nested_repeated_schema(client, to_delete): - dataset_id = "create_table_nested_repeated_{}".format(_millis()) - dataset_ref = client.dataset(dataset_id) - dataset = bigquery.Dataset(dataset_ref) - client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_nested_repeated_schema] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_ref = client.dataset('my_dataset') - - schema = [ - bigquery.SchemaField("id", "STRING", mode="NULLABLE"), - bigquery.SchemaField("first_name", "STRING", mode="NULLABLE"), - bigquery.SchemaField("last_name", "STRING", mode="NULLABLE"), - bigquery.SchemaField("dob", "DATE", mode="NULLABLE"), - bigquery.SchemaField( - "addresses", - "RECORD", - mode="REPEATED", - fields=[ - bigquery.SchemaField("status", "STRING", mode="NULLABLE"), - bigquery.SchemaField("address", "STRING", mode="NULLABLE"), - bigquery.SchemaField("city", "STRING", mode="NULLABLE"), - bigquery.SchemaField("state", "STRING", mode="NULLABLE"), - bigquery.SchemaField("zip", "STRING", mode="NULLABLE"), - bigquery.SchemaField("numberOfYears", "STRING", mode="NULLABLE"), - ], - ), - ] - table_ref = dataset_ref.table("my_table") - table = bigquery.Table(table_ref, schema=schema) - table = client.create_table(table) # API request - - print("Created table {}".format(table.full_table_id)) - # [END bigquery_nested_repeated_schema] - - -def test_create_table_cmek(client, to_delete): - dataset_id = "create_table_cmek_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_create_table_cmek] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' - - table_ref = client.dataset(dataset_id).table("my_table") - table = bigquery.Table(table_ref) - - # Set the encryption key to use for the table. - # TODO: Replace this key with a key you have created in Cloud KMS. - kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format( - "cloud-samples-tests", "us-central1", "test", "test" - ) - table.encryption_configuration = bigquery.EncryptionConfiguration( - kms_key_name=kms_key_name - ) - - table = client.create_table(table) # API request - - assert table.encryption_configuration.kms_key_name == kms_key_name - # [END bigquery_create_table_cmek] - - -def test_create_partitioned_table(client, to_delete): - dataset_id = "create_table_partitioned_{}".format(_millis()) - dataset_ref = bigquery.Dataset(client.dataset(dataset_id)) - dataset = client.create_dataset(dataset_ref) - to_delete.append(dataset) - - # [START bigquery_create_table_partitioned] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_ref = client.dataset('my_dataset') - - table_ref = dataset_ref.table("my_partitioned_table") - schema = [ - bigquery.SchemaField("name", "STRING"), - bigquery.SchemaField("post_abbr", "STRING"), - bigquery.SchemaField("date", "DATE"), - ] - table = bigquery.Table(table_ref, schema=schema) - table.time_partitioning = bigquery.TimePartitioning( - type_=bigquery.TimePartitioningType.DAY, - field="date", # name of column to use for partitioning - expiration_ms=7776000000, - ) # 90 days - - table = client.create_table(table) - - print( - "Created table {}, partitioned on column {}".format( - table.table_id, table.time_partitioning.field - ) - ) - # [END bigquery_create_table_partitioned] - - assert table.time_partitioning.type_ == "DAY" - assert table.time_partitioning.field == "date" - assert table.time_partitioning.expiration_ms == 7776000000 - - -def test_load_and_query_partitioned_table(client, to_delete): - dataset_id = "load_partitioned_table_dataset_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_load_table_partitioned] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' - table_id = "us_states_by_date" - - dataset_ref = client.dataset(dataset_id) - job_config = bigquery.LoadJobConfig() - job_config.schema = [ - bigquery.SchemaField("name", "STRING"), - bigquery.SchemaField("post_abbr", "STRING"), - bigquery.SchemaField("date", "DATE"), - ] - job_config.skip_leading_rows = 1 - job_config.time_partitioning = bigquery.TimePartitioning( - type_=bigquery.TimePartitioningType.DAY, - field="date", # name of column to use for partitioning - expiration_ms=7776000000, - ) # 90 days - uri = "gs://cloud-samples-data/bigquery/us-states/us-states-by-date.csv" - - load_job = client.load_table_from_uri( - uri, dataset_ref.table(table_id), job_config=job_config - ) # API request - - assert load_job.job_type == "load" - - load_job.result() # Waits for table load to complete. - - table = client.get_table(dataset_ref.table(table_id)) - print("Loaded {} rows to table {}".format(table.num_rows, table_id)) - # [END bigquery_load_table_partitioned] - assert table.num_rows == 50 - - project_id = client.project - - # [START bigquery_query_partitioned_table] - import datetime - - # from google.cloud import bigquery - # client = bigquery.Client() - # project_id = 'my-project' - # dataset_id = 'my_dataset' - table_id = "us_states_by_date" - - sql_template = """ - SELECT * - FROM `{}.{}.{}` - WHERE date BETWEEN @start_date AND @end_date - """ - sql = sql_template.format(project_id, dataset_id, table_id) - job_config = bigquery.QueryJobConfig() - job_config.query_parameters = [ - bigquery.ScalarQueryParameter("start_date", "DATE", datetime.date(1800, 1, 1)), - bigquery.ScalarQueryParameter("end_date", "DATE", datetime.date(1899, 12, 31)), - ] - - # API request - query_job = client.query(sql, job_config=job_config) - - rows = list(query_job) - print("{} states were admitted to the US in the 1800s".format(len(rows))) - # [END bigquery_query_partitioned_table] - assert len(rows) == 29 - - -# [START bigquery_table_exists] def table_exists(client, table_reference): """Return if a table exists. @@ -452,2341 +129,5 @@ def table_exists(client, table_reference): return False -# [END bigquery_table_exists] - - -def test_table_exists(client, to_delete): - """Determine if a table exists.""" - DATASET_ID = "get_table_dataset_{}".format(_millis()) - TABLE_ID = "get_table_table_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(DATASET_ID)) - dataset = client.create_dataset(dataset) - to_delete.append(dataset) - - table_ref = dataset.table(TABLE_ID) - table = bigquery.Table(table_ref, schema=SCHEMA) - table = client.create_table(table) - - assert table_exists(client, table_ref) - assert not table_exists(client, dataset.table("i_dont_exist")) - - -@pytest.mark.skip( - reason=( - "update_table() is flaky " - "https://github.com/GoogleCloudPlatform/google-cloud-python/issues/5589" - ) -) -def test_manage_table_labels(client, to_delete): - dataset_id = "label_table_dataset_{}".format(_millis()) - table_id = "label_table_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - client.create_dataset(dataset) - to_delete.append(dataset) - - table = bigquery.Table(dataset.table(table_id), schema=SCHEMA) - table = client.create_table(table) - - # [START bigquery_label_table] - # from google.cloud import bigquery - # client = bigquery.Client() - # table_ref = client.dataset('my_dataset').table('my_table') - # table = client.get_table(table_ref) # API request - - assert table.labels == {} - labels = {"color": "green"} - table.labels = labels - - table = client.update_table(table, ["labels"]) # API request - - assert table.labels == labels - # [END bigquery_label_table] - - # [START bigquery_get_table_labels] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' - # table_id = 'my_table' - - dataset_ref = client.dataset(dataset_id) - table_ref = dataset_ref.table(table_id) - table = client.get_table(table_ref) # API Request - - # View table labels - print("Table ID: {}".format(table_id)) - print("Labels:") - if table.labels: - for label, value in table.labels.items(): - print("\t{}: {}".format(label, value)) - else: - print("\tTable has no labels defined.") - # [END bigquery_get_table_labels] - assert table.labels == labels - - # [START bigquery_delete_label_table] - # from google.cloud import bigquery - # client = bigquery.Client() - # table_ref = client.dataset('my_dataset').table('my_table') - # table = client.get_table(table_ref) # API request - - # This example table starts with one label - assert table.labels == {"color": "green"} - # To delete a label from a table, set its value to None - table.labels["color"] = None - - table = client.update_table(table, ["labels"]) # API request - - assert table.labels == {} - # [END bigquery_delete_label_table] - - -@pytest.mark.skip( - reason=( - "update_table() is flaky " - "https://github.com/GoogleCloudPlatform/google-cloud-python/issues/5589" - ) -) -def test_update_table_description(client, to_delete): - """Update a table's description.""" - dataset_id = "update_table_description_dataset_{}".format(_millis()) - table_id = "update_table_description_table_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - client.create_dataset(dataset) - to_delete.append(dataset) - - table = bigquery.Table(dataset.table(table_id), schema=SCHEMA) - table.description = "Original description." - table = client.create_table(table) - - # [START bigquery_update_table_description] - # from google.cloud import bigquery - # client = bigquery.Client() - # table_ref = client.dataset('my_dataset').table('my_table') - # table = client.get_table(table_ref) # API request - - assert table.description == "Original description." - table.description = "Updated description." - - table = client.update_table(table, ["description"]) # API request - - assert table.description == "Updated description." - # [END bigquery_update_table_description] - - -@pytest.mark.skip( - reason=( - "update_table() is flaky " - "https://github.com/GoogleCloudPlatform/google-cloud-python/issues/5589" - ) -) -def test_update_table_expiration(client, to_delete): - """Update a table's expiration time.""" - dataset_id = "update_table_expiration_dataset_{}".format(_millis()) - table_id = "update_table_expiration_table_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - client.create_dataset(dataset) - to_delete.append(dataset) - - table = bigquery.Table(dataset.table(table_id), schema=SCHEMA) - table = client.create_table(table) - - # [START bigquery_update_table_expiration] - import datetime - import pytz - - # from google.cloud import bigquery - # client = bigquery.Client() - # table_ref = client.dataset('my_dataset').table('my_table') - # table = client.get_table(table_ref) # API request - - assert table.expires is None - - # set table to expire 5 days from now - expiration = datetime.datetime.now(pytz.utc) + datetime.timedelta(days=5) - table.expires = expiration - table = client.update_table(table, ["expires"]) # API request - - # expiration is stored in milliseconds - margin = datetime.timedelta(microseconds=1000) - assert expiration - margin <= table.expires <= expiration + margin - # [END bigquery_update_table_expiration] - - -@pytest.mark.skip( - reason=( - "update_table() is flaky " - "https://github.com/GoogleCloudPlatform/google-cloud-python/issues/5589" - ) -) -def test_add_empty_column(client, to_delete): - """Adds an empty column to an existing table.""" - dataset_id = "add_empty_column_dataset_{}".format(_millis()) - table_id = "add_empty_column_table_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - dataset = client.create_dataset(dataset) - to_delete.append(dataset) - - table = bigquery.Table(dataset.table(table_id), schema=SCHEMA) - table = client.create_table(table) - - # [START bigquery_add_empty_column] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' - # table_id = 'my_table' - - table_ref = client.dataset(dataset_id).table(table_id) - table = client.get_table(table_ref) # API request - - original_schema = table.schema - new_schema = original_schema[:] # creates a copy of the schema - new_schema.append(bigquery.SchemaField("phone", "STRING")) - - table.schema = new_schema - table = client.update_table(table, ["schema"]) # API request - - assert len(table.schema) == len(original_schema) + 1 == len(new_schema) - # [END bigquery_add_empty_column] - - -@pytest.mark.skip( - reason=( - "update_table() is flaky " - "https://github.com/GoogleCloudPlatform/google-cloud-python/issues/5589" - ) -) -def test_relax_column(client, to_delete): - """Updates a schema field from required to nullable.""" - dataset_id = "relax_column_dataset_{}".format(_millis()) - table_id = "relax_column_table_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - dataset = client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_relax_column] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' - # table_id = 'my_table' - - original_schema = [ - bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), - ] - table_ref = client.dataset(dataset_id).table(table_id) - table = bigquery.Table(table_ref, schema=original_schema) - table = client.create_table(table) - assert all(field.mode == "REQUIRED" for field in table.schema) - - # SchemaField properties cannot be edited after initialization. - # To make changes, construct new SchemaField objects. - relaxed_schema = [ - bigquery.SchemaField("full_name", "STRING", mode="NULLABLE"), - bigquery.SchemaField("age", "INTEGER", mode="NULLABLE"), - ] - table.schema = relaxed_schema - table = client.update_table(table, ["schema"]) - - assert all(field.mode == "NULLABLE" for field in table.schema) - # [END bigquery_relax_column] - - -@pytest.mark.skip( - reason=( - "update_table() is flaky " - "https://github.com/GoogleCloudPlatform/google-cloud-python/issues/5589" - ) -) -def test_update_table_cmek(client, to_delete): - """Patch a table's metadata.""" - dataset_id = "update_table_cmek_{}".format(_millis()) - table_id = "update_table_cmek_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - client.create_dataset(dataset) - to_delete.append(dataset) - - table = bigquery.Table(dataset.table(table_id)) - original_kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format( - "cloud-samples-tests", "us-central1", "test", "test" - ) - table.encryption_configuration = bigquery.EncryptionConfiguration( - kms_key_name=original_kms_key_name - ) - table = client.create_table(table) - - # [START bigquery_update_table_cmek] - # from google.cloud import bigquery - # client = bigquery.Client() - - assert table.encryption_configuration.kms_key_name == original_kms_key_name - - # Set a new encryption key to use for the destination. - # TODO: Replace this key with a key you have created in KMS. - updated_kms_key_name = ( - "projects/cloud-samples-tests/locations/us-central1/" - "keyRings/test/cryptoKeys/otherkey" - ) - table.encryption_configuration = bigquery.EncryptionConfiguration( - kms_key_name=updated_kms_key_name - ) - - table = client.update_table(table, ["encryption_configuration"]) # API request - - assert table.encryption_configuration.kms_key_name == updated_kms_key_name - assert original_kms_key_name != updated_kms_key_name - # [END bigquery_update_table_cmek] - - -def test_browse_table_data(client, to_delete, capsys): - """Retreive selected row data from a table.""" - - # [START bigquery_browse_table] - # from google.cloud import bigquery - # client = bigquery.Client() - - dataset_ref = client.dataset("samples", project="bigquery-public-data") - table_ref = dataset_ref.table("shakespeare") - table = client.get_table(table_ref) # API call - - # Load all rows from a table - rows = client.list_rows(table) - assert len(list(rows)) == table.num_rows - - # Load the first 10 rows - rows = client.list_rows(table, max_results=10) - assert len(list(rows)) == 10 - - # Specify selected fields to limit the results to certain columns - fields = table.schema[:2] # first two columns - rows = client.list_rows(table, selected_fields=fields, max_results=10) - assert len(rows.schema) == 2 - assert len(list(rows)) == 10 - - # Use the start index to load an arbitrary portion of the table - rows = client.list_rows(table, start_index=10, max_results=10) - - # Print row data in tabular format - format_string = "{!s:<16} " * len(rows.schema) - field_names = [field.name for field in rows.schema] - print(format_string.format(*field_names)) # prints column headers - for row in rows: - print(format_string.format(*row)) # prints row data - # [END bigquery_browse_table] - - out, err = capsys.readouterr() - out = list(filter(bool, out.split("\n"))) # list of non-blank lines - assert len(out) == 11 - - -@pytest.mark.skip( - reason=( - "update_table() is flaky " - "https://github.com/GoogleCloudPlatform/google-cloud-python/issues/5589" - ) -) -def test_manage_views(client, to_delete): - project = client.project - source_dataset_id = "source_dataset_{}".format(_millis()) - source_dataset_ref = client.dataset(source_dataset_id) - source_dataset = bigquery.Dataset(source_dataset_ref) - source_dataset = client.create_dataset(source_dataset) - to_delete.append(source_dataset) - - job_config = bigquery.LoadJobConfig() - job_config.schema = [ - bigquery.SchemaField("name", "STRING"), - bigquery.SchemaField("post_abbr", "STRING"), - ] - job_config.skip_leading_rows = 1 - uri = "gs://cloud-samples-data/bigquery/us-states/us-states.csv" - source_table_id = "us_states" - load_job = client.load_table_from_uri( - uri, source_dataset.table(source_table_id), job_config=job_config - ) - load_job.result() - - shared_dataset_id = "shared_dataset_{}".format(_millis()) - shared_dataset_ref = client.dataset(shared_dataset_id) - shared_dataset = bigquery.Dataset(shared_dataset_ref) - shared_dataset = client.create_dataset(shared_dataset) - to_delete.append(shared_dataset) - - # [START bigquery_create_view] - # from google.cloud import bigquery - # client = bigquery.Client() - # project = 'my-project' - # source_dataset_id = 'my_source_dataset' - # source_table_id = 'us_states' - # shared_dataset_ref = client.dataset('my_shared_dataset') - - # This example shows how to create a shared view of a source table of - # US States. The source table contains all 50 states, while the view will - # contain only states with names starting with 'W'. - view_ref = shared_dataset_ref.table("my_shared_view") - view = bigquery.Table(view_ref) - sql_template = 'SELECT name, post_abbr FROM `{}.{}.{}` WHERE name LIKE "W%"' - view.view_query = sql_template.format(project, source_dataset_id, source_table_id) - view = client.create_table(view) # API request - - print("Successfully created view at {}".format(view.full_table_id)) - # [END bigquery_create_view] - - # [START bigquery_update_view_query] - # from google.cloud import bigquery - # client = bigquery.Client() - # project = 'my-project' - # source_dataset_id = 'my_source_dataset' - # source_table_id = 'us_states' - # shared_dataset_ref = client.dataset('my_shared_dataset') - - # This example shows how to update a shared view of a source table of - # US States. The view's query will be updated to contain only states with - # names starting with 'M'. - view_ref = shared_dataset_ref.table("my_shared_view") - view = bigquery.Table(view_ref) - sql_template = 'SELECT name, post_abbr FROM `{}.{}.{}` WHERE name LIKE "M%"' - view.view_query = sql_template.format(project, source_dataset_id, source_table_id) - view = client.update_table(view, ["view_query"]) # API request - # [END bigquery_update_view_query] - - # [START bigquery_get_view] - # from google.cloud import bigquery - # client = bigquery.Client() - # shared_dataset_id = 'my_shared_dataset' - - view_ref = client.dataset(shared_dataset_id).table("my_shared_view") - view = client.get_table(view_ref) # API Request - - # Display view properties - print("View at {}".format(view.full_table_id)) - print("View Query:\n{}".format(view.view_query)) - # [END bigquery_get_view] - assert view.view_query is not None - - analyst_group_email = "example-analyst-group@google.com" - # [START bigquery_grant_view_access] - # from google.cloud import bigquery - # client = bigquery.Client() - - # Assign access controls to the dataset containing the view - # shared_dataset_id = 'my_shared_dataset' - # analyst_group_email = 'data_analysts@example.com' - shared_dataset = client.get_dataset( - client.dataset(shared_dataset_id) - ) # API request - access_entries = shared_dataset.access_entries - access_entries.append( - bigquery.AccessEntry("READER", "groupByEmail", analyst_group_email) - ) - shared_dataset.access_entries = access_entries - shared_dataset = client.update_dataset( - shared_dataset, ["access_entries"] - ) # API request - - # Authorize the view to access the source dataset - # project = 'my-project' - # source_dataset_id = 'my_source_dataset' - source_dataset = client.get_dataset( - client.dataset(source_dataset_id) - ) # API request - view_reference = { - "projectId": project, - "datasetId": shared_dataset_id, - "tableId": "my_shared_view", - } - access_entries = source_dataset.access_entries - access_entries.append(bigquery.AccessEntry(None, "view", view_reference)) - source_dataset.access_entries = access_entries - source_dataset = client.update_dataset( - source_dataset, ["access_entries"] - ) # API request - # [END bigquery_grant_view_access] - - -def test_table_insert_rows(client, to_delete): - """Insert / fetch table data.""" - dataset_id = "table_insert_rows_dataset_{}".format(_millis()) - table_id = "table_insert_rows_table_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - dataset = client.create_dataset(dataset) - dataset.location = "US" - to_delete.append(dataset) - - table = bigquery.Table(dataset.table(table_id), schema=SCHEMA) - table = client.create_table(table) - - # [START bigquery_table_insert_rows] - # TODO(developer): Uncomment the lines below and replace with your values. - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' # replace with your dataset ID - # For this sample, the table must already exist and have a defined schema - # table_id = 'my_table' # replace with your table ID - # table_ref = client.dataset(dataset_id).table(table_id) - # table = client.get_table(table_ref) # API request - - rows_to_insert = [(u"Phred Phlyntstone", 32), (u"Wylma Phlyntstone", 29)] - - errors = client.insert_rows(table, rows_to_insert) # API request - - assert errors == [] - # [END bigquery_table_insert_rows] - - -def test_load_table_from_file(client, to_delete): - """Upload table data from a CSV file.""" - dataset_id = "load_table_from_file_dataset_{}".format(_millis()) - table_id = "load_table_from_file_table_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - dataset.location = "US" - client.create_dataset(dataset) - to_delete.append(dataset) - snippets_dir = os.path.abspath(os.path.dirname(__file__)) - filename = os.path.join( - snippets_dir, "..", "..", "bigquery", "tests", "data", "people.csv" - ) - - # [START bigquery_load_from_file] - # from google.cloud import bigquery - # client = bigquery.Client() - # filename = '/path/to/file.csv' - # dataset_id = 'my_dataset' - # table_id = 'my_table' - - dataset_ref = client.dataset(dataset_id) - table_ref = dataset_ref.table(table_id) - job_config = bigquery.LoadJobConfig() - job_config.source_format = bigquery.SourceFormat.CSV - job_config.skip_leading_rows = 1 - job_config.autodetect = True - - with open(filename, "rb") as source_file: - job = client.load_table_from_file(source_file, table_ref, job_config=job_config) - - job.result() # Waits for table load to complete. - - print("Loaded {} rows into {}:{}.".format(job.output_rows, dataset_id, table_id)) - # [END bigquery_load_from_file] - - table = client.get_table(table_ref) - rows = list(client.list_rows(table)) # API request - - assert len(rows) == 2 - # Order is not preserved, so compare individually - row1 = bigquery.Row(("Wylma Phlyntstone", 29), {"full_name": 0, "age": 1}) - assert row1 in rows - row2 = bigquery.Row(("Phred Phlyntstone", 32), {"full_name": 0, "age": 1}) - assert row2 in rows - - -def test_load_table_from_uri_avro(client, to_delete, capsys): - dataset_id = "load_table_from_uri_avro_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_load_table_gcs_avro] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' - - dataset_ref = client.dataset(dataset_id) - job_config = bigquery.LoadJobConfig() - job_config.source_format = bigquery.SourceFormat.AVRO - uri = "gs://cloud-samples-data/bigquery/us-states/us-states.avro" - - load_job = client.load_table_from_uri( - uri, dataset_ref.table("us_states"), job_config=job_config - ) # API request - print("Starting job {}".format(load_job.job_id)) - - load_job.result() # Waits for table load to complete. - print("Job finished.") - - destination_table = client.get_table(dataset_ref.table("us_states")) - print("Loaded {} rows.".format(destination_table.num_rows)) - # [END bigquery_load_table_gcs_avro] - - out, _ = capsys.readouterr() - assert "Loaded 50 rows." in out - - -def test_load_table_from_uri_csv(client, to_delete, capsys): - dataset_id = "load_table_from_uri_csv_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_load_table_gcs_csv] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' - - dataset_ref = client.dataset(dataset_id) - job_config = bigquery.LoadJobConfig() - job_config.schema = [ - bigquery.SchemaField("name", "STRING"), - bigquery.SchemaField("post_abbr", "STRING"), - ] - job_config.skip_leading_rows = 1 - # The source format defaults to CSV, so the line below is optional. - job_config.source_format = bigquery.SourceFormat.CSV - uri = "gs://cloud-samples-data/bigquery/us-states/us-states.csv" - - load_job = client.load_table_from_uri( - uri, dataset_ref.table("us_states"), job_config=job_config - ) # API request - print("Starting job {}".format(load_job.job_id)) - - load_job.result() # Waits for table load to complete. - print("Job finished.") - - destination_table = client.get_table(dataset_ref.table("us_states")) - print("Loaded {} rows.".format(destination_table.num_rows)) - # [END bigquery_load_table_gcs_csv] - - out, _ = capsys.readouterr() - assert "Loaded 50 rows." in out - - -def test_load_table_from_uri_json(client, to_delete, capsys): - dataset_id = "load_table_from_uri_json_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - dataset.location = "US" - client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_load_table_gcs_json] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' - - dataset_ref = client.dataset(dataset_id) - job_config = bigquery.LoadJobConfig() - job_config.schema = [ - bigquery.SchemaField("name", "STRING"), - bigquery.SchemaField("post_abbr", "STRING"), - ] - job_config.source_format = bigquery.SourceFormat.NEWLINE_DELIMITED_JSON - uri = "gs://cloud-samples-data/bigquery/us-states/us-states.json" - - load_job = client.load_table_from_uri( - uri, - dataset_ref.table("us_states"), - location="US", # Location must match that of the destination dataset. - job_config=job_config, - ) # API request - print("Starting job {}".format(load_job.job_id)) - - load_job.result() # Waits for table load to complete. - print("Job finished.") - - destination_table = client.get_table(dataset_ref.table("us_states")) - print("Loaded {} rows.".format(destination_table.num_rows)) - # [END bigquery_load_table_gcs_json] - - out, _ = capsys.readouterr() - assert "Loaded 50 rows." in out - - -def test_load_table_from_uri_cmek(client, to_delete): - dataset_id = "load_table_from_uri_cmek_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - dataset.location = "US" - client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_load_table_gcs_json_cmek] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' - - dataset_ref = client.dataset(dataset_id) - job_config = bigquery.LoadJobConfig() - job_config.autodetect = True - job_config.source_format = bigquery.SourceFormat.NEWLINE_DELIMITED_JSON - - # Set the encryption key to use for the destination. - # TODO: Replace this key with a key you have created in KMS. - kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format( - "cloud-samples-tests", "us-central1", "test", "test" - ) - encryption_config = bigquery.EncryptionConfiguration(kms_key_name=kms_key_name) - job_config.destination_encryption_configuration = encryption_config - uri = "gs://cloud-samples-data/bigquery/us-states/us-states.json" - - load_job = client.load_table_from_uri( - uri, - dataset_ref.table("us_states"), - location="US", # Location must match that of the destination dataset. - job_config=job_config, - ) # API request - - assert load_job.job_type == "load" - - load_job.result() # Waits for table load to complete. - - assert load_job.state == "DONE" - table = client.get_table(dataset_ref.table("us_states")) - assert table.encryption_configuration.kms_key_name == kms_key_name - # [END bigquery_load_table_gcs_json_cmek] - - -def test_load_table_from_uri_parquet(client, to_delete, capsys): - dataset_id = "load_table_from_uri_parquet_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_load_table_gcs_parquet] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' - - dataset_ref = client.dataset(dataset_id) - job_config = bigquery.LoadJobConfig() - job_config.source_format = bigquery.SourceFormat.PARQUET - uri = "gs://cloud-samples-data/bigquery/us-states/us-states.parquet" - - load_job = client.load_table_from_uri( - uri, dataset_ref.table("us_states"), job_config=job_config - ) # API request - print("Starting job {}".format(load_job.job_id)) - - load_job.result() # Waits for table load to complete. - print("Job finished.") - - destination_table = client.get_table(dataset_ref.table("us_states")) - print("Loaded {} rows.".format(destination_table.num_rows)) - # [END bigquery_load_table_gcs_parquet] - - out, _ = capsys.readouterr() - assert "Loaded 50 rows." in out - - -def test_load_table_from_uri_orc(client, to_delete, capsys): - dataset_id = "load_table_from_uri_orc_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_load_table_gcs_orc] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' - - dataset_ref = client.dataset(dataset_id) - job_config = bigquery.LoadJobConfig() - job_config.source_format = bigquery.SourceFormat.ORC - uri = "gs://cloud-samples-data/bigquery/us-states/us-states.orc" - - load_job = client.load_table_from_uri( - uri, dataset_ref.table("us_states"), job_config=job_config - ) # API request - print("Starting job {}".format(load_job.job_id)) - - load_job.result() # Waits for table load to complete. - print("Job finished.") - - destination_table = client.get_table(dataset_ref.table("us_states")) - print("Loaded {} rows.".format(destination_table.num_rows)) - # [END bigquery_load_table_gcs_orc] - - out, _ = capsys.readouterr() - assert "Loaded 50 rows." in out - - -def test_load_table_from_uri_autodetect(client, to_delete, capsys): - """Load table from a GCS URI using various formats and auto-detected schema - - Each file format has its own tested load from URI sample. Because most of - the code is common for autodetect, append, and truncate, this sample - includes snippets for all supported formats but only calls a single load - job. - - This code snippet is made up of shared code, then format-specific code, - followed by more shared code. Note that only the last format in the - format-specific code section will be tested in this test. - """ - dataset_id = "load_table_from_uri_auto_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - client.create_dataset(dataset) - to_delete.append(dataset) - - # Shared code - # [START bigquery_load_table_gcs_csv_autodetect] - # [START bigquery_load_table_gcs_json_autodetect] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' - - dataset_ref = client.dataset(dataset_id) - job_config = bigquery.LoadJobConfig() - job_config.autodetect = True - # [END bigquery_load_table_gcs_csv_autodetect] - # [END bigquery_load_table_gcs_json_autodetect] - - # Format-specific code - # [START bigquery_load_table_gcs_csv_autodetect] - job_config.skip_leading_rows = 1 - # The source format defaults to CSV, so the line below is optional. - job_config.source_format = bigquery.SourceFormat.CSV - uri = "gs://cloud-samples-data/bigquery/us-states/us-states.csv" - # [END bigquery_load_table_gcs_csv_autodetect] - # unset csv-specific attribute - del job_config._properties["load"]["skipLeadingRows"] - - # [START bigquery_load_table_gcs_json_autodetect] - job_config.source_format = bigquery.SourceFormat.NEWLINE_DELIMITED_JSON - uri = "gs://cloud-samples-data/bigquery/us-states/us-states.json" - # [END bigquery_load_table_gcs_json_autodetect] - - # Shared code - # [START bigquery_load_table_gcs_csv_autodetect] - # [START bigquery_load_table_gcs_json_autodetect] - load_job = client.load_table_from_uri( - uri, dataset_ref.table("us_states"), job_config=job_config - ) # API request - print("Starting job {}".format(load_job.job_id)) - - load_job.result() # Waits for table load to complete. - print("Job finished.") - - destination_table = client.get_table(dataset_ref.table("us_states")) - print("Loaded {} rows.".format(destination_table.num_rows)) - # [END bigquery_load_table_gcs_csv_autodetect] - # [END bigquery_load_table_gcs_json_autodetect] - - out, _ = capsys.readouterr() - assert "Loaded 50 rows." in out - - -def test_load_table_from_uri_truncate(client, to_delete, capsys): - """Replaces table data with data from a GCS URI using various formats - - Each file format has its own tested load from URI sample. Because most of - the code is common for autodetect, append, and truncate, this sample - includes snippets for all supported formats but only calls a single load - job. - - This code snippet is made up of shared code, then format-specific code, - followed by more shared code. Note that only the last format in the - format-specific code section will be tested in this test. - """ - dataset_id = "load_table_from_uri_trunc_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - client.create_dataset(dataset) - to_delete.append(dataset) - - job_config = bigquery.LoadJobConfig() - job_config.schema = [ - bigquery.SchemaField("name", "STRING"), - bigquery.SchemaField("post_abbr", "STRING"), - ] - table_ref = dataset.table("us_states") - body = six.BytesIO(b"Washington,WA") - client.load_table_from_file(body, table_ref, job_config=job_config).result() - previous_rows = client.get_table(table_ref).num_rows - assert previous_rows > 0 - - # Shared code - # [START bigquery_load_table_gcs_avro_truncate] - # [START bigquery_load_table_gcs_csv_truncate] - # [START bigquery_load_table_gcs_json_truncate] - # [START bigquery_load_table_gcs_parquet_truncate] - # [START bigquery_load_table_gcs_orc_truncate] - # from google.cloud import bigquery - # client = bigquery.Client() - # table_ref = client.dataset('my_dataset').table('existing_table') - - job_config = bigquery.LoadJobConfig() - job_config.write_disposition = bigquery.WriteDisposition.WRITE_TRUNCATE - # [END bigquery_load_table_gcs_avro_truncate] - # [END bigquery_load_table_gcs_csv_truncate] - # [END bigquery_load_table_gcs_json_truncate] - # [END bigquery_load_table_gcs_parquet_truncate] - # [END bigquery_load_table_gcs_orc_truncate] - - # Format-specific code - # [START bigquery_load_table_gcs_avro_truncate] - job_config.source_format = bigquery.SourceFormat.AVRO - uri = "gs://cloud-samples-data/bigquery/us-states/us-states.avro" - # [END bigquery_load_table_gcs_avro_truncate] - - # [START bigquery_load_table_gcs_csv_truncate] - job_config.skip_leading_rows = 1 - # The source format defaults to CSV, so the line below is optional. - job_config.source_format = bigquery.SourceFormat.CSV - uri = "gs://cloud-samples-data/bigquery/us-states/us-states.csv" - # [END bigquery_load_table_gcs_csv_truncate] - # unset csv-specific attribute - del job_config._properties["load"]["skipLeadingRows"] - - # [START bigquery_load_table_gcs_json_truncate] - job_config.source_format = bigquery.SourceFormat.NEWLINE_DELIMITED_JSON - uri = "gs://cloud-samples-data/bigquery/us-states/us-states.json" - # [END bigquery_load_table_gcs_json_truncate] - - # [START bigquery_load_table_gcs_parquet_truncate] - job_config.source_format = bigquery.SourceFormat.PARQUET - uri = "gs://cloud-samples-data/bigquery/us-states/us-states.parquet" - # [END bigquery_load_table_gcs_parquet_truncate] - - # [START bigquery_load_table_gcs_orc_truncate] - job_config.source_format = bigquery.SourceFormat.ORC - uri = "gs://cloud-samples-data/bigquery/us-states/us-states.orc" - # [END bigquery_load_table_gcs_orc_truncate] - - # Shared code - # [START bigquery_load_table_gcs_avro_truncate] - # [START bigquery_load_table_gcs_csv_truncate] - # [START bigquery_load_table_gcs_json_truncate] - # [START bigquery_load_table_gcs_parquet_truncate] - # [START bigquery_load_table_gcs_orc_truncate] - load_job = client.load_table_from_uri( - uri, table_ref, job_config=job_config - ) # API request - print("Starting job {}".format(load_job.job_id)) - - load_job.result() # Waits for table load to complete. - print("Job finished.") - - destination_table = client.get_table(table_ref) - print("Loaded {} rows.".format(destination_table.num_rows)) - # [END bigquery_load_table_gcs_avro_truncate] - # [END bigquery_load_table_gcs_csv_truncate] - # [END bigquery_load_table_gcs_json_truncate] - # [END bigquery_load_table_gcs_parquet_truncate] - # [END bigquery_load_table_gcs_orc_truncate] - - out, _ = capsys.readouterr() - assert "Loaded 50 rows." in out - - -def test_load_table_add_column(client, to_delete): - dataset_id = "load_table_add_column_{}".format(_millis()) - dataset_ref = client.dataset(dataset_id) - dataset = bigquery.Dataset(dataset_ref) - dataset.location = "US" - dataset = client.create_dataset(dataset) - to_delete.append(dataset) - - snippets_dir = os.path.abspath(os.path.dirname(__file__)) - filepath = os.path.join( - snippets_dir, "..", "..", "bigquery", "tests", "data", "people.csv" - ) - table_ref = dataset_ref.table("my_table") - old_schema = [bigquery.SchemaField("full_name", "STRING", mode="REQUIRED")] - table = client.create_table(bigquery.Table(table_ref, schema=old_schema)) - - # [START bigquery_add_column_load_append] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_ref = client.dataset('my_dataset') - # filepath = 'path/to/your_file.csv' - - # Retrieves the destination table and checks the length of the schema - table_id = "my_table" - table_ref = dataset_ref.table(table_id) - table = client.get_table(table_ref) - print("Table {} contains {} columns.".format(table_id, len(table.schema))) - - # Configures the load job to append the data to the destination table, - # allowing field addition - job_config = bigquery.LoadJobConfig() - job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND - job_config.schema_update_options = [ - bigquery.SchemaUpdateOption.ALLOW_FIELD_ADDITION - ] - # In this example, the existing table contains only the 'full_name' column. - # 'REQUIRED' fields cannot be added to an existing schema, so the - # additional column must be 'NULLABLE'. - job_config.schema = [ - bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("age", "INTEGER", mode="NULLABLE"), - ] - job_config.source_format = bigquery.SourceFormat.CSV - job_config.skip_leading_rows = 1 - - with open(filepath, "rb") as source_file: - job = client.load_table_from_file( - source_file, - table_ref, - location="US", # Must match the destination dataset location. - job_config=job_config, - ) # API request - - job.result() # Waits for table load to complete. - print( - "Loaded {} rows into {}:{}.".format( - job.output_rows, dataset_id, table_ref.table_id - ) - ) - - # Checks the updated length of the schema - table = client.get_table(table) - print("Table {} now contains {} columns.".format(table_id, len(table.schema))) - # [END bigquery_add_column_load_append] - assert len(table.schema) == 2 - assert table.num_rows > 0 - - -def test_load_table_relax_column(client, to_delete): - dataset_id = "load_table_relax_column_{}".format(_millis()) - dataset_ref = client.dataset(dataset_id) - dataset = bigquery.Dataset(dataset_ref) - dataset.location = "US" - dataset = client.create_dataset(dataset) - to_delete.append(dataset) - - snippets_dir = os.path.abspath(os.path.dirname(__file__)) - filepath = os.path.join( - snippets_dir, "..", "..", "bigquery", "tests", "data", "people.csv" - ) - table_ref = dataset_ref.table("my_table") - old_schema = [ - bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), - bigquery.SchemaField("favorite_color", "STRING", mode="REQUIRED"), - ] - table = client.create_table(bigquery.Table(table_ref, schema=old_schema)) - - # [START bigquery_relax_column_load_append] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_ref = client.dataset('my_dataset') - # filepath = 'path/to/your_file.csv' - - # Retrieves the destination table and checks the number of required fields - table_id = "my_table" - table_ref = dataset_ref.table(table_id) - table = client.get_table(table_ref) - original_required_fields = sum(field.mode == "REQUIRED" for field in table.schema) - # In this example, the existing table has 3 required fields. - print("{} fields in the schema are required.".format(original_required_fields)) - - # Configures the load job to append the data to a destination table, - # allowing field relaxation - job_config = bigquery.LoadJobConfig() - job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND - job_config.schema_update_options = [ - bigquery.SchemaUpdateOption.ALLOW_FIELD_RELAXATION - ] - # In this example, the existing table contains three required fields - # ('full_name', 'age', and 'favorite_color'), while the data to load - # contains only the first two fields. - job_config.schema = [ - bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), - ] - job_config.source_format = bigquery.SourceFormat.CSV - job_config.skip_leading_rows = 1 - - with open(filepath, "rb") as source_file: - job = client.load_table_from_file( - source_file, - table_ref, - location="US", # Must match the destination dataset location. - job_config=job_config, - ) # API request - - job.result() # Waits for table load to complete. - print( - "Loaded {} rows into {}:{}.".format( - job.output_rows, dataset_id, table_ref.table_id - ) - ) - - # Checks the updated number of required fields - table = client.get_table(table) - current_required_fields = sum(field.mode == "REQUIRED" for field in table.schema) - print("{} fields in the schema are now required.".format(current_required_fields)) - # [END bigquery_relax_column_load_append] - assert original_required_fields - current_required_fields == 1 - assert len(table.schema) == 3 - assert table.schema[2].mode == "NULLABLE" - assert table.num_rows > 0 - - -def test_copy_table(client, to_delete): - dataset_id = "copy_table_dataset_{}".format(_millis()) - dest_dataset = bigquery.Dataset(client.dataset(dataset_id)) - dest_dataset.location = "US" - dest_dataset = client.create_dataset(dest_dataset) - to_delete.append(dest_dataset) - - # [START bigquery_copy_table] - # from google.cloud import bigquery - # client = bigquery.Client() - - source_dataset = client.dataset("samples", project="bigquery-public-data") - source_table_ref = source_dataset.table("shakespeare") - - # dataset_id = 'my_dataset' - dest_table_ref = client.dataset(dataset_id).table("destination_table") - - job = client.copy_table( - source_table_ref, - dest_table_ref, - # Location must match that of the source and destination tables. - location="US", - ) # API request - - job.result() # Waits for job to complete. - - assert job.state == "DONE" - dest_table = client.get_table(dest_table_ref) # API request - assert dest_table.num_rows > 0 - # [END bigquery_copy_table] - - -def test_copy_table_multiple_source(client, to_delete): - dest_dataset_id = "dest_dataset_{}".format(_millis()) - dest_dataset = bigquery.Dataset(client.dataset(dest_dataset_id)) - dest_dataset.location = "US" - dest_dataset = client.create_dataset(dest_dataset) - to_delete.append(dest_dataset) - - source_dataset_id = "source_dataset_{}".format(_millis()) - source_dataset = bigquery.Dataset(client.dataset(source_dataset_id)) - source_dataset.location = "US" - source_dataset = client.create_dataset(source_dataset) - to_delete.append(source_dataset) - - schema = [ - bigquery.SchemaField("name", "STRING"), - bigquery.SchemaField("post_abbr", "STRING"), - ] - - table_data = {"table1": b"Washington,WA", "table2": b"California,CA"} - for table_id, data in table_data.items(): - table_ref = source_dataset.table(table_id) - job_config = bigquery.LoadJobConfig() - job_config.schema = schema - body = six.BytesIO(data) - client.load_table_from_file( - body, - table_ref, - # Location must match that of the destination dataset. - location="US", - job_config=job_config, - ).result() - - # [START bigquery_copy_table_multiple_source] - # from google.cloud import bigquery - # client = bigquery.Client() - # source_dataset_id = 'my_source_dataset' - # dest_dataset_id = 'my_destination_dataset' - - table1_ref = client.dataset(source_dataset_id).table("table1") - table2_ref = client.dataset(source_dataset_id).table("table2") - dest_table_ref = client.dataset(dest_dataset_id).table("destination_table") - - job = client.copy_table( - [table1_ref, table2_ref], - dest_table_ref, - # Location must match that of the source and destination tables. - location="US", - ) # API request - job.result() # Waits for job to complete. - - assert job.state == "DONE" - dest_table = client.get_table(dest_table_ref) # API request - assert dest_table.num_rows > 0 - # [END bigquery_copy_table_multiple_source] - - assert dest_table.num_rows == 2 - - -def test_copy_table_cmek(client, to_delete): - dataset_id = "copy_table_cmek_{}".format(_millis()) - dest_dataset = bigquery.Dataset(client.dataset(dataset_id)) - dest_dataset.location = "US" - dest_dataset = client.create_dataset(dest_dataset) - to_delete.append(dest_dataset) - - # [START bigquery_copy_table_cmek] - # from google.cloud import bigquery - # client = bigquery.Client() - - source_dataset = bigquery.DatasetReference("bigquery-public-data", "samples") - source_table_ref = source_dataset.table("shakespeare") - - # dataset_id = 'my_dataset' - dest_dataset_ref = client.dataset(dataset_id) - dest_table_ref = dest_dataset_ref.table("destination_table") - - # Set the encryption key to use for the destination. - # TODO: Replace this key with a key you have created in KMS. - kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format( - "cloud-samples-tests", "us-central1", "test", "test" - ) - encryption_config = bigquery.EncryptionConfiguration(kms_key_name=kms_key_name) - job_config = bigquery.CopyJobConfig() - job_config.destination_encryption_configuration = encryption_config - - job = client.copy_table( - source_table_ref, - dest_table_ref, - # Location must match that of the source and destination tables. - location="US", - job_config=job_config, - ) # API request - job.result() # Waits for job to complete. - - assert job.state == "DONE" - dest_table = client.get_table(dest_table_ref) - assert dest_table.encryption_configuration.kms_key_name == kms_key_name - # [END bigquery_copy_table_cmek] - - -def test_extract_table(client, to_delete): - bucket_name = "extract_shakespeare_{}".format(_millis()) - storage_client = storage.Client() - bucket = retry_storage_errors(storage_client.create_bucket)(bucket_name) - to_delete.append(bucket) - - # [START bigquery_extract_table] - # from google.cloud import bigquery - # client = bigquery.Client() - # bucket_name = 'my-bucket' - project = "bigquery-public-data" - dataset_id = "samples" - table_id = "shakespeare" - - destination_uri = "gs://{}/{}".format(bucket_name, "shakespeare.csv") - dataset_ref = client.dataset(dataset_id, project=project) - table_ref = dataset_ref.table(table_id) - - extract_job = client.extract_table( - table_ref, - destination_uri, - # Location must match that of the source table. - location="US", - ) # API request - extract_job.result() # Waits for job to complete. - - print( - "Exported {}:{}.{} to {}".format(project, dataset_id, table_id, destination_uri) - ) - # [END bigquery_extract_table] - - blob = retry_storage_errors(bucket.get_blob)("shakespeare.csv") - assert blob.exists - assert blob.size > 0 - to_delete.insert(0, blob) - - -def test_extract_table_json(client, to_delete): - bucket_name = "extract_shakespeare_json_{}".format(_millis()) - storage_client = storage.Client() - bucket = retry_storage_errors(storage_client.create_bucket)(bucket_name) - to_delete.append(bucket) - - # [START bigquery_extract_table_json] - # from google.cloud import bigquery - # client = bigquery.Client() - # bucket_name = 'my-bucket' - - destination_uri = "gs://{}/{}".format(bucket_name, "shakespeare.json") - dataset_ref = client.dataset("samples", project="bigquery-public-data") - table_ref = dataset_ref.table("shakespeare") - job_config = bigquery.job.ExtractJobConfig() - job_config.destination_format = bigquery.DestinationFormat.NEWLINE_DELIMITED_JSON - - extract_job = client.extract_table( - table_ref, - destination_uri, - job_config=job_config, - # Location must match that of the source table. - location="US", - ) # API request - extract_job.result() # Waits for job to complete. - # [END bigquery_extract_table_json] - - blob = retry_storage_errors(bucket.get_blob)("shakespeare.json") - assert blob.exists - assert blob.size > 0 - to_delete.insert(0, blob) - - -def test_extract_table_compressed(client, to_delete): - bucket_name = "extract_shakespeare_compress_{}".format(_millis()) - storage_client = storage.Client() - bucket = retry_storage_errors(storage_client.create_bucket)(bucket_name) - to_delete.append(bucket) - - # [START bigquery_extract_table_compressed] - # from google.cloud import bigquery - # client = bigquery.Client() - # bucket_name = 'my-bucket' - - destination_uri = "gs://{}/{}".format(bucket_name, "shakespeare.csv.gz") - dataset_ref = client.dataset("samples", project="bigquery-public-data") - table_ref = dataset_ref.table("shakespeare") - job_config = bigquery.job.ExtractJobConfig() - job_config.compression = bigquery.Compression.GZIP - - extract_job = client.extract_table( - table_ref, - destination_uri, - # Location must match that of the source table. - location="US", - job_config=job_config, - ) # API request - extract_job.result() # Waits for job to complete. - # [END bigquery_extract_table_compressed] - - blob = retry_storage_errors(bucket.get_blob)("shakespeare.csv.gz") - assert blob.exists - assert blob.size > 0 - to_delete.insert(0, blob) - - -def test_undelete_table(client, to_delete): - dataset_id = "undelete_table_dataset_{}".format(_millis()) - table_id = "undelete_table_table_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - dataset.location = "US" - dataset = client.create_dataset(dataset) - to_delete.append(dataset) - - table = bigquery.Table(dataset.table(table_id), schema=SCHEMA) - client.create_table(table) - - # [START bigquery_undelete_table] - # TODO(developer): Uncomment the lines below and replace with your values. - # import time - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' # Replace with your dataset ID. - # table_id = 'my_table' # Replace with your table ID. - - table_ref = client.dataset(dataset_id).table(table_id) - - # TODO(developer): Choose an appropriate snapshot point as epoch - # milliseconds. For this example, we choose the current time as we're about - # to delete the table immediately afterwards. - snapshot_epoch = int(time.time() * 1000) - # [END bigquery_undelete_table] - - # Due to very short lifecycle of the table, ensure we're not picking a time - # prior to the table creation due to time drift between backend and client. - table = client.get_table(table_ref) - created_epoch = datetime_helpers.to_microseconds(table.created) - if created_epoch > snapshot_epoch: - snapshot_epoch = created_epoch - - # [START bigquery_undelete_table] - - # "Accidentally" delete the table. - client.delete_table(table_ref) # API request - - # Construct the restore-from table ID using a snapshot decorator. - snapshot_table_id = "{}@{}".format(table_id, snapshot_epoch) - source_table_ref = client.dataset(dataset_id).table(snapshot_table_id) - - # Choose a new table ID for the recovered table data. - recovered_table_id = "{}_recovered".format(table_id) - dest_table_ref = client.dataset(dataset_id).table(recovered_table_id) - - # Construct and run a copy job. - job = client.copy_table( - source_table_ref, - dest_table_ref, - # Location must match that of the source and destination tables. - location="US", - ) # API request - - job.result() # Waits for job to complete. - - print( - "Copied data from deleted table {} to {}".format(table_id, recovered_table_id) - ) - # [END bigquery_undelete_table] - - -def test_client_query(client): - """Run a simple query.""" - - # [START bigquery_query] - # from google.cloud import bigquery - # client = bigquery.Client() - - query = ( - "SELECT name FROM `bigquery-public-data.usa_names.usa_1910_2013` " - 'WHERE state = "TX" ' - "LIMIT 100" - ) - query_job = client.query( - query, - # Location must match that of the dataset(s) referenced in the query. - location="US", - ) # API request - starts the query - - for row in query_job: # API request - fetches results - # Row values can be accessed by field name or index - assert row[0] == row.name == row["name"] - print(row) - # [END bigquery_query] - - -def test_client_query_legacy_sql(client): - """Run a query with Legacy SQL explicitly set""" - # [START bigquery_query_legacy] - # from google.cloud import bigquery - # client = bigquery.Client() - - query = ( - "SELECT name FROM [bigquery-public-data:usa_names.usa_1910_2013] " - 'WHERE state = "TX" ' - "LIMIT 100" - ) - - # Set use_legacy_sql to True to use legacy SQL syntax. - job_config = bigquery.QueryJobConfig() - job_config.use_legacy_sql = True - - query_job = client.query( - query, - # Location must match that of the dataset(s) referenced in the query. - location="US", - job_config=job_config, - ) # API request - starts the query - - # Print the results. - for row in query_job: # API request - fetches results - print(row) - # [END bigquery_query_legacy] - - -def test_client_query_total_rows(client, capsys): - """Run a query and just check for how many rows.""" - # [START bigquery_query_total_rows] - # from google.cloud import bigquery - # client = bigquery.Client() - - query = ( - "SELECT name FROM `bigquery-public-data.usa_names.usa_1910_2013` " - 'WHERE state = "TX" ' - "LIMIT 100" - ) - query_job = client.query( - query, - # Location must match that of the dataset(s) referenced in the query. - location="US", - ) # API request - starts the query - - results = query_job.result() # Wait for query to complete. - print("Got {} rows.".format(results.total_rows)) - # [END bigquery_query_total_rows] - - out, _ = capsys.readouterr() - assert "Got 100 rows." in out - - -def test_manage_job(client): - sql = """ - SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus; - """ - location = "us" - job = client.query(sql, location=location) - job_id = job.job_id - - # [START bigquery_cancel_job] - # TODO(developer): Uncomment the lines below and replace with your values. - # from google.cloud import bigquery - # client = bigquery.Client() - # job_id = 'bq-job-123x456-123y123z123c' # replace with your job ID - # location = 'us' # replace with your location - - job = client.cancel_job(job_id, location=location) - # [END bigquery_cancel_job] - - # [START bigquery_get_job] - # TODO(developer): Uncomment the lines below and replace with your values. - # from google.cloud import bigquery - # client = bigquery.Client() - # job_id = 'bq-job-123x456-123y123z123c' # replace with your job ID - # location = 'us' # replace with your location - - job = client.get_job(job_id, location=location) # API request - - # Print selected job properties - print("Details for job {} running in {}:".format(job_id, location)) - print( - "\tType: {}\n\tState: {}\n\tCreated: {}".format( - job.job_type, job.state, job.created - ) - ) - # [END bigquery_get_job] - - -def test_client_query_destination_table(client, to_delete): - """Run a query""" - dataset_id = "query_destination_table_{}".format(_millis()) - dataset_ref = client.dataset(dataset_id) - to_delete.append(dataset_ref) - dataset = bigquery.Dataset(dataset_ref) - dataset.location = "US" - client.create_dataset(dataset) - - # [START bigquery_query_destination_table] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'your_dataset_id' - - job_config = bigquery.QueryJobConfig() - # Set the destination table - table_ref = client.dataset(dataset_id).table("your_table_id") - job_config.destination = table_ref - sql = """ - SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus; - """ - - # Start the query, passing in the extra configuration. - query_job = client.query( - sql, - # Location must match that of the dataset(s) referenced in the query - # and of the destination table. - location="US", - job_config=job_config, - ) # API request - starts the query - - query_job.result() # Waits for the query to finish - print("Query results loaded to table {}".format(table_ref.path)) - # [END bigquery_query_destination_table] - - -def test_client_query_destination_table_legacy(client, to_delete): - dataset_id = "query_destination_table_legacy_{}".format(_millis()) - dataset_ref = client.dataset(dataset_id) - to_delete.append(dataset_ref) - dataset = bigquery.Dataset(dataset_ref) - dataset.location = "US" - client.create_dataset(dataset) - - # [START bigquery_query_legacy_large_results] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'your_dataset_id' - - job_config = bigquery.QueryJobConfig() - # Set use_legacy_sql to True to use legacy SQL syntax. - job_config.use_legacy_sql = True - # Set the destination table - table_ref = client.dataset(dataset_id).table("your_table_id") - job_config.destination = table_ref - job_config.allow_large_results = True - sql = """ - SELECT corpus - FROM [bigquery-public-data:samples.shakespeare] - GROUP BY corpus; - """ - # Start the query, passing in the extra configuration. - query_job = client.query( - sql, - # Location must match that of the dataset(s) referenced in the query - # and of the destination table. - location="US", - job_config=job_config, - ) # API request - starts the query - - query_job.result() # Waits for the query to finish - print("Query results loaded to table {}".format(table_ref.path)) - # [END bigquery_query_legacy_large_results] - - -def test_client_query_destination_table_cmek(client, to_delete): - """Run a query""" - dataset_id = "query_destination_table_{}".format(_millis()) - dataset_ref = client.dataset(dataset_id) - to_delete.append(dataset_ref) - dataset = bigquery.Dataset(dataset_ref) - dataset.location = "US" - client.create_dataset(dataset) - - # [START bigquery_query_destination_table_cmek] - # from google.cloud import bigquery - # client = bigquery.Client() - - job_config = bigquery.QueryJobConfig() - - # Set the destination table. Here, dataset_id is a string, such as: - # dataset_id = 'your_dataset_id' - table_ref = client.dataset(dataset_id).table("your_table_id") - job_config.destination = table_ref - - # Set the encryption key to use for the destination. - # TODO: Replace this key with a key you have created in KMS. - kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format( - "cloud-samples-tests", "us-central1", "test", "test" - ) - encryption_config = bigquery.EncryptionConfiguration(kms_key_name=kms_key_name) - job_config.destination_encryption_configuration = encryption_config - - # Start the query, passing in the extra configuration. - query_job = client.query( - "SELECT 17 AS my_col;", - # Location must match that of the dataset(s) referenced in the query - # and of the destination table. - location="US", - job_config=job_config, - ) # API request - starts the query - query_job.result() - - # The destination table is written using the encryption configuration. - table = client.get_table(table_ref) - assert table.encryption_configuration.kms_key_name == kms_key_name - # [END bigquery_query_destination_table_cmek] - - -def test_client_query_batch(client, to_delete): - # [START bigquery_query_batch] - # from google.cloud import bigquery - # client = bigquery.Client() - - job_config = bigquery.QueryJobConfig() - # Run at batch priority, which won't count toward concurrent rate limit. - job_config.priority = bigquery.QueryPriority.BATCH - sql = """ - SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus; - """ - # Location must match that of the dataset(s) referenced in the query. - location = "US" - - # API request - starts the query - query_job = client.query(sql, location=location, job_config=job_config) - - # Check on the progress by getting the job's updated state. Once the state - # is `DONE`, the results are ready. - query_job = client.get_job( - query_job.job_id, location=location - ) # API request - fetches job - print("Job {} is currently in state {}".format(query_job.job_id, query_job.state)) - # [END bigquery_query_batch] - - -def test_client_query_relax_column(client, to_delete): - dataset_id = "query_relax_column_{}".format(_millis()) - dataset_ref = client.dataset(dataset_id) - dataset = bigquery.Dataset(dataset_ref) - dataset.location = "US" - dataset = client.create_dataset(dataset) - to_delete.append(dataset) - - table_ref = dataset_ref.table("my_table") - schema = [ - bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), - ] - table = client.create_table(bigquery.Table(table_ref, schema=schema)) - - # [START bigquery_relax_column_query_append] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_ref = client.dataset('my_dataset') - - # Retrieves the destination table and checks the number of required fields - table_id = "my_table" - table_ref = dataset_ref.table(table_id) - table = client.get_table(table_ref) - original_required_fields = sum(field.mode == "REQUIRED" for field in table.schema) - # In this example, the existing table has 2 required fields - print("{} fields in the schema are required.".format(original_required_fields)) - - # Configures the query to append the results to a destination table, - # allowing field relaxation - job_config = bigquery.QueryJobConfig() - job_config.schema_update_options = [ - bigquery.SchemaUpdateOption.ALLOW_FIELD_RELAXATION - ] - job_config.destination = table_ref - job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND - - query_job = client.query( - # In this example, the existing table contains 'full_name' and 'age' as - # required columns, but the query results will omit the second column. - 'SELECT "Beyonce" as full_name;', - # Location must match that of the dataset(s) referenced in the query - # and of the destination table. - location="US", - job_config=job_config, - ) # API request - starts the query - - query_job.result() # Waits for the query to finish - print("Query job {} complete.".format(query_job.job_id)) - - # Checks the updated number of required fields - table = client.get_table(table) - current_required_fields = sum(field.mode == "REQUIRED" for field in table.schema) - print("{} fields in the schema are now required.".format(current_required_fields)) - # [END bigquery_relax_column_query_append] - assert original_required_fields - current_required_fields > 0 - assert len(table.schema) == 2 - assert table.schema[1].mode == "NULLABLE" - assert table.num_rows > 0 - - -def test_client_query_add_column(client, to_delete): - dataset_id = "query_add_column_{}".format(_millis()) - dataset_ref = client.dataset(dataset_id) - dataset = bigquery.Dataset(dataset_ref) - dataset.location = "US" - dataset = client.create_dataset(dataset) - to_delete.append(dataset) - - table_ref = dataset_ref.table("my_table") - schema = [ - bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), - ] - table = client.create_table(bigquery.Table(table_ref, schema=schema)) - - # [START bigquery_add_column_query_append] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_ref = client.dataset('my_dataset') - - # Retrieves the destination table and checks the length of the schema - table_id = "my_table" - table_ref = dataset_ref.table(table_id) - table = client.get_table(table_ref) - print("Table {} contains {} columns.".format(table_id, len(table.schema))) - - # Configures the query to append the results to a destination table, - # allowing field addition - job_config = bigquery.QueryJobConfig() - job_config.schema_update_options = [ - bigquery.SchemaUpdateOption.ALLOW_FIELD_ADDITION - ] - job_config.destination = table_ref - job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND - - query_job = client.query( - # In this example, the existing table contains only the 'full_name' and - # 'age' columns, while the results of this query will contain an - # additional 'favorite_color' column. - 'SELECT "Timmy" as full_name, 85 as age, "Blue" as favorite_color;', - # Location must match that of the dataset(s) referenced in the query - # and of the destination table. - location="US", - job_config=job_config, - ) # API request - starts the query - - query_job.result() # Waits for the query to finish - print("Query job {} complete.".format(query_job.job_id)) - - # Checks the updated length of the schema - table = client.get_table(table) - print("Table {} now contains {} columns.".format(table_id, len(table.schema))) - # [END bigquery_add_column_query_append] - assert len(table.schema) == 3 - assert table.num_rows > 0 - - -def test_client_query_w_named_params(client, capsys): - """Run a query using named query parameters""" - - # [START bigquery_query_params_named] - # from google.cloud import bigquery - # client = bigquery.Client() - - query = """ - SELECT word, word_count - FROM `bigquery-public-data.samples.shakespeare` - WHERE corpus = @corpus - AND word_count >= @min_word_count - ORDER BY word_count DESC; - """ - query_params = [ - bigquery.ScalarQueryParameter("corpus", "STRING", "romeoandjuliet"), - bigquery.ScalarQueryParameter("min_word_count", "INT64", 250), - ] - job_config = bigquery.QueryJobConfig() - job_config.query_parameters = query_params - query_job = client.query( - query, - # Location must match that of the dataset(s) referenced in the query. - location="US", - job_config=job_config, - ) # API request - starts the query - - # Print the results - for row in query_job: - print("{}: \t{}".format(row.word, row.word_count)) - - assert query_job.state == "DONE" - # [END bigquery_query_params_named] - - out, _ = capsys.readouterr() - assert "the" in out - - -def test_client_query_w_positional_params(client, capsys): - """Run a query using query parameters""" - - # [START bigquery_query_params_positional] - # from google.cloud import bigquery - # client = bigquery.Client() - - query = """ - SELECT word, word_count - FROM `bigquery-public-data.samples.shakespeare` - WHERE corpus = ? - AND word_count >= ? - ORDER BY word_count DESC; - """ - # Set the name to None to use positional parameters. - # Note that you cannot mix named and positional parameters. - query_params = [ - bigquery.ScalarQueryParameter(None, "STRING", "romeoandjuliet"), - bigquery.ScalarQueryParameter(None, "INT64", 250), - ] - job_config = bigquery.QueryJobConfig() - job_config.query_parameters = query_params - query_job = client.query( - query, - # Location must match that of the dataset(s) referenced in the query. - location="US", - job_config=job_config, - ) # API request - starts the query - - # Print the results - for row in query_job: - print("{}: \t{}".format(row.word, row.word_count)) - - assert query_job.state == "DONE" - # [END bigquery_query_params_positional] - - out, _ = capsys.readouterr() - assert "the" in out - - -def test_client_query_w_timestamp_params(client, capsys): - """Run a query using query parameters""" - - # [START bigquery_query_params_timestamps] - # from google.cloud import bigquery - # client = bigquery.Client() - - import datetime - import pytz - - query = "SELECT TIMESTAMP_ADD(@ts_value, INTERVAL 1 HOUR);" - query_params = [ - bigquery.ScalarQueryParameter( - "ts_value", - "TIMESTAMP", - datetime.datetime(2016, 12, 7, 8, 0, tzinfo=pytz.UTC), - ) - ] - job_config = bigquery.QueryJobConfig() - job_config.query_parameters = query_params - query_job = client.query( - query, - # Location must match that of the dataset(s) referenced in the query. - location="US", - job_config=job_config, - ) # API request - starts the query - - # Print the results - for row in query_job: - print(row) - - assert query_job.state == "DONE" - # [END bigquery_query_params_timestamps] - - out, _ = capsys.readouterr() - assert "2016, 12, 7, 9, 0" in out - - -def test_client_query_w_array_params(client, capsys): - """Run a query using array query parameters""" - # [START bigquery_query_params_arrays] - # from google.cloud import bigquery - # client = bigquery.Client() - - query = """ - SELECT name, sum(number) as count - FROM `bigquery-public-data.usa_names.usa_1910_2013` - WHERE gender = @gender - AND state IN UNNEST(@states) - GROUP BY name - ORDER BY count DESC - LIMIT 10; - """ - query_params = [ - bigquery.ScalarQueryParameter("gender", "STRING", "M"), - bigquery.ArrayQueryParameter("states", "STRING", ["WA", "WI", "WV", "WY"]), - ] - job_config = bigquery.QueryJobConfig() - job_config.query_parameters = query_params - query_job = client.query( - query, - # Location must match that of the dataset(s) referenced in the query. - location="US", - job_config=job_config, - ) # API request - starts the query - - # Print the results - for row in query_job: - print("{}: \t{}".format(row.name, row.count)) - - assert query_job.state == "DONE" - # [END bigquery_query_params_arrays] - - out, _ = capsys.readouterr() - assert "James" in out - - -def test_client_query_w_struct_params(client, capsys): - """Run a query using struct query parameters""" - # [START bigquery_query_params_structs] - # from google.cloud import bigquery - # client = bigquery.Client() - - query = "SELECT @struct_value AS s;" - query_params = [ - bigquery.StructQueryParameter( - "struct_value", - bigquery.ScalarQueryParameter("x", "INT64", 1), - bigquery.ScalarQueryParameter("y", "STRING", "foo"), - ) - ] - job_config = bigquery.QueryJobConfig() - job_config.query_parameters = query_params - query_job = client.query( - query, - # Location must match that of the dataset(s) referenced in the query. - location="US", - job_config=job_config, - ) # API request - starts the query - - # Print the results - for row in query_job: - print(row.s) - - assert query_job.state == "DONE" - # [END bigquery_query_params_structs] - - out, _ = capsys.readouterr() - assert "1" in out - assert "foo" in out - - -def test_client_query_dry_run(client): - """Run a dry run query""" - - # [START bigquery_query_dry_run] - # from google.cloud import bigquery - # client = bigquery.Client() - - job_config = bigquery.QueryJobConfig() - job_config.dry_run = True - job_config.use_query_cache = False - query_job = client.query( - ( - "SELECT name, COUNT(*) as name_count " - "FROM `bigquery-public-data.usa_names.usa_1910_2013` " - "WHERE state = 'WA' " - "GROUP BY name" - ), - # Location must match that of the dataset(s) referenced in the query. - location="US", - job_config=job_config, - ) # API request - - # A dry run query completes immediately. - assert query_job.state == "DONE" - assert query_job.dry_run - - print("This query will process {} bytes.".format(query_job.total_bytes_processed)) - # [END bigquery_query_dry_run] - - assert query_job.total_bytes_processed > 0 - - -def test_query_no_cache(client): - # [START bigquery_query_no_cache] - # from google.cloud import bigquery - # client = bigquery.Client() - - job_config = bigquery.QueryJobConfig() - job_config.use_query_cache = False - sql = """ - SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus; - """ - query_job = client.query( - sql, - # Location must match that of the dataset(s) referenced in the query. - location="US", - job_config=job_config, - ) # API request - - # Print the results. - for row in query_job: # API request - fetches results - print(row) - # [END bigquery_query_no_cache] - - -def test_query_external_gcs_temporary_table(client): - # [START bigquery_query_external_gcs_temp] - # from google.cloud import bigquery - # client = bigquery.Client() - - # Configure the external data source and query job - external_config = bigquery.ExternalConfig("CSV") - external_config.source_uris = [ - "gs://cloud-samples-data/bigquery/us-states/us-states.csv" - ] - external_config.schema = [ - bigquery.SchemaField("name", "STRING"), - bigquery.SchemaField("post_abbr", "STRING"), - ] - external_config.options.skip_leading_rows = 1 # optionally skip header row - table_id = "us_states" - job_config = bigquery.QueryJobConfig() - job_config.table_definitions = {table_id: external_config} - - # Example query to find states starting with 'W' - sql = 'SELECT * FROM `{}` WHERE name LIKE "W%"'.format(table_id) - - query_job = client.query(sql, job_config=job_config) # API request - - w_states = list(query_job) # Waits for query to finish - print("There are {} states with names starting with W.".format(len(w_states))) - # [END bigquery_query_external_gcs_temp] - assert len(w_states) == 4 - - -def test_query_external_gcs_permanent_table(client, to_delete): - dataset_id = "query_external_gcs_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_query_external_gcs_perm] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' - - # Configure the external data source - dataset_ref = client.dataset(dataset_id) - table_id = "us_states" - schema = [ - bigquery.SchemaField("name", "STRING"), - bigquery.SchemaField("post_abbr", "STRING"), - ] - table = bigquery.Table(dataset_ref.table(table_id), schema=schema) - external_config = bigquery.ExternalConfig("CSV") - external_config.source_uris = [ - "gs://cloud-samples-data/bigquery/us-states/us-states.csv" - ] - external_config.options.skip_leading_rows = 1 # optionally skip header row - table.external_data_configuration = external_config - - # Create a permanent table linked to the GCS file - table = client.create_table(table) # API request - - # Example query to find states starting with 'W' - sql = 'SELECT * FROM `{}.{}` WHERE name LIKE "W%"'.format(dataset_id, table_id) - - query_job = client.query(sql) # API request - - w_states = list(query_job) # Waits for query to finish - print("There are {} states with names starting with W.".format(len(w_states))) - # [END bigquery_query_external_gcs_perm] - assert len(w_states) == 4 - - -def test_query_external_sheets_temporary_table(client): - # [START bigquery_query_external_sheets_temp] - # [START bigquery_auth_drive_scope] - import google.auth - - # from google.cloud import bigquery - - # Create credentials with Drive & BigQuery API scopes - # Both APIs must be enabled for your project before running this code - credentials, project = google.auth.default( - scopes=[ - "https://www.googleapis.com/auth/drive", - "https://www.googleapis.com/auth/bigquery", - ] - ) - client = bigquery.Client(credentials=credentials, project=project) - # [END bigquery_auth_drive_scope] - - # Configure the external data source and query job - external_config = bigquery.ExternalConfig("GOOGLE_SHEETS") - # Use a shareable link or grant viewing access to the email address you - # used to authenticate with BigQuery (this example Sheet is public) - sheet_url = ( - "https://docs.google.com/spreadsheets" - "/d/1i_QCL-7HcSyUZmIbP9E6lO_T5u3HnpLe7dnpHaijg_E/edit?usp=sharing" - ) - external_config.source_uris = [sheet_url] - external_config.schema = [ - bigquery.SchemaField("name", "STRING"), - bigquery.SchemaField("post_abbr", "STRING"), - ] - external_config.options.skip_leading_rows = 1 # optionally skip header row - table_id = "us_states" - job_config = bigquery.QueryJobConfig() - job_config.table_definitions = {table_id: external_config} - - # Example query to find states starting with 'W' - sql = 'SELECT * FROM `{}` WHERE name LIKE "W%"'.format(table_id) - - query_job = client.query(sql, job_config=job_config) # API request - - w_states = list(query_job) # Waits for query to finish - print("There are {} states with names starting with W.".format(len(w_states))) - # [END bigquery_query_external_sheets_temp] - assert len(w_states) == 4 - - -def test_query_external_sheets_permanent_table(client, to_delete): - dataset_id = "query_external_sheets_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_query_external_sheets_perm] - import google.auth - - # from google.cloud import bigquery - # dataset_id = 'my_dataset' - - # Create credentials with Drive & BigQuery API scopes - # Both APIs must be enabled for your project before running this code - credentials, project = google.auth.default( - scopes=[ - "https://www.googleapis.com/auth/drive", - "https://www.googleapis.com/auth/bigquery", - ] - ) - client = bigquery.Client(credentials=credentials, project=project) - - # Configure the external data source - dataset_ref = client.dataset(dataset_id) - table_id = "us_states" - schema = [ - bigquery.SchemaField("name", "STRING"), - bigquery.SchemaField("post_abbr", "STRING"), - ] - table = bigquery.Table(dataset_ref.table(table_id), schema=schema) - external_config = bigquery.ExternalConfig("GOOGLE_SHEETS") - # Use a shareable link or grant viewing access to the email address you - # used to authenticate with BigQuery (this example Sheet is public) - sheet_url = ( - "https://docs.google.com/spreadsheets" - "/d/1i_QCL-7HcSyUZmIbP9E6lO_T5u3HnpLe7dnpHaijg_E/edit?usp=sharing" - ) - external_config.source_uris = [sheet_url] - external_config.options.skip_leading_rows = 1 # optionally skip header row - table.external_data_configuration = external_config - - # Create a permanent table linked to the Sheets file - table = client.create_table(table) # API request - - # Example query to find states starting with 'W' - sql = 'SELECT * FROM `{}.{}` WHERE name LIKE "W%"'.format(dataset_id, table_id) - - query_job = client.query(sql) # API request - - w_states = list(query_job) # Waits for query to finish - print("There are {} states with names starting with W.".format(len(w_states))) - # [END bigquery_query_external_sheets_perm] - assert len(w_states) == 4 - - -def test_ddl_create_view(client, to_delete, capsys): - """Create a view via a DDL query.""" - project = client.project - dataset_id = "ddl_view_{}".format(_millis()) - table_id = "new_view" - dataset = bigquery.Dataset(client.dataset(dataset_id)) - client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_ddl_create_view] - # from google.cloud import bigquery - # project = 'my-project' - # dataset_id = 'my_dataset' - # table_id = 'new_view' - # client = bigquery.Client(project=project) - - sql = """ - CREATE VIEW `{}.{}.{}` - OPTIONS( - expiration_timestamp=TIMESTAMP_ADD( - CURRENT_TIMESTAMP(), INTERVAL 48 HOUR), - friendly_name="new_view", - description="a view that expires in 2 days", - labels=[("org_unit", "development")] - ) - AS SELECT name, state, year, number - FROM `bigquery-public-data.usa_names.usa_1910_current` - WHERE state LIKE 'W%' - """.format( - project, dataset_id, table_id - ) - - job = client.query(sql) # API request. - job.result() # Waits for the query to finish. - - print( - 'Created new view "{}.{}.{}".'.format( - job.destination.project, - job.destination.dataset_id, - job.destination.table_id, - ) - ) - # [END bigquery_ddl_create_view] - - out, _ = capsys.readouterr() - assert 'Created new view "{}.{}.{}".'.format(project, dataset_id, table_id) in out - - # Test that listing query result rows succeeds so that generic query - # processing tools work with DDL statements. - rows = list(job) - assert len(rows) == 0 - - if pandas is not None: - df = job.to_dataframe() - assert len(df) == 0 - - -def test_client_list_jobs(client): - """List jobs for a project.""" - - # [START bigquery_list_jobs] - # TODO(developer): Uncomment the lines below and replace with your values. - # from google.cloud import bigquery - # project = 'my_project' # replace with your project ID - # client = bigquery.Client(project=project) - import datetime - - # List the 10 most recent jobs in reverse chronological order. - # Omit the max_results parameter to list jobs from the past 6 months. - print("Last 10 jobs:") - for job in client.list_jobs(max_results=10): # API request(s) - print(job.job_id) - - # The following are examples of additional optional parameters: - - # Use min_creation_time and/or max_creation_time to specify a time window. - print("Jobs from the last ten minutes:") - ten_mins_ago = datetime.datetime.utcnow() - datetime.timedelta(minutes=10) - for job in client.list_jobs(min_creation_time=ten_mins_ago): - print(job.job_id) - - # Use all_users to include jobs run by all users in the project. - print("Last 10 jobs run by all users:") - for job in client.list_jobs(max_results=10, all_users=True): - print("{} run by user: {}".format(job.job_id, job.user_email)) - - # Use state_filter to filter by job state. - print("Jobs currently running:") - for job in client.list_jobs(state_filter="RUNNING"): - print(job.job_id) - # [END bigquery_list_jobs] - - -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -def test_query_results_as_dataframe(client): - # [START bigquery_query_results_dataframe] - # from google.cloud import bigquery - # client = bigquery.Client() - - sql = """ - SELECT name, SUM(number) as count - FROM `bigquery-public-data.usa_names.usa_1910_current` - GROUP BY name - ORDER BY count DESC - LIMIT 10 - """ - - df = client.query(sql).to_dataframe() - # [END bigquery_query_results_dataframe] - assert isinstance(df, pandas.DataFrame) - assert len(list(df)) == 2 # verify the number of columns - assert len(df) == 10 # verify the number of rows - - -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -def test_list_rows_as_dataframe(client): - # [START bigquery_list_rows_dataframe] - # from google.cloud import bigquery - # client = bigquery.Client() - - dataset_ref = client.dataset("samples", project="bigquery-public-data") - table_ref = dataset_ref.table("shakespeare") - table = client.get_table(table_ref) - - df = client.list_rows(table).to_dataframe() - # [END bigquery_list_rows_dataframe] - assert isinstance(df, pandas.DataFrame) - assert len(list(df)) == len(table.schema) # verify the number of columns - assert len(df) == table.num_rows # verify the number of rows - - -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.parametrize("parquet_engine", ["pyarrow", "fastparquet"]) -def test_load_table_from_dataframe(client, to_delete, parquet_engine): - if parquet_engine == "pyarrow" and pyarrow is None: - pytest.skip("Requires `pyarrow`") - if parquet_engine == "fastparquet" and fastparquet is None: - pytest.skip("Requires `fastparquet`") - - pandas.set_option("io.parquet.engine", parquet_engine) - - dataset_id = "load_table_from_dataframe_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_load_table_dataframe] - # from google.cloud import bigquery - # import pandas - # client = bigquery.Client() - # dataset_id = 'my_dataset' - - dataset_ref = client.dataset(dataset_id) - table_ref = dataset_ref.table("monty_python") - records = [ - {"title": "The Meaning of Life", "release_year": 1983}, - {"title": "Monty Python and the Holy Grail", "release_year": 1975}, - {"title": "Life of Brian", "release_year": 1979}, - {"title": "And Now for Something Completely Different", "release_year": 1971}, - ] - # Optionally set explicit indices. - # If indices are not specified, a column will be created for the default - # indices created by pandas. - index = ["Q24980", "Q25043", "Q24953", "Q16403"] - dataframe = pandas.DataFrame(records, index=pandas.Index(index, name="wikidata_id")) - - job = client.load_table_from_dataframe(dataframe, table_ref, location="US") - - job.result() # Waits for table load to complete. - - assert job.state == "DONE" - table = client.get_table(table_ref) - assert table.num_rows == 4 - # [END bigquery_load_table_dataframe] - column_names = [field.name for field in table.schema] - assert sorted(column_names) == ["release_year", "title", "wikidata_id"] - - if __name__ == "__main__": pytest.main() diff --git a/bigquery/docs/usage/datasets.rst b/bigquery/docs/usage/datasets.rst index d5646355c00d..6d5cda7d84aa 100644 --- a/bigquery/docs/usage/datasets.rst +++ b/bigquery/docs/usage/datasets.rst @@ -81,3 +81,30 @@ Delete a dataset with the :dedent: 4 :start-after: [START bigquery_delete_dataset] :end-before: [END bigquery_delete_dataset] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/dataset_exists.py + :language: python + :dedent: 4 + :start-after: [START bigquery_dataset_exists] + :end-before: [END bigquery_dataset_exists] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/list_datasets_by_label.py + :language: python + :dedent: 4 + :start-after: [START bigquery_list_datasets_by_label] + :end-before: [END bigquery_list_datasets_by_label] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/manage_dataset_labels.py + :language: python + :dedent: 4 + :start-after: [START bigquery_delete_label_dataset] + :end-before: [END bigquery_delete_label_dataset] diff --git a/bigquery/docs/usage/tables.rst b/bigquery/docs/usage/tables.rst index 4aede9545cd8..4bffe3fc11bb 100644 --- a/bigquery/docs/usage/tables.rst +++ b/bigquery/docs/usage/tables.rst @@ -31,7 +31,7 @@ Get a table resource with the Browse data rows in a table with the :func:`~google.cloud.bigquery.client.Client.list_rows` method: -.. literalinclude:: ../snippets.py +.. literalinclude:: ../samples/browse_table_data.py :language: python :dedent: 4 :start-after: [START bigquery_browse_table] @@ -52,7 +52,7 @@ Create an empty table with the Load table data from a file with the :func:`~google.cloud.bigquery.client.Client.load_table_from_file` method: -.. literalinclude:: ../snippets.py +.. literalinclude:: ../samples/load_table_from_file.py :language: python :dedent: 4 :start-after: [START bigquery_load_from_file] @@ -61,7 +61,7 @@ Load table data from a file with the Load a CSV file from Cloud Storage with the :func:`~google.cloud.bigquery.client.Client.load_table_from_uri` method: -.. literalinclude:: ../snippets.py +.. literalinclude:: ../samples/load_table_from_uri_csv.py :language: python :dedent: 4 :start-after: [START bigquery_load_table_gcs_csv] @@ -72,7 +72,7 @@ See also: `Loading CSV data from Cloud Storage Load a JSON file from Cloud Storage: -.. literalinclude:: ../snippets.py +.. literalinclude:: ../samples/load_table_from_uri_json.py :language: python :dedent: 4 :start-after: [START bigquery_load_table_gcs_json] @@ -83,7 +83,7 @@ See also: `Loading JSON data from Cloud Storage Load a Parquet file from Cloud Storage: -.. literalinclude:: ../snippets.py +.. literalinclude:: ../samples/load_table_from_uri_parquet.py :language: python :dedent: 4 :start-after: [START bigquery_load_table_gcs_parquet] @@ -98,7 +98,7 @@ Updating a Table Update a property in a table's metadata with the :func:`~google.cloud.bigquery.client.Client.update_table` method: -.. literalinclude:: ../snippets.py +.. literalinclude:: ../samples/update_table_description.py :language: python :dedent: 4 :start-after: [START bigquery_update_table_description] @@ -107,7 +107,7 @@ Update a property in a table's metadata with the Insert rows into a table's data with the :func:`~google.cloud.bigquery.client.Client.insert_rows` method: -.. literalinclude:: ../snippets.py +.. literalinclude:: ../samples/table_insert_rows.py :language: python :dedent: 4 :start-after: [START bigquery_table_insert_rows] @@ -119,7 +119,7 @@ Copying a Table Copy a table with the :func:`~google.cloud.bigquery.client.Client.copy_table` method: -.. literalinclude:: ../snippets.py +.. literalinclude:: ../samples/copy_table.py :language: python :dedent: 4 :start-after: [START bigquery_copy_table] @@ -128,7 +128,7 @@ Copy a table with the Copy table data to Google Cloud Storage with the :func:`~google.cloud.bigquery.client.Client.extract_table` method: -.. literalinclude:: ../snippets.py +.. literalinclude:: ../samples/extract_table.py :language: python :dedent: 4 :start-after: [START bigquery_extract_table] @@ -145,3 +145,435 @@ Delete a table with the :dedent: 4 :start-after: [START bigquery_delete_table] :end-before: [END bigquery_delete_table] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/add_empty_column.py + :language: python + :dedent: 4 + :start-after: [START bigquery_add_empty_column] + :end-before: [END bigquery_add_empty_column] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/client_list_jobs.py + :language: python + :dedent: 4 + :start-after: [START bigquery_list_jobs] + :end-before: [END bigquery_list_jobs] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/client_query.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query] + :end-before: [END bigquery_query] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/client_query_add_column.py + :language: python + :dedent: 4 + :start-after: [START bigquery_add_column_query_append] + :end-before: [END bigquery_add_column_query_append] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/client_query_batch.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query_batch] + :end-before: [END bigquery_query_batch] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/client_query_destination_table.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query_destination_table] + :end-before: [END bigquery_query_destination_table] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/client_query_destination_table_cmek.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query_destination_table_cmek] + :end-before: [END bigquery_query_destination_table_cmek] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/client_query_destination_table_legacy.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query_legacy_large_results] + :end-before: [END bigquery_query_legacy_large_results] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/client_query_dry_run.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query_dry_run] + :end-before: [END bigquery_query_dry_run] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/client_query_relax_column.py + :language: python + :dedent: 4 + :start-after: [START bigquery_relax_column_query_append] + :end-before: [END bigquery_relax_column_query_append] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/client_query_total_rows.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query_total_rows] + :end-before: [END bigquery_query_total_rows] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/client_query_w_array_params.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query_params_arrays] + :end-before: [END bigquery_query_params_arrays] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/client_query_w_named_params.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query_params_named] + :end-before: [END bigquery_query_params_named] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/client_query_w_positional_params.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query_params_positional] + :end-before: [END bigquery_query_params_positional] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/client_query_w_struct_params.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query_params_structs] + :end-before: [END bigquery_query_params_structs] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/client_query_w_timestamp_params.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query_params_timestamps] + :end-before: [END bigquery_query_params_timestamps] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/copy_table_cmek.py + :language: python + :dedent: 4 + :start-after: [START bigquery_copy_table_cmek] + :end-before: [END bigquery_copy_table_cmek] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/copy_table_multiple_source.py + :language: python + :dedent: 4 + :start-after: [START bigquery_copy_table_multiple_source] + :end-before: [END bigquery_copy_table_multiple_source] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/create_client_default_credentials.py + :language: python + :dedent: 4 + :start-after: [START bigquery_client_default_credentials] + :end-before: [END bigquery_client_default_credentials] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/create_partitioned_table.py + :language: python + :dedent: 4 + :start-after: [START bigquery_create_table_partitioned] + :end-before: [END bigquery_create_table_partitioned] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/create_table_cmek.py + :language: python + :dedent: 4 + :start-after: [START bigquery_create_table_cmek] + :end-before: [END bigquery_create_table_cmek] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/create_table_nested_repeated_schema.py + :language: python + :dedent: 4 + :start-after: [START bigquery_nested_repeated_schema] + :end-before: [END bigquery_nested_repeated_schema] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/ddl_create_view.py + :language: python + :dedent: 4 + :start-after: [START bigquery_ddl_create_view] + :end-before: [END bigquery_ddl_create_view] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/extract_table_compressed.py + :language: python + :dedent: 4 + :start-after: [START bigquery_extract_table_compressed] + :end-before: [END bigquery_extract_table_compressed] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/extract_table_json.py + :language: python + :dedent: 4 + :start-after: [START bigquery_extract_table_json] + :end-before: [END bigquery_extract_table_json] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/list_rows_as_dataframe.py + :language: python + :dedent: 4 + :start-after: [START bigquery_list_rows_dataframe] + :end-before: [END bigquery_list_rows_dataframe] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/load_and_query_partitioned_table.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query_partitioned_table] + :end-before: [END bigquery_query_partitioned_table] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/load_table_add_column.py + :language: python + :dedent: 4 + :start-after: [START bigquery_add_column_load_append] + :end-before: [END bigquery_add_column_load_append] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/load_table_from_dataframe.py + :language: python + :dedent: 4 + :start-after: [START bigquery_load_table_dataframe] + :end-before: [END bigquery_load_table_dataframe] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/load_table_from_uri_autodetect.py + :language: python + :dedent: 4 + :start-after: [START bigquery_load_table_gcs_json_autodetect] + :end-before: [END bigquery_load_table_gcs_json_autodetect] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/load_table_from_uri_avro.py + :language: python + :dedent: 4 + :start-after: [START bigquery_load_table_gcs_avro] + :end-before: [END bigquery_load_table_gcs_avro] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/load_table_from_uri_cmek.py + :language: python + :dedent: 4 + :start-after: [START bigquery_load_table_gcs_json_cmek] + :end-before: [END bigquery_load_table_gcs_json_cmek] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/load_table_from_uri_orc.py + :language: python + :dedent: 4 + :start-after: [START bigquery_load_table_gcs_orc] + :end-before: [END bigquery_load_table_gcs_orc] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/load_table_from_uri_truncate.py + :language: python + :dedent: 4 + :start-after: [START bigquery_load_table_gcs_orc_truncate] + :end-before: [END bigquery_load_table_gcs_orc_truncate] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/load_table_relax_column.py + :language: python + :dedent: 4 + :start-after: [START bigquery_relax_column_load_append] + :end-before: [END bigquery_relax_column_load_append] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/manage_job.py + :language: python + :dedent: 4 + :start-after: [START bigquery_get_job] + :end-before: [END bigquery_get_job] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/manage_table_labels.py + :language: python + :dedent: 4 + :start-after: [START bigquery_delete_label_table] + :end-before: [END bigquery_delete_label_table] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/manage_views.py + :language: python + :dedent: 4 + :start-after: [START bigquery_grant_view_access] + :end-before: [END bigquery_grant_view_access] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/query_external_gcs_permanent_table.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query_external_gcs_perm] + :end-before: [END bigquery_query_external_gcs_perm] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/query_external_gcs_temporary_table.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query_external_gcs_temp] + :end-before: [END bigquery_query_external_gcs_temp] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/query_external_sheets_permanent_table.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query_external_sheets_perm] + :end-before: [END bigquery_query_external_sheets_perm] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/query_external_sheets_temporary_table.py + :language: python + :dedent: 4 + :start-after: [START bigquery_auth_drive_scope] + :end-before: [END bigquery_auth_drive_scope] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/query_no_cache.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query_no_cache] + :end-before: [END bigquery_query_no_cache] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/query_results_as_dataframe.py + :language: python + :dedent: 4 + :start-after: [START bigquery_query_results_dataframe] + :end-before: [END bigquery_query_results_dataframe] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/table_exists.py + :language: python + :dedent: 4 + :start-after: [START bigquery_table_exists] + :end-before: [END bigquery_table_exists] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/undelete_table.py + :language: python + :dedent: 4 + :start-after: [START bigquery_undelete_table] + :end-before: [END bigquery_undelete_table] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/update_table_cmek.py + :language: python + :dedent: 4 + :start-after: [START bigquery_update_table_cmek] + :end-before: [END bigquery_update_table_cmek] + +[-REPLACE_COMMENT-] +:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: + +.. literalinclude:: ../samples/samples/update_table_expiration.py + :language: python + :dedent: 4 + :start-after: [START bigquery_update_table_expiration] + :end-before: [END bigquery_update_table_expiration] diff --git a/bigquery/samples/add_empty_column.py b/bigquery/samples/add_empty_column.py new file mode 100644 index 000000000000..29d6241f8591 --- /dev/null +++ b/bigquery/samples/add_empty_column.py @@ -0,0 +1,46 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def add_empty_column(client, to_delete): + + # [START bigquery_add_empty_column] + """Adds an empty column to an existing table.""" + dataset_id = "add_empty_column_dataset_{}".format(_millis()) + table_id = "add_empty_column_table_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + dataset = client.create_dataset(dataset) + to_delete.append(dataset) + + table = bigquery.Table(dataset.table(table_id), schema=SCHEMA) + table = client.create_table(table) + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'my_dataset' + # table_id = 'my_table' + + table_ref = client.dataset(dataset_id).table(table_id) + table = client.get_table(table_ref) # API request + + original_schema = table.schema + new_schema = original_schema[:] # creates a copy of the schema + new_schema.append(bigquery.SchemaField("phone", "STRING")) + + table.schema = new_schema + table = client.update_table(table, ["schema"]) # API request + + assert len(table.schema) == len(original_schema) + 1 == len(new_schema) + + # [END bigquery_add_empty_column] \ No newline at end of file diff --git a/bigquery/samples/browse_table_data.py b/bigquery/samples/browse_table_data.py new file mode 100644 index 000000000000..35dda3d55582 --- /dev/null +++ b/bigquery/samples/browse_table_data.py @@ -0,0 +1,56 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def browse_table_data(client, to_delete, capsys): + + # [START bigquery_browse_table] + """Retreive selected row data from a table.""" + + # from google.cloud import bigquery + # client = bigquery.Client() + + dataset_ref = client.dataset("samples", project="bigquery-public-data") + table_ref = dataset_ref.table("shakespeare") + table = client.get_table(table_ref) # API call + + # Load all rows from a table + rows = client.list_rows(table) + assert len(list(rows)) == table.num_rows + + # Load the first 10 rows + rows = client.list_rows(table, max_results=10) + assert len(list(rows)) == 10 + + # Specify selected fields to limit the results to certain columns + fields = table.schema[:2] # first two columns + rows = client.list_rows(table, selected_fields=fields, max_results=10) + assert len(rows.schema) == 2 + assert len(list(rows)) == 10 + + # Use the start index to load an arbitrary portion of the table + rows = client.list_rows(table, start_index=10, max_results=10) + + # Print row data in tabular format + format_string = "{!s:<16} " * len(rows.schema) + field_names = [field.name for field in rows.schema] + print(format_string.format(*field_names)) # prints column headers + for row in rows: + print(format_string.format(*row)) # prints row data + + out, err = capsys.readouterr() + out = list(filter(bool, out.split("\n"))) # list of non-blank lines + assert len(out) == 11 + + # [END bigquery_browse_table] \ No newline at end of file diff --git a/bigquery/samples/client_list_jobs.py b/bigquery/samples/client_list_jobs.py new file mode 100644 index 000000000000..d63c6a7c9b6c --- /dev/null +++ b/bigquery/samples/client_list_jobs.py @@ -0,0 +1,51 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_list_jobs(client): + + # [START bigquery_list_jobs] + """List jobs for a project.""" + + # TODO(developer): Uncomment the lines below and replace with your values. + # from google.cloud import bigquery + # project = 'my_project' # replace with your project ID + # client = bigquery.Client(project=project) + import datetime + + # List the 10 most recent jobs in reverse chronological order. + # Omit the max_results parameter to list jobs from the past 6 months. + print("Last 10 jobs:") + for job in client.list_jobs(max_results=10): # API request(s) + print(job.job_id) + + # The following are examples of additional optional parameters: + + # Use min_creation_time and/or max_creation_time to specify a time window. + print("Jobs from the last ten minutes:") + ten_mins_ago = datetime.datetime.utcnow() - datetime.timedelta(minutes=10) + for job in client.list_jobs(min_creation_time=ten_mins_ago): + print(job.job_id) + + # Use all_users to include jobs run by all users in the project. + print("Last 10 jobs run by all users:") + for job in client.list_jobs(max_results=10, all_users=True): + print("{} run by user: {}".format(job.job_id, job.user_email)) + + # Use state_filter to filter by job state. + print("Jobs currently running:") + for job in client.list_jobs(state_filter="RUNNING"): + print(job.job_id) + + # [END bigquery_list_jobs] \ No newline at end of file diff --git a/bigquery/samples/client_query.py b/bigquery/samples/client_query.py new file mode 100644 index 000000000000..b18e9c6e16f4 --- /dev/null +++ b/bigquery/samples/client_query.py @@ -0,0 +1,40 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query(client): + + # [START bigquery_query] + """Run a simple query.""" + + # from google.cloud import bigquery + # client = bigquery.Client() + + query = ( + "SELECT name FROM `bigquery-public-data.usa_names.usa_1910_2013` " + 'WHERE state = "TX" ' + "LIMIT 100" + ) + query_job = client.query( + query, + # Location must match that of the dataset(s) referenced in the query. + location="US", + ) # API request - starts the query + + for row in query_job: # API request - fetches results + # Row values can be accessed by field name or index + assert row[0] == row.name == row["name"] + print(row) + + # [END bigquery_query] \ No newline at end of file diff --git a/bigquery/samples/client_query_add_column.py b/bigquery/samples/client_query_add_column.py new file mode 100644 index 000000000000..4464b2665b7d --- /dev/null +++ b/bigquery/samples/client_query_add_column.py @@ -0,0 +1,72 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_add_column(client, to_delete): + + # [START bigquery_add_column_query_append] + dataset_id = "query_add_column_{}".format(_millis()) + dataset_ref = client.dataset(dataset_id) + dataset = bigquery.Dataset(dataset_ref) + dataset.location = "US" + dataset = client.create_dataset(dataset) + to_delete.append(dataset) + + table_ref = dataset_ref.table("my_table") + schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + ] + table = client.create_table(bigquery.Table(table_ref, schema=schema)) + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_ref = client.dataset('my_dataset') + + # Retrieves the destination table and checks the length of the schema + table_id = "my_table" + table_ref = dataset_ref.table(table_id) + table = client.get_table(table_ref) + print("Table {} contains {} columns.".format(table_id, len(table.schema))) + + # Configures the query to append the results to a destination table, + # allowing field addition + job_config = bigquery.QueryJobConfig() + job_config.schema_update_options = [ + bigquery.SchemaUpdateOption.ALLOW_FIELD_ADDITION + ] + job_config.destination = table_ref + job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND + + query_job = client.query( + # In this example, the existing table contains only the 'full_name' and + # 'age' columns, while the results of this query will contain an + # additional 'favorite_color' column. + 'SELECT "Timmy" as full_name, 85 as age, "Blue" as favorite_color;', + # Location must match that of the dataset(s) referenced in the query + # and of the destination table. + location="US", + job_config=job_config, + ) # API request - starts the query + + query_job.result() # Waits for the query to finish + print("Query job {} complete.".format(query_job.job_id)) + + # Checks the updated length of the schema + table = client.get_table(table) + print("Table {} now contains {} columns.".format(table_id, len(table.schema))) + assert len(table.schema) == 3 + assert table.num_rows > 0 + + # [END bigquery_add_column_query_append] \ No newline at end of file diff --git a/bigquery/samples/client_query_batch.py b/bigquery/samples/client_query_batch.py new file mode 100644 index 000000000000..465ca3648860 --- /dev/null +++ b/bigquery/samples/client_query_batch.py @@ -0,0 +1,43 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_batch(client, to_delete): + + # [START bigquery_query_batch] + # from google.cloud import bigquery + # client = bigquery.Client() + + job_config = bigquery.QueryJobConfig() + # Run at batch priority, which won't count toward concurrent rate limit. + job_config.priority = bigquery.QueryPriority.BATCH + sql = """ + SELECT corpus + FROM `bigquery-public-data.samples.shakespeare` + GROUP BY corpus; + """ + # Location must match that of the dataset(s) referenced in the query. + location = "US" + + # API request - starts the query + query_job = client.query(sql, location=location, job_config=job_config) + + # Check on the progress by getting the job's updated state. Once the state + # is `DONE`, the results are ready. + query_job = client.get_job( + query_job.job_id, location=location + ) # API request - fetches job + print("Job {} is currently in state {}".format(query_job.job_id, query_job.state)) + + # [END bigquery_query_batch] \ No newline at end of file diff --git a/bigquery/samples/client_query_destination_table.py b/bigquery/samples/client_query_destination_table.py new file mode 100644 index 000000000000..1b3f3ddbd0fe --- /dev/null +++ b/bigquery/samples/client_query_destination_table.py @@ -0,0 +1,53 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_destination_table(client, to_delete): + + # [START bigquery_query_destination_table] + """Run a query""" + dataset_id = "query_destination_table_{}".format(_millis()) + dataset_ref = client.dataset(dataset_id) + to_delete.append(dataset_ref) + dataset = bigquery.Dataset(dataset_ref) + dataset.location = "US" + client.create_dataset(dataset) + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'your_dataset_id' + + job_config = bigquery.QueryJobConfig() + # Set the destination table + table_ref = client.dataset(dataset_id).table("your_table_id") + job_config.destination = table_ref + sql = """ + SELECT corpus + FROM `bigquery-public-data.samples.shakespeare` + GROUP BY corpus; + """ + + # Start the query, passing in the extra configuration. + query_job = client.query( + sql, + # Location must match that of the dataset(s) referenced in the query + # and of the destination table. + location="US", + job_config=job_config, + ) # API request - starts the query + + query_job.result() # Waits for the query to finish + print("Query results loaded to table {}".format(table_ref.path)) + + # [END bigquery_query_destination_table] \ No newline at end of file diff --git a/bigquery/samples/client_query_destination_table_cmek.py b/bigquery/samples/client_query_destination_table_cmek.py new file mode 100644 index 000000000000..44c9a6259cd1 --- /dev/null +++ b/bigquery/samples/client_query_destination_table_cmek.py @@ -0,0 +1,59 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_destination_table_cmek(client, to_delete): + + # [START bigquery_query_destination_table_cmek] + """Run a query""" + dataset_id = "query_destination_table_{}".format(_millis()) + dataset_ref = client.dataset(dataset_id) + to_delete.append(dataset_ref) + dataset = bigquery.Dataset(dataset_ref) + dataset.location = "US" + client.create_dataset(dataset) + + # from google.cloud import bigquery + # client = bigquery.Client() + + job_config = bigquery.QueryJobConfig() + + # Set the destination table. Here, dataset_id is a string, such as: + # dataset_id = 'your_dataset_id' + table_ref = client.dataset(dataset_id).table("your_table_id") + job_config.destination = table_ref + + # Set the encryption key to use for the destination. + # TODO: Replace this key with a key you have created in KMS. + kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format( + "cloud-samples-tests", "us-central1", "test", "test" + ) + encryption_config = bigquery.EncryptionConfiguration(kms_key_name=kms_key_name) + job_config.destination_encryption_configuration = encryption_config + + # Start the query, passing in the extra configuration. + query_job = client.query( + "SELECT 17 AS my_col;", + # Location must match that of the dataset(s) referenced in the query + # and of the destination table. + location="US", + job_config=job_config, + ) # API request - starts the query + query_job.result() + + # The destination table is written using the encryption configuration. + table = client.get_table(table_ref) + assert table.encryption_configuration.kms_key_name == kms_key_name + + # [END bigquery_query_destination_table_cmek] \ No newline at end of file diff --git a/bigquery/samples/client_query_destination_table_legacy.py b/bigquery/samples/client_query_destination_table_legacy.py new file mode 100644 index 000000000000..8d0780af2377 --- /dev/null +++ b/bigquery/samples/client_query_destination_table_legacy.py @@ -0,0 +1,54 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_destination_table_legacy(client, to_delete): + + # [START bigquery_query_legacy_large_results] + dataset_id = "query_destination_table_legacy_{}".format(_millis()) + dataset_ref = client.dataset(dataset_id) + to_delete.append(dataset_ref) + dataset = bigquery.Dataset(dataset_ref) + dataset.location = "US" + client.create_dataset(dataset) + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'your_dataset_id' + + job_config = bigquery.QueryJobConfig() + # Set use_legacy_sql to True to use legacy SQL syntax. + job_config.use_legacy_sql = True + # Set the destination table + table_ref = client.dataset(dataset_id).table("your_table_id") + job_config.destination = table_ref + job_config.allow_large_results = True + sql = """ + SELECT corpus + FROM [bigquery-public-data:samples.shakespeare] + GROUP BY corpus; + """ + # Start the query, passing in the extra configuration. + query_job = client.query( + sql, + # Location must match that of the dataset(s) referenced in the query + # and of the destination table. + location="US", + job_config=job_config, + ) # API request - starts the query + + query_job.result() # Waits for the query to finish + print("Query results loaded to table {}".format(table_ref.path)) + + # [END bigquery_query_legacy_large_results] \ No newline at end of file diff --git a/bigquery/samples/client_query_dry_run.py b/bigquery/samples/client_query_dry_run.py new file mode 100644 index 000000000000..74f26f612581 --- /dev/null +++ b/bigquery/samples/client_query_dry_run.py @@ -0,0 +1,47 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_dry_run(client): + + # [START bigquery_query_dry_run] + """Run a dry run query""" + + # from google.cloud import bigquery + # client = bigquery.Client() + + job_config = bigquery.QueryJobConfig() + job_config.dry_run = True + job_config.use_query_cache = False + query_job = client.query( + ( + "SELECT name, COUNT(*) as name_count " + "FROM `bigquery-public-data.usa_names.usa_1910_2013` " + "WHERE state = 'WA' " + "GROUP BY name" + ), + # Location must match that of the dataset(s) referenced in the query. + location="US", + job_config=job_config, + ) # API request + + # A dry run query completes immediately. + assert query_job.state == "DONE" + assert query_job.dry_run + + print("This query will process {} bytes.".format(query_job.total_bytes_processed)) + + assert query_job.total_bytes_processed > 0 + + # [END bigquery_query_dry_run] \ No newline at end of file diff --git a/bigquery/samples/client_query_legacy_sql.py b/bigquery/samples/client_query_legacy_sql.py new file mode 100644 index 000000000000..17464026ba44 --- /dev/null +++ b/bigquery/samples/client_query_legacy_sql.py @@ -0,0 +1,44 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_legacy_sql(client): + + # [START bigquery_query_legacy] + """Run a query with Legacy SQL explicitly set""" + # from google.cloud import bigquery + # client = bigquery.Client() + + query = ( + "SELECT name FROM [bigquery-public-data:usa_names.usa_1910_2013] " + 'WHERE state = "TX" ' + "LIMIT 100" + ) + + # Set use_legacy_sql to True to use legacy SQL syntax. + job_config = bigquery.QueryJobConfig() + job_config.use_legacy_sql = True + + query_job = client.query( + query, + # Location must match that of the dataset(s) referenced in the query. + location="US", + job_config=job_config, + ) # API request - starts the query + + # Print the results. + for row in query_job: # API request - fetches results + print(row) + + # [END bigquery_query_legacy] \ No newline at end of file diff --git a/bigquery/samples/client_query_relax_column.py b/bigquery/samples/client_query_relax_column.py new file mode 100644 index 000000000000..2a8079cd9817 --- /dev/null +++ b/bigquery/samples/client_query_relax_column.py @@ -0,0 +1,76 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_relax_column(client, to_delete): + + # [START bigquery_relax_column_query_append] + dataset_id = "query_relax_column_{}".format(_millis()) + dataset_ref = client.dataset(dataset_id) + dataset = bigquery.Dataset(dataset_ref) + dataset.location = "US" + dataset = client.create_dataset(dataset) + to_delete.append(dataset) + + table_ref = dataset_ref.table("my_table") + schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + ] + table = client.create_table(bigquery.Table(table_ref, schema=schema)) + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_ref = client.dataset('my_dataset') + + # Retrieves the destination table and checks the number of required fields + table_id = "my_table" + table_ref = dataset_ref.table(table_id) + table = client.get_table(table_ref) + original_required_fields = sum(field.mode == "REQUIRED" for field in table.schema) + # In this example, the existing table has 2 required fields + print("{} fields in the schema are required.".format(original_required_fields)) + + # Configures the query to append the results to a destination table, + # allowing field relaxation + job_config = bigquery.QueryJobConfig() + job_config.schema_update_options = [ + bigquery.SchemaUpdateOption.ALLOW_FIELD_RELAXATION + ] + job_config.destination = table_ref + job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND + + query_job = client.query( + # In this example, the existing table contains 'full_name' and 'age' as + # required columns, but the query results will omit the second column. + 'SELECT "Beyonce" as full_name;', + # Location must match that of the dataset(s) referenced in the query + # and of the destination table. + location="US", + job_config=job_config, + ) # API request - starts the query + + query_job.result() # Waits for the query to finish + print("Query job {} complete.".format(query_job.job_id)) + + # Checks the updated number of required fields + table = client.get_table(table) + current_required_fields = sum(field.mode == "REQUIRED" for field in table.schema) + print("{} fields in the schema are now required.".format(current_required_fields)) + assert original_required_fields - current_required_fields > 0 + assert len(table.schema) == 2 + assert table.schema[1].mode == "NULLABLE" + assert table.num_rows > 0 + + # [END bigquery_relax_column_query_append] \ No newline at end of file diff --git a/bigquery/samples/client_query_total_rows.py b/bigquery/samples/client_query_total_rows.py new file mode 100644 index 000000000000..033b8eb68a16 --- /dev/null +++ b/bigquery/samples/client_query_total_rows.py @@ -0,0 +1,40 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_total_rows(client, capsys): + + # [START bigquery_query_total_rows] + """Run a query and just check for how many rows.""" + # from google.cloud import bigquery + # client = bigquery.Client() + + query = ( + "SELECT name FROM `bigquery-public-data.usa_names.usa_1910_2013` " + 'WHERE state = "TX" ' + "LIMIT 100" + ) + query_job = client.query( + query, + # Location must match that of the dataset(s) referenced in the query. + location="US", + ) # API request - starts the query + + results = query_job.result() # Wait for query to complete. + print("Got {} rows.".format(results.total_rows)) + + out, _ = capsys.readouterr() + assert "Got 100 rows." in out + + # [END bigquery_query_total_rows] \ No newline at end of file diff --git a/bigquery/samples/client_query_w_array_params.py b/bigquery/samples/client_query_w_array_params.py new file mode 100644 index 000000000000..8fdece2149f1 --- /dev/null +++ b/bigquery/samples/client_query_w_array_params.py @@ -0,0 +1,54 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_w_array_params(client, capsys): + + # [START bigquery_query_params_arrays] + """Run a query using array query parameters""" + # from google.cloud import bigquery + # client = bigquery.Client() + + query = """ + SELECT name, sum(number) as count + FROM `bigquery-public-data.usa_names.usa_1910_2013` + WHERE gender = @gender + AND state IN UNNEST(@states) + GROUP BY name + ORDER BY count DESC + LIMIT 10; + """ + query_params = [ + bigquery.ScalarQueryParameter("gender", "STRING", "M"), + bigquery.ArrayQueryParameter("states", "STRING", ["WA", "WI", "WV", "WY"]), + ] + job_config = bigquery.QueryJobConfig() + job_config.query_parameters = query_params + query_job = client.query( + query, + # Location must match that of the dataset(s) referenced in the query. + location="US", + job_config=job_config, + ) # API request - starts the query + + # Print the results + for row in query_job: + print("{}: \t{}".format(row.name, row.count)) + + assert query_job.state == "DONE" + + out, _ = capsys.readouterr() + assert "James" in out + + # [END bigquery_query_params_arrays] \ No newline at end of file diff --git a/bigquery/samples/client_query_w_named_params.py b/bigquery/samples/client_query_w_named_params.py new file mode 100644 index 000000000000..9ef49a29cfe3 --- /dev/null +++ b/bigquery/samples/client_query_w_named_params.py @@ -0,0 +1,53 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_w_named_params(client, capsys): + + # [START bigquery_query_params_named] + """Run a query using named query parameters""" + + # from google.cloud import bigquery + # client = bigquery.Client() + + query = """ + SELECT word, word_count + FROM `bigquery-public-data.samples.shakespeare` + WHERE corpus = @corpus + AND word_count >= @min_word_count + ORDER BY word_count DESC; + """ + query_params = [ + bigquery.ScalarQueryParameter("corpus", "STRING", "romeoandjuliet"), + bigquery.ScalarQueryParameter("min_word_count", "INT64", 250), + ] + job_config = bigquery.QueryJobConfig() + job_config.query_parameters = query_params + query_job = client.query( + query, + # Location must match that of the dataset(s) referenced in the query. + location="US", + job_config=job_config, + ) # API request - starts the query + + # Print the results + for row in query_job: + print("{}: \t{}".format(row.word, row.word_count)) + + assert query_job.state == "DONE" + + out, _ = capsys.readouterr() + assert "the" in out + + # [END bigquery_query_params_named] \ No newline at end of file diff --git a/bigquery/samples/client_query_w_positional_params.py b/bigquery/samples/client_query_w_positional_params.py new file mode 100644 index 000000000000..de22ac5d94ac --- /dev/null +++ b/bigquery/samples/client_query_w_positional_params.py @@ -0,0 +1,55 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_w_positional_params(client, capsys): + + # [START bigquery_query_params_positional] + """Run a query using query parameters""" + + # from google.cloud import bigquery + # client = bigquery.Client() + + query = """ + SELECT word, word_count + FROM `bigquery-public-data.samples.shakespeare` + WHERE corpus = ? + AND word_count >= ? + ORDER BY word_count DESC; + """ + # Set the name to None to use positional parameters. + # Note that you cannot mix named and positional parameters. + query_params = [ + bigquery.ScalarQueryParameter(None, "STRING", "romeoandjuliet"), + bigquery.ScalarQueryParameter(None, "INT64", 250), + ] + job_config = bigquery.QueryJobConfig() + job_config.query_parameters = query_params + query_job = client.query( + query, + # Location must match that of the dataset(s) referenced in the query. + location="US", + job_config=job_config, + ) # API request - starts the query + + # Print the results + for row in query_job: + print("{}: \t{}".format(row.word, row.word_count)) + + assert query_job.state == "DONE" + + out, _ = capsys.readouterr() + assert "the" in out + + # [END bigquery_query_params_positional] \ No newline at end of file diff --git a/bigquery/samples/client_query_w_struct_params.py b/bigquery/samples/client_query_w_struct_params.py new file mode 100644 index 000000000000..af44d49cd712 --- /dev/null +++ b/bigquery/samples/client_query_w_struct_params.py @@ -0,0 +1,50 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_w_struct_params(client, capsys): + + # [START bigquery_query_params_structs] + """Run a query using struct query parameters""" + # from google.cloud import bigquery + # client = bigquery.Client() + + query = "SELECT @struct_value AS s;" + query_params = [ + bigquery.StructQueryParameter( + "struct_value", + bigquery.ScalarQueryParameter("x", "INT64", 1), + bigquery.ScalarQueryParameter("y", "STRING", "foo"), + ) + ] + job_config = bigquery.QueryJobConfig() + job_config.query_parameters = query_params + query_job = client.query( + query, + # Location must match that of the dataset(s) referenced in the query. + location="US", + job_config=job_config, + ) # API request - starts the query + + # Print the results + for row in query_job: + print(row.s) + + assert query_job.state == "DONE" + + out, _ = capsys.readouterr() + assert "1" in out + assert "foo" in out + + # [END bigquery_query_params_structs] \ No newline at end of file diff --git a/bigquery/samples/client_query_w_timestamp_params.py b/bigquery/samples/client_query_w_timestamp_params.py new file mode 100644 index 000000000000..3f668b2c75ef --- /dev/null +++ b/bigquery/samples/client_query_w_timestamp_params.py @@ -0,0 +1,53 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_w_timestamp_params(client, capsys): + + # [START bigquery_query_params_timestamps] + """Run a query using query parameters""" + + # from google.cloud import bigquery + # client = bigquery.Client() + + import datetime + import pytz + + query = "SELECT TIMESTAMP_ADD(@ts_value, INTERVAL 1 HOUR);" + query_params = [ + bigquery.ScalarQueryParameter( + "ts_value", + "TIMESTAMP", + datetime.datetime(2016, 12, 7, 8, 0, tzinfo=pytz.UTC), + ) + ] + job_config = bigquery.QueryJobConfig() + job_config.query_parameters = query_params + query_job = client.query( + query, + # Location must match that of the dataset(s) referenced in the query. + location="US", + job_config=job_config, + ) # API request - starts the query + + # Print the results + for row in query_job: + print(row) + + assert query_job.state == "DONE" + + out, _ = capsys.readouterr() + assert "2016, 12, 7, 9, 0" in out + + # [END bigquery_query_params_timestamps] \ No newline at end of file diff --git a/bigquery/samples/copy_table.py b/bigquery/samples/copy_table.py new file mode 100644 index 000000000000..1e12b43a43cc --- /dev/null +++ b/bigquery/samples/copy_table.py @@ -0,0 +1,47 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def copy_table(client, to_delete): + + # [START bigquery_copy_table] + dataset_id = "copy_table_dataset_{}".format(_millis()) + dest_dataset = bigquery.Dataset(client.dataset(dataset_id)) + dest_dataset.location = "US" + dest_dataset = client.create_dataset(dest_dataset) + to_delete.append(dest_dataset) + + # from google.cloud import bigquery + # client = bigquery.Client() + + source_dataset = client.dataset("samples", project="bigquery-public-data") + source_table_ref = source_dataset.table("shakespeare") + + # dataset_id = 'my_dataset' + dest_table_ref = client.dataset(dataset_id).table("destination_table") + + job = client.copy_table( + source_table_ref, + dest_table_ref, + # Location must match that of the source and destination tables. + location="US", + ) # API request + + job.result() # Waits for job to complete. + + assert job.state == "DONE" + dest_table = client.get_table(dest_table_ref) # API request + assert dest_table.num_rows > 0 + + # [END bigquery_copy_table] \ No newline at end of file diff --git a/bigquery/samples/copy_table_cmek.py b/bigquery/samples/copy_table_cmek.py new file mode 100644 index 000000000000..53324f439d7f --- /dev/null +++ b/bigquery/samples/copy_table_cmek.py @@ -0,0 +1,57 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def copy_table_cmek(client, to_delete): + + # [START bigquery_copy_table_cmek] + dataset_id = "copy_table_cmek_{}".format(_millis()) + dest_dataset = bigquery.Dataset(client.dataset(dataset_id)) + dest_dataset.location = "US" + dest_dataset = client.create_dataset(dest_dataset) + to_delete.append(dest_dataset) + + # from google.cloud import bigquery + # client = bigquery.Client() + + source_dataset = bigquery.DatasetReference("bigquery-public-data", "samples") + source_table_ref = source_dataset.table("shakespeare") + + # dataset_id = 'my_dataset' + dest_dataset_ref = client.dataset(dataset_id) + dest_table_ref = dest_dataset_ref.table("destination_table") + + # Set the encryption key to use for the destination. + # TODO: Replace this key with a key you have created in KMS. + kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format( + "cloud-samples-tests", "us-central1", "test", "test" + ) + encryption_config = bigquery.EncryptionConfiguration(kms_key_name=kms_key_name) + job_config = bigquery.CopyJobConfig() + job_config.destination_encryption_configuration = encryption_config + + job = client.copy_table( + source_table_ref, + dest_table_ref, + # Location must match that of the source and destination tables. + location="US", + job_config=job_config, + ) # API request + job.result() # Waits for job to complete. + + assert job.state == "DONE" + dest_table = client.get_table(dest_table_ref) + assert dest_table.encryption_configuration.kms_key_name == kms_key_name + + # [END bigquery_copy_table_cmek] \ No newline at end of file diff --git a/bigquery/samples/copy_table_multiple_source.py b/bigquery/samples/copy_table_multiple_source.py new file mode 100644 index 000000000000..20b74c7b8149 --- /dev/null +++ b/bigquery/samples/copy_table_multiple_source.py @@ -0,0 +1,73 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def copy_table_multiple_source(client, to_delete): + + # [START bigquery_copy_table_multiple_source] + dest_dataset_id = "dest_dataset_{}".format(_millis()) + dest_dataset = bigquery.Dataset(client.dataset(dest_dataset_id)) + dest_dataset.location = "US" + dest_dataset = client.create_dataset(dest_dataset) + to_delete.append(dest_dataset) + + source_dataset_id = "source_dataset_{}".format(_millis()) + source_dataset = bigquery.Dataset(client.dataset(source_dataset_id)) + source_dataset.location = "US" + source_dataset = client.create_dataset(source_dataset) + to_delete.append(source_dataset) + + schema = [ + bigquery.SchemaField("name", "STRING"), + bigquery.SchemaField("post_abbr", "STRING"), + ] + + table_data = {"table1": b"Washington,WA", "table2": b"California,CA"} + for table_id, data in table_data.items(): + table_ref = source_dataset.table(table_id) + job_config = bigquery.LoadJobConfig() + job_config.schema = schema + body = six.BytesIO(data) + client.load_table_from_file( + body, + table_ref, + # Location must match that of the destination dataset. + location="US", + job_config=job_config, + ).result() + + # from google.cloud import bigquery + # client = bigquery.Client() + # source_dataset_id = 'my_source_dataset' + # dest_dataset_id = 'my_destination_dataset' + + table1_ref = client.dataset(source_dataset_id).table("table1") + table2_ref = client.dataset(source_dataset_id).table("table2") + dest_table_ref = client.dataset(dest_dataset_id).table("destination_table") + + job = client.copy_table( + [table1_ref, table2_ref], + dest_table_ref, + # Location must match that of the source and destination tables. + location="US", + ) # API request + job.result() # Waits for job to complete. + + assert job.state == "DONE" + dest_table = client.get_table(dest_table_ref) # API request + assert dest_table.num_rows > 0 + + assert dest_table.num_rows == 2 + + # [END bigquery_copy_table_multiple_source] \ No newline at end of file diff --git a/bigquery/samples/create_client_default_credentials.py b/bigquery/samples/create_client_default_credentials.py new file mode 100644 index 000000000000..2716cdbffc66 --- /dev/null +++ b/bigquery/samples/create_client_default_credentials.py @@ -0,0 +1,29 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def create_client_default_credentials(): + + # [START bigquery_client_default_credentials] + """Create a BigQuery client with Application Default Credentials""" + + from google.cloud import bigquery + + # If you don't specify credentials when constructing the client, the + # client library will look for credentials in the environment. + client = bigquery.Client() + + assert client is not None + + # [END bigquery_client_default_credentials] \ No newline at end of file diff --git a/bigquery/samples/create_partitioned_table.py b/bigquery/samples/create_partitioned_table.py new file mode 100644 index 000000000000..4fc4eba0652c --- /dev/null +++ b/bigquery/samples/create_partitioned_table.py @@ -0,0 +1,53 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def create_partitioned_table(client, to_delete): + + # [START bigquery_create_table_partitioned] + dataset_id = "create_table_partitioned_{}".format(_millis()) + dataset_ref = bigquery.Dataset(client.dataset(dataset_id)) + dataset = client.create_dataset(dataset_ref) + to_delete.append(dataset) + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_ref = client.dataset('my_dataset') + + table_ref = dataset_ref.table("my_partitioned_table") + schema = [ + bigquery.SchemaField("name", "STRING"), + bigquery.SchemaField("post_abbr", "STRING"), + bigquery.SchemaField("date", "DATE"), + ] + table = bigquery.Table(table_ref, schema=schema) + table.time_partitioning = bigquery.TimePartitioning( + type_=bigquery.TimePartitioningType.DAY, + field="date", # name of column to use for partitioning + expiration_ms=7776000000, + ) # 90 days + + table = client.create_table(table) + + print( + "Created table {}, partitioned on column {}".format( + table.table_id, table.time_partitioning.field + ) + ) + + assert table.time_partitioning.type_ == "DAY" + assert table.time_partitioning.field == "date" + assert table.time_partitioning.expiration_ms == 7776000000 + + # [END bigquery_create_table_partitioned] \ No newline at end of file diff --git a/bigquery/samples/create_table_cmek.py b/bigquery/samples/create_table_cmek.py new file mode 100644 index 000000000000..f60a9785ac40 --- /dev/null +++ b/bigquery/samples/create_table_cmek.py @@ -0,0 +1,44 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def create_table_cmek(client, to_delete): + + # [START bigquery_create_table_cmek] + dataset_id = "create_table_cmek_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + client.create_dataset(dataset) + to_delete.append(dataset) + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'my_dataset' + + table_ref = client.dataset(dataset_id).table("my_table") + table = bigquery.Table(table_ref) + + # Set the encryption key to use for the table. + # TODO: Replace this key with a key you have created in Cloud KMS. + kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format( + "cloud-samples-tests", "us-central1", "test", "test" + ) + table.encryption_configuration = bigquery.EncryptionConfiguration( + kms_key_name=kms_key_name + ) + + table = client.create_table(table) # API request + + assert table.encryption_configuration.kms_key_name == kms_key_name + + # [END bigquery_create_table_cmek] \ No newline at end of file diff --git a/bigquery/samples/create_table_nested_repeated_schema.py b/bigquery/samples/create_table_nested_repeated_schema.py new file mode 100644 index 000000000000..5e891c8bcc4a --- /dev/null +++ b/bigquery/samples/create_table_nested_repeated_schema.py @@ -0,0 +1,54 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def create_table_nested_repeated_schema(client, to_delete): + + # [START bigquery_nested_repeated_schema] + dataset_id = "create_table_nested_repeated_{}".format(_millis()) + dataset_ref = client.dataset(dataset_id) + dataset = bigquery.Dataset(dataset_ref) + client.create_dataset(dataset) + to_delete.append(dataset) + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_ref = client.dataset('my_dataset') + + schema = [ + bigquery.SchemaField("id", "STRING", mode="NULLABLE"), + bigquery.SchemaField("first_name", "STRING", mode="NULLABLE"), + bigquery.SchemaField("last_name", "STRING", mode="NULLABLE"), + bigquery.SchemaField("dob", "DATE", mode="NULLABLE"), + bigquery.SchemaField( + "addresses", + "RECORD", + mode="REPEATED", + fields=[ + bigquery.SchemaField("status", "STRING", mode="NULLABLE"), + bigquery.SchemaField("address", "STRING", mode="NULLABLE"), + bigquery.SchemaField("city", "STRING", mode="NULLABLE"), + bigquery.SchemaField("state", "STRING", mode="NULLABLE"), + bigquery.SchemaField("zip", "STRING", mode="NULLABLE"), + bigquery.SchemaField("numberOfYears", "STRING", mode="NULLABLE"), + ], + ), + ] + table_ref = dataset_ref.table("my_table") + table = bigquery.Table(table_ref, schema=schema) + table = client.create_table(table) # API request + + print("Created table {}".format(table.full_table_id)) + + # [END bigquery_nested_repeated_schema] \ No newline at end of file diff --git a/bigquery/samples/dataset_exists.py b/bigquery/samples/dataset_exists.py new file mode 100644 index 000000000000..ed09173ca89a --- /dev/null +++ b/bigquery/samples/dataset_exists.py @@ -0,0 +1,29 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def dataset_exists(client, to_delete): + + # [START bigquery_dataset_exists] + """Determine if a dataset exists.""" + DATASET_ID = "get_table_dataset_{}".format(_millis()) + dataset_ref = client.dataset(DATASET_ID) + dataset = bigquery.Dataset(dataset_ref) + dataset = client.create_dataset(dataset) + to_delete.append(dataset) + + assert dataset_exists(client, dataset_ref) + assert not dataset_exists(client, client.dataset("i_dont_exist")) + + # [END bigquery_dataset_exists] \ No newline at end of file diff --git a/bigquery/samples/ddl_create_view.py b/bigquery/samples/ddl_create_view.py new file mode 100644 index 000000000000..70d5324bcb24 --- /dev/null +++ b/bigquery/samples/ddl_create_view.py @@ -0,0 +1,72 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def ddl_create_view(client, to_delete, capsys): + + # [START bigquery_ddl_create_view] + """Create a view via a DDL query.""" + project = client.project + dataset_id = "ddl_view_{}".format(_millis()) + table_id = "new_view" + dataset = bigquery.Dataset(client.dataset(dataset_id)) + client.create_dataset(dataset) + to_delete.append(dataset) + + # from google.cloud import bigquery + # project = 'my-project' + # dataset_id = 'my_dataset' + # table_id = 'new_view' + # client = bigquery.Client(project=project) + + sql = """ + CREATE VIEW `{}.{}.{}` + OPTIONS( + expiration_timestamp=TIMESTAMP_ADD( + CURRENT_TIMESTAMP(), INTERVAL 48 HOUR), + friendly_name="new_view", + description="a view that expires in 2 days", + labels=[("org_unit", "development")] + ) + AS SELECT name, state, year, number + FROM `bigquery-public-data.usa_names.usa_1910_current` + WHERE state LIKE 'W%' + """.format( + project, dataset_id, table_id + ) + + job = client.query(sql) # API request. + job.result() # Waits for the query to finish. + + print( + 'Created new view "{}.{}.{}".'.format( + job.destination.project, + job.destination.dataset_id, + job.destination.table_id, + ) + ) + + out, _ = capsys.readouterr() + assert 'Created new view "{}.{}.{}".'.format(project, dataset_id, table_id) in out + + # Test that listing query result rows succeeds so that generic query + # processing tools work with DDL statements. + rows = list(job) + assert len(rows) == 0 + + if pandas is not None: + df = job.to_dataframe() + assert len(df) == 0 + + # [END bigquery_ddl_create_view] \ No newline at end of file diff --git a/bigquery/samples/extract_table.py b/bigquery/samples/extract_table.py new file mode 100644 index 000000000000..6d97db98dbfc --- /dev/null +++ b/bigquery/samples/extract_table.py @@ -0,0 +1,52 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def extract_table(client, to_delete): + + # [START bigquery_extract_table] + bucket_name = "extract_shakespeare_{}".format(_millis()) + storage_client = storage.Client() + bucket = retry_storage_errors(storage_client.create_bucket)(bucket_name) + to_delete.append(bucket) + + # from google.cloud import bigquery + # client = bigquery.Client() + # bucket_name = 'my-bucket' + project = "bigquery-public-data" + dataset_id = "samples" + table_id = "shakespeare" + + destination_uri = "gs://{}/{}".format(bucket_name, "shakespeare.csv") + dataset_ref = client.dataset(dataset_id, project=project) + table_ref = dataset_ref.table(table_id) + + extract_job = client.extract_table( + table_ref, + destination_uri, + # Location must match that of the source table. + location="US", + ) # API request + extract_job.result() # Waits for job to complete. + + print( + "Exported {}:{}.{} to {}".format(project, dataset_id, table_id, destination_uri) + ) + + blob = retry_storage_errors(bucket.get_blob)("shakespeare.csv") + assert blob.exists + assert blob.size > 0 + to_delete.insert(0, blob) + + # [END bigquery_extract_table] \ No newline at end of file diff --git a/bigquery/samples/extract_table_compressed.py b/bigquery/samples/extract_table_compressed.py new file mode 100644 index 000000000000..efb916757559 --- /dev/null +++ b/bigquery/samples/extract_table_compressed.py @@ -0,0 +1,48 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def extract_table_compressed(client, to_delete): + + # [START bigquery_extract_table_compressed] + bucket_name = "extract_shakespeare_compress_{}".format(_millis()) + storage_client = storage.Client() + bucket = retry_storage_errors(storage_client.create_bucket)(bucket_name) + to_delete.append(bucket) + + # from google.cloud import bigquery + # client = bigquery.Client() + # bucket_name = 'my-bucket' + + destination_uri = "gs://{}/{}".format(bucket_name, "shakespeare.csv.gz") + dataset_ref = client.dataset("samples", project="bigquery-public-data") + table_ref = dataset_ref.table("shakespeare") + job_config = bigquery.job.ExtractJobConfig() + job_config.compression = bigquery.Compression.GZIP + + extract_job = client.extract_table( + table_ref, + destination_uri, + # Location must match that of the source table. + location="US", + job_config=job_config, + ) # API request + extract_job.result() # Waits for job to complete. + + blob = retry_storage_errors(bucket.get_blob)("shakespeare.csv.gz") + assert blob.exists + assert blob.size > 0 + to_delete.insert(0, blob) + + # [END bigquery_extract_table_compressed] \ No newline at end of file diff --git a/bigquery/samples/extract_table_json.py b/bigquery/samples/extract_table_json.py new file mode 100644 index 000000000000..b9fea870f4dd --- /dev/null +++ b/bigquery/samples/extract_table_json.py @@ -0,0 +1,48 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def extract_table_json(client, to_delete): + + # [START bigquery_extract_table_json] + bucket_name = "extract_shakespeare_json_{}".format(_millis()) + storage_client = storage.Client() + bucket = retry_storage_errors(storage_client.create_bucket)(bucket_name) + to_delete.append(bucket) + + # from google.cloud import bigquery + # client = bigquery.Client() + # bucket_name = 'my-bucket' + + destination_uri = "gs://{}/{}".format(bucket_name, "shakespeare.json") + dataset_ref = client.dataset("samples", project="bigquery-public-data") + table_ref = dataset_ref.table("shakespeare") + job_config = bigquery.job.ExtractJobConfig() + job_config.destination_format = bigquery.DestinationFormat.NEWLINE_DELIMITED_JSON + + extract_job = client.extract_table( + table_ref, + destination_uri, + job_config=job_config, + # Location must match that of the source table. + location="US", + ) # API request + extract_job.result() # Waits for job to complete. + + blob = retry_storage_errors(bucket.get_blob)("shakespeare.json") + assert blob.exists + assert blob.size > 0 + to_delete.insert(0, blob) + + # [END bigquery_extract_table_json] \ No newline at end of file diff --git a/bigquery/samples/list_datasets_by_label.py b/bigquery/samples/list_datasets_by_label.py new file mode 100644 index 000000000000..ce7636f9f0c7 --- /dev/null +++ b/bigquery/samples/list_datasets_by_label.py @@ -0,0 +1,42 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def list_datasets_by_label(client, to_delete): + + # [START bigquery_list_datasets_by_label] + dataset_id = "list_datasets_by_label_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + dataset.labels = {"color": "green"} + dataset = client.create_dataset(dataset) # API request + to_delete.append(dataset) + + # from google.cloud import bigquery + # client = bigquery.Client() + + # The following label filter example will find datasets with an + # arbitrary 'color' label set to 'green' + label_filter = "labels.color:green" + datasets = list(client.list_datasets(filter=label_filter)) + + if datasets: + print("Datasets filtered by {}:".format(label_filter)) + for dataset in datasets: # API request(s) + print("\t{}".format(dataset.dataset_id)) + else: + print("No datasets found with this filter.") + found = set([dataset.dataset_id for dataset in datasets]) + assert dataset_id in found + + # [END bigquery_list_datasets_by_label] \ No newline at end of file diff --git a/bigquery/samples/list_rows_as_dataframe.py b/bigquery/samples/list_rows_as_dataframe.py new file mode 100644 index 000000000000..24113376f5ca --- /dev/null +++ b/bigquery/samples/list_rows_as_dataframe.py @@ -0,0 +1,31 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def list_rows_as_dataframe(client): + + # [START bigquery_list_rows_dataframe] + # from google.cloud import bigquery + # client = bigquery.Client() + + dataset_ref = client.dataset("samples", project="bigquery-public-data") + table_ref = dataset_ref.table("shakespeare") + table = client.get_table(table_ref) + + df = client.list_rows(table).to_dataframe() + assert isinstance(df, pandas.DataFrame) + assert len(list(df)) == len(table.schema) # verify the number of columns + assert len(df) == table.num_rows # verify the number of rows + + # [END bigquery_list_rows_dataframe] \ No newline at end of file diff --git a/bigquery/samples/load_and_query_partitioned_table.py b/bigquery/samples/load_and_query_partitioned_table.py new file mode 100644 index 000000000000..881cf3ce92dd --- /dev/null +++ b/bigquery/samples/load_and_query_partitioned_table.py @@ -0,0 +1,87 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def load_and_query_partitioned_table(client, to_delete): + + # [START bigquery_query_partitioned_table] + dataset_id = "load_partitioned_table_dataset_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + client.create_dataset(dataset) + to_delete.append(dataset) + + # [START bigquery_load_table_partitioned] + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'my_dataset' + table_id = "us_states_by_date" + + dataset_ref = client.dataset(dataset_id) + job_config = bigquery.LoadJobConfig() + job_config.schema = [ + bigquery.SchemaField("name", "STRING"), + bigquery.SchemaField("post_abbr", "STRING"), + bigquery.SchemaField("date", "DATE"), + ] + job_config.skip_leading_rows = 1 + job_config.time_partitioning = bigquery.TimePartitioning( + type_=bigquery.TimePartitioningType.DAY, + field="date", # name of column to use for partitioning + expiration_ms=7776000000, + ) # 90 days + uri = "gs://cloud-samples-data/bigquery/us-states/us-states-by-date.csv" + + load_job = client.load_table_from_uri( + uri, dataset_ref.table(table_id), job_config=job_config + ) # API request + + assert load_job.job_type == "load" + + load_job.result() # Waits for table load to complete. + + table = client.get_table(dataset_ref.table(table_id)) + print("Loaded {} rows to table {}".format(table.num_rows, table_id)) + # [END bigquery_load_table_partitioned] + assert table.num_rows == 50 + + project_id = client.project + + import datetime + + # from google.cloud import bigquery + # client = bigquery.Client() + # project_id = 'my-project' + # dataset_id = 'my_dataset' + table_id = "us_states_by_date" + + sql_template = """ + SELECT * + FROM `{}.{}.{}` + WHERE date BETWEEN @start_date AND @end_date + """ + sql = sql_template.format(project_id, dataset_id, table_id) + job_config = bigquery.QueryJobConfig() + job_config.query_parameters = [ + bigquery.ScalarQueryParameter("start_date", "DATE", datetime.date(1800, 1, 1)), + bigquery.ScalarQueryParameter("end_date", "DATE", datetime.date(1899, 12, 31)), + ] + + # API request + query_job = client.query(sql, job_config=job_config) + + rows = list(query_job) + print("{} states were admitted to the US in the 1800s".format(len(rows))) + assert len(rows) == 29 + + # [END bigquery_query_partitioned_table] \ No newline at end of file diff --git a/bigquery/samples/load_table_add_column.py b/bigquery/samples/load_table_add_column.py new file mode 100644 index 000000000000..e4f945e5a6f9 --- /dev/null +++ b/bigquery/samples/load_table_add_column.py @@ -0,0 +1,83 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def load_table_add_column(client, to_delete): + + # [START bigquery_add_column_load_append] + dataset_id = "load_table_add_column_{}".format(_millis()) + dataset_ref = client.dataset(dataset_id) + dataset = bigquery.Dataset(dataset_ref) + dataset.location = "US" + dataset = client.create_dataset(dataset) + to_delete.append(dataset) + + snippets_dir = os.path.abspath(os.path.dirname(__file__)) + filepath = os.path.join( + snippets_dir, "..", "..", "bigquery", "tests", "data", "people.csv" + ) + table_ref = dataset_ref.table("my_table") + old_schema = [bigquery.SchemaField("full_name", "STRING", mode="REQUIRED")] + table = client.create_table(bigquery.Table(table_ref, schema=old_schema)) + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_ref = client.dataset('my_dataset') + # filepath = 'path/to/your_file.csv' + + # Retrieves the destination table and checks the length of the schema + table_id = "my_table" + table_ref = dataset_ref.table(table_id) + table = client.get_table(table_ref) + print("Table {} contains {} columns.".format(table_id, len(table.schema))) + + # Configures the load job to append the data to the destination table, + # allowing field addition + job_config = bigquery.LoadJobConfig() + job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND + job_config.schema_update_options = [ + bigquery.SchemaUpdateOption.ALLOW_FIELD_ADDITION + ] + # In this example, the existing table contains only the 'full_name' column. + # 'REQUIRED' fields cannot be added to an existing schema, so the + # additional column must be 'NULLABLE'. + job_config.schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="NULLABLE"), + ] + job_config.source_format = bigquery.SourceFormat.CSV + job_config.skip_leading_rows = 1 + + with open(filepath, "rb") as source_file: + job = client.load_table_from_file( + source_file, + table_ref, + location="US", # Must match the destination dataset location. + job_config=job_config, + ) # API request + + job.result() # Waits for table load to complete. + print( + "Loaded {} rows into {}:{}.".format( + job.output_rows, dataset_id, table_ref.table_id + ) + ) + + # Checks the updated length of the schema + table = client.get_table(table) + print("Table {} now contains {} columns.".format(table_id, len(table.schema))) + assert len(table.schema) == 2 + assert table.num_rows > 0 + + # [END bigquery_add_column_load_append] \ No newline at end of file diff --git a/bigquery/samples/load_table_from_dataframe.py b/bigquery/samples/load_table_from_dataframe.py new file mode 100644 index 000000000000..9ac11fb7bfd5 --- /dev/null +++ b/bigquery/samples/load_table_from_dataframe.py @@ -0,0 +1,60 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def load_table_from_dataframe(client, to_delete, parquet_engine): + + # [START bigquery_load_table_dataframe] + if parquet_engine == "pyarrow" and pyarrow is None: + pytest.skip("Requires `pyarrow`") + if parquet_engine == "fastparquet" and fastparquet is None: + pytest.skip("Requires `fastparquet`") + + pandas.set_option("io.parquet.engine", parquet_engine) + + dataset_id = "load_table_from_dataframe_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + client.create_dataset(dataset) + to_delete.append(dataset) + + # from google.cloud import bigquery + # import pandas + # client = bigquery.Client() + # dataset_id = 'my_dataset' + + dataset_ref = client.dataset(dataset_id) + table_ref = dataset_ref.table("monty_python") + records = [ + {"title": "The Meaning of Life", "release_year": 1983}, + {"title": "Monty Python and the Holy Grail", "release_year": 1975}, + {"title": "Life of Brian", "release_year": 1979}, + {"title": "And Now for Something Completely Different", "release_year": 1971}, + ] + # Optionally set explicit indices. + # If indices are not specified, a column will be created for the default + # indices created by pandas. + index = ["Q24980", "Q25043", "Q24953", "Q16403"] + dataframe = pandas.DataFrame(records, index=pandas.Index(index, name="wikidata_id")) + + job = client.load_table_from_dataframe(dataframe, table_ref, location="US") + + job.result() # Waits for table load to complete. + + assert job.state == "DONE" + table = client.get_table(table_ref) + assert table.num_rows == 4 + column_names = [field.name for field in table.schema] + assert sorted(column_names) == ["release_year", "title", "wikidata_id"] + + # [END bigquery_load_table_dataframe] \ No newline at end of file diff --git a/bigquery/samples/load_table_from_file.py b/bigquery/samples/load_table_from_file.py new file mode 100644 index 000000000000..87e933cc0ffe --- /dev/null +++ b/bigquery/samples/load_table_from_file.py @@ -0,0 +1,61 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def load_table_from_file(client, to_delete): + + # [START bigquery_load_from_file] + """Upload table data from a CSV file.""" + dataset_id = "load_table_from_file_dataset_{}".format(_millis()) + table_id = "load_table_from_file_table_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + dataset.location = "US" + client.create_dataset(dataset) + to_delete.append(dataset) + snippets_dir = os.path.abspath(os.path.dirname(__file__)) + filename = os.path.join( + snippets_dir, "..", "..", "bigquery", "tests", "data", "people.csv" + ) + + # from google.cloud import bigquery + # client = bigquery.Client() + # filename = '/path/to/file.csv' + # dataset_id = 'my_dataset' + # table_id = 'my_table' + + dataset_ref = client.dataset(dataset_id) + table_ref = dataset_ref.table(table_id) + job_config = bigquery.LoadJobConfig() + job_config.source_format = bigquery.SourceFormat.CSV + job_config.skip_leading_rows = 1 + job_config.autodetect = True + + with open(filename, "rb") as source_file: + job = client.load_table_from_file(source_file, table_ref, job_config=job_config) + + job.result() # Waits for table load to complete. + + print("Loaded {} rows into {}:{}.".format(job.output_rows, dataset_id, table_id)) + + table = client.get_table(table_ref) + rows = list(client.list_rows(table)) # API request + + assert len(rows) == 2 + # Order is not preserved, so compare individually + row1 = bigquery.Row(("Wylma Phlyntstone", 29), {"full_name": 0, "age": 1}) + assert row1 in rows + row2 = bigquery.Row(("Phred Phlyntstone", 32), {"full_name": 0, "age": 1}) + assert row2 in rows + + # [END bigquery_load_from_file] \ No newline at end of file diff --git a/bigquery/samples/load_table_from_uri_autodetect.py b/bigquery/samples/load_table_from_uri_autodetect.py new file mode 100644 index 000000000000..d99bf1154a39 --- /dev/null +++ b/bigquery/samples/load_table_from_uri_autodetect.py @@ -0,0 +1,80 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def load_table_from_uri_autodetect(client, to_delete, capsys): + + # [START bigquery_load_table_gcs_json_autodetect] + """Load table from a GCS URI using various formats and auto-detected schema + + Each file format has its own tested load from URI sample. Because most of + the code is common for autodetect, append, and truncate, this sample + includes snippets for all supported formats but only calls a single load + job. + + This code snippet is made up of shared code, then format-specific code, + followed by more shared code. Note that only the last format in the + format-specific code section will be tested in this test. + """ + dataset_id = "load_table_from_uri_auto_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + client.create_dataset(dataset) + to_delete.append(dataset) + + # Shared code + # [START bigquery_load_table_gcs_csv_autodetect] + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'my_dataset' + + dataset_ref = client.dataset(dataset_id) + job_config = bigquery.LoadJobConfig() + job_config.autodetect = True + # [END bigquery_load_table_gcs_csv_autodetect] + + # Format-specific code + # [START bigquery_load_table_gcs_csv_autodetect] + job_config.skip_leading_rows = 1 + # The source format defaults to CSV, so the line below is optional. + job_config.source_format = bigquery.SourceFormat.CSV + uri = "gs://cloud-samples-data/bigquery/us-states/us-states.csv" + # [END bigquery_load_table_gcs_csv_autodetect] + # unset csv-specific attribute + del job_config._properties["load"]["skipLeadingRows"] + + # [START bigquery_load_table_gcs_json_autodetect] + job_config.source_format = bigquery.SourceFormat.NEWLINE_DELIMITED_JSON + uri = "gs://cloud-samples-data/bigquery/us-states/us-states.json" + # [END bigquery_load_table_gcs_json_autodetect] + + # Shared code + # [START bigquery_load_table_gcs_csv_autodetect] + # [START bigquery_load_table_gcs_json_autodetect] + load_job = client.load_table_from_uri( + uri, dataset_ref.table("us_states"), job_config=job_config + ) # API request + print("Starting job {}".format(load_job.job_id)) + + load_job.result() # Waits for table load to complete. + print("Job finished.") + + destination_table = client.get_table(dataset_ref.table("us_states")) + print("Loaded {} rows.".format(destination_table.num_rows)) + # [END bigquery_load_table_gcs_csv_autodetect] + # [END bigquery_load_table_gcs_json_autodetect] + + out, _ = capsys.readouterr() + assert "Loaded 50 rows." in out + + # [END bigquery_load_table_gcs_json_autodetect] \ No newline at end of file diff --git a/bigquery/samples/load_table_from_uri_avro.py b/bigquery/samples/load_table_from_uri_avro.py new file mode 100644 index 000000000000..987a440f1c6e --- /dev/null +++ b/bigquery/samples/load_table_from_uri_avro.py @@ -0,0 +1,47 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def load_table_from_uri_avro(client, to_delete, capsys): + + # [START bigquery_load_table_gcs_avro] + dataset_id = "load_table_from_uri_avro_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + client.create_dataset(dataset) + to_delete.append(dataset) + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'my_dataset' + + dataset_ref = client.dataset(dataset_id) + job_config = bigquery.LoadJobConfig() + job_config.source_format = bigquery.SourceFormat.AVRO + uri = "gs://cloud-samples-data/bigquery/us-states/us-states.avro" + + load_job = client.load_table_from_uri( + uri, dataset_ref.table("us_states"), job_config=job_config + ) # API request + print("Starting job {}".format(load_job.job_id)) + + load_job.result() # Waits for table load to complete. + print("Job finished.") + + destination_table = client.get_table(dataset_ref.table("us_states")) + print("Loaded {} rows.".format(destination_table.num_rows)) + + out, _ = capsys.readouterr() + assert "Loaded 50 rows." in out + + # [END bigquery_load_table_gcs_avro] \ No newline at end of file diff --git a/bigquery/samples/load_table_from_uri_cmek.py b/bigquery/samples/load_table_from_uri_cmek.py new file mode 100644 index 000000000000..2c44bcbd4bc4 --- /dev/null +++ b/bigquery/samples/load_table_from_uri_cmek.py @@ -0,0 +1,58 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def load_table_from_uri_cmek(client, to_delete): + + # [START bigquery_load_table_gcs_json_cmek] + dataset_id = "load_table_from_uri_cmek_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + dataset.location = "US" + client.create_dataset(dataset) + to_delete.append(dataset) + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'my_dataset' + + dataset_ref = client.dataset(dataset_id) + job_config = bigquery.LoadJobConfig() + job_config.autodetect = True + job_config.source_format = bigquery.SourceFormat.NEWLINE_DELIMITED_JSON + + # Set the encryption key to use for the destination. + # TODO: Replace this key with a key you have created in KMS. + kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format( + "cloud-samples-tests", "us-central1", "test", "test" + ) + encryption_config = bigquery.EncryptionConfiguration(kms_key_name=kms_key_name) + job_config.destination_encryption_configuration = encryption_config + uri = "gs://cloud-samples-data/bigquery/us-states/us-states.json" + + load_job = client.load_table_from_uri( + uri, + dataset_ref.table("us_states"), + location="US", # Location must match that of the destination dataset. + job_config=job_config, + ) # API request + + assert load_job.job_type == "load" + + load_job.result() # Waits for table load to complete. + + assert load_job.state == "DONE" + table = client.get_table(dataset_ref.table("us_states")) + assert table.encryption_configuration.kms_key_name == kms_key_name + + # [END bigquery_load_table_gcs_json_cmek] \ No newline at end of file diff --git a/bigquery/samples/load_table_from_uri_csv.py b/bigquery/samples/load_table_from_uri_csv.py new file mode 100644 index 000000000000..31707fb16902 --- /dev/null +++ b/bigquery/samples/load_table_from_uri_csv.py @@ -0,0 +1,53 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def load_table_from_uri_csv(client, to_delete, capsys): + + # [START bigquery_load_table_gcs_csv] + dataset_id = "load_table_from_uri_csv_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + client.create_dataset(dataset) + to_delete.append(dataset) + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'my_dataset' + + dataset_ref = client.dataset(dataset_id) + job_config = bigquery.LoadJobConfig() + job_config.schema = [ + bigquery.SchemaField("name", "STRING"), + bigquery.SchemaField("post_abbr", "STRING"), + ] + job_config.skip_leading_rows = 1 + # The source format defaults to CSV, so the line below is optional. + job_config.source_format = bigquery.SourceFormat.CSV + uri = "gs://cloud-samples-data/bigquery/us-states/us-states.csv" + + load_job = client.load_table_from_uri( + uri, dataset_ref.table("us_states"), job_config=job_config + ) # API request + print("Starting job {}".format(load_job.job_id)) + + load_job.result() # Waits for table load to complete. + print("Job finished.") + + destination_table = client.get_table(dataset_ref.table("us_states")) + print("Loaded {} rows.".format(destination_table.num_rows)) + + out, _ = capsys.readouterr() + assert "Loaded 50 rows." in out + + # [END bigquery_load_table_gcs_csv] \ No newline at end of file diff --git a/bigquery/samples/load_table_from_uri_json.py b/bigquery/samples/load_table_from_uri_json.py new file mode 100644 index 000000000000..c12f06b1994c --- /dev/null +++ b/bigquery/samples/load_table_from_uri_json.py @@ -0,0 +1,55 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def load_table_from_uri_json(client, to_delete, capsys): + + # [START bigquery_load_table_gcs_json] + dataset_id = "load_table_from_uri_json_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + dataset.location = "US" + client.create_dataset(dataset) + to_delete.append(dataset) + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'my_dataset' + + dataset_ref = client.dataset(dataset_id) + job_config = bigquery.LoadJobConfig() + job_config.schema = [ + bigquery.SchemaField("name", "STRING"), + bigquery.SchemaField("post_abbr", "STRING"), + ] + job_config.source_format = bigquery.SourceFormat.NEWLINE_DELIMITED_JSON + uri = "gs://cloud-samples-data/bigquery/us-states/us-states.json" + + load_job = client.load_table_from_uri( + uri, + dataset_ref.table("us_states"), + location="US", # Location must match that of the destination dataset. + job_config=job_config, + ) # API request + print("Starting job {}".format(load_job.job_id)) + + load_job.result() # Waits for table load to complete. + print("Job finished.") + + destination_table = client.get_table(dataset_ref.table("us_states")) + print("Loaded {} rows.".format(destination_table.num_rows)) + + out, _ = capsys.readouterr() + assert "Loaded 50 rows." in out + + # [END bigquery_load_table_gcs_json] \ No newline at end of file diff --git a/bigquery/samples/load_table_from_uri_orc.py b/bigquery/samples/load_table_from_uri_orc.py new file mode 100644 index 000000000000..94e954aebfbb --- /dev/null +++ b/bigquery/samples/load_table_from_uri_orc.py @@ -0,0 +1,47 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def load_table_from_uri_orc(client, to_delete, capsys): + + # [START bigquery_load_table_gcs_orc] + dataset_id = "load_table_from_uri_orc_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + client.create_dataset(dataset) + to_delete.append(dataset) + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'my_dataset' + + dataset_ref = client.dataset(dataset_id) + job_config = bigquery.LoadJobConfig() + job_config.source_format = bigquery.SourceFormat.ORC + uri = "gs://cloud-samples-data/bigquery/us-states/us-states.orc" + + load_job = client.load_table_from_uri( + uri, dataset_ref.table("us_states"), job_config=job_config + ) # API request + print("Starting job {}".format(load_job.job_id)) + + load_job.result() # Waits for table load to complete. + print("Job finished.") + + destination_table = client.get_table(dataset_ref.table("us_states")) + print("Loaded {} rows.".format(destination_table.num_rows)) + + out, _ = capsys.readouterr() + assert "Loaded 50 rows." in out + + # [END bigquery_load_table_gcs_orc] \ No newline at end of file diff --git a/bigquery/samples/load_table_from_uri_parquet.py b/bigquery/samples/load_table_from_uri_parquet.py new file mode 100644 index 000000000000..45ad49cf7eba --- /dev/null +++ b/bigquery/samples/load_table_from_uri_parquet.py @@ -0,0 +1,47 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def load_table_from_uri_parquet(client, to_delete, capsys): + + # [START bigquery_load_table_gcs_parquet] + dataset_id = "load_table_from_uri_parquet_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + client.create_dataset(dataset) + to_delete.append(dataset) + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'my_dataset' + + dataset_ref = client.dataset(dataset_id) + job_config = bigquery.LoadJobConfig() + job_config.source_format = bigquery.SourceFormat.PARQUET + uri = "gs://cloud-samples-data/bigquery/us-states/us-states.parquet" + + load_job = client.load_table_from_uri( + uri, dataset_ref.table("us_states"), job_config=job_config + ) # API request + print("Starting job {}".format(load_job.job_id)) + + load_job.result() # Waits for table load to complete. + print("Job finished.") + + destination_table = client.get_table(dataset_ref.table("us_states")) + print("Loaded {} rows.".format(destination_table.num_rows)) + + out, _ = capsys.readouterr() + assert "Loaded 50 rows." in out + + # [END bigquery_load_table_gcs_parquet] \ No newline at end of file diff --git a/bigquery/samples/load_table_from_uri_truncate.py b/bigquery/samples/load_table_from_uri_truncate.py new file mode 100644 index 000000000000..0f4ce2210569 --- /dev/null +++ b/bigquery/samples/load_table_from_uri_truncate.py @@ -0,0 +1,117 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def load_table_from_uri_truncate(client, to_delete, capsys): + + # [START bigquery_load_table_gcs_orc_truncate] + """Replaces table data with data from a GCS URI using various formats + + Each file format has its own tested load from URI sample. Because most of + the code is common for autodetect, append, and truncate, this sample + includes snippets for all supported formats but only calls a single load + job. + + This code snippet is made up of shared code, then format-specific code, + followed by more shared code. Note that only the last format in the + format-specific code section will be tested in this test. + """ + dataset_id = "load_table_from_uri_trunc_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + client.create_dataset(dataset) + to_delete.append(dataset) + + job_config = bigquery.LoadJobConfig() + job_config.schema = [ + bigquery.SchemaField("name", "STRING"), + bigquery.SchemaField("post_abbr", "STRING"), + ] + table_ref = dataset.table("us_states") + body = six.BytesIO(b"Washington,WA") + client.load_table_from_file(body, table_ref, job_config=job_config).result() + previous_rows = client.get_table(table_ref).num_rows + assert previous_rows > 0 + + # Shared code + # [START bigquery_load_table_gcs_avro_truncate] + # [START bigquery_load_table_gcs_csv_truncate] + # [START bigquery_load_table_gcs_json_truncate] + # [START bigquery_load_table_gcs_parquet_truncate] + # from google.cloud import bigquery + # client = bigquery.Client() + # table_ref = client.dataset('my_dataset').table('existing_table') + + job_config = bigquery.LoadJobConfig() + job_config.write_disposition = bigquery.WriteDisposition.WRITE_TRUNCATE + # [END bigquery_load_table_gcs_avro_truncate] + # [END bigquery_load_table_gcs_csv_truncate] + # [END bigquery_load_table_gcs_json_truncate] + # [END bigquery_load_table_gcs_parquet_truncate] + + # Format-specific code + # [START bigquery_load_table_gcs_avro_truncate] + job_config.source_format = bigquery.SourceFormat.AVRO + uri = "gs://cloud-samples-data/bigquery/us-states/us-states.avro" + # [END bigquery_load_table_gcs_avro_truncate] + + # [START bigquery_load_table_gcs_csv_truncate] + job_config.skip_leading_rows = 1 + # The source format defaults to CSV, so the line below is optional. + job_config.source_format = bigquery.SourceFormat.CSV + uri = "gs://cloud-samples-data/bigquery/us-states/us-states.csv" + # [END bigquery_load_table_gcs_csv_truncate] + # unset csv-specific attribute + del job_config._properties["load"]["skipLeadingRows"] + + # [START bigquery_load_table_gcs_json_truncate] + job_config.source_format = bigquery.SourceFormat.NEWLINE_DELIMITED_JSON + uri = "gs://cloud-samples-data/bigquery/us-states/us-states.json" + # [END bigquery_load_table_gcs_json_truncate] + + # [START bigquery_load_table_gcs_parquet_truncate] + job_config.source_format = bigquery.SourceFormat.PARQUET + uri = "gs://cloud-samples-data/bigquery/us-states/us-states.parquet" + # [END bigquery_load_table_gcs_parquet_truncate] + + # [START bigquery_load_table_gcs_orc_truncate] + job_config.source_format = bigquery.SourceFormat.ORC + uri = "gs://cloud-samples-data/bigquery/us-states/us-states.orc" + # [END bigquery_load_table_gcs_orc_truncate] + + # Shared code + # [START bigquery_load_table_gcs_avro_truncate] + # [START bigquery_load_table_gcs_csv_truncate] + # [START bigquery_load_table_gcs_json_truncate] + # [START bigquery_load_table_gcs_parquet_truncate] + # [START bigquery_load_table_gcs_orc_truncate] + load_job = client.load_table_from_uri( + uri, table_ref, job_config=job_config + ) # API request + print("Starting job {}".format(load_job.job_id)) + + load_job.result() # Waits for table load to complete. + print("Job finished.") + + destination_table = client.get_table(table_ref) + print("Loaded {} rows.".format(destination_table.num_rows)) + # [END bigquery_load_table_gcs_avro_truncate] + # [END bigquery_load_table_gcs_csv_truncate] + # [END bigquery_load_table_gcs_json_truncate] + # [END bigquery_load_table_gcs_parquet_truncate] + # [END bigquery_load_table_gcs_orc_truncate] + + out, _ = capsys.readouterr() + assert "Loaded 50 rows." in out + + # [END bigquery_load_table_gcs_orc_truncate] \ No newline at end of file diff --git a/bigquery/samples/load_table_relax_column.py b/bigquery/samples/load_table_relax_column.py new file mode 100644 index 000000000000..b57a4c42ad03 --- /dev/null +++ b/bigquery/samples/load_table_relax_column.py @@ -0,0 +1,92 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def load_table_relax_column(client, to_delete): + + # [START bigquery_relax_column_load_append] + dataset_id = "load_table_relax_column_{}".format(_millis()) + dataset_ref = client.dataset(dataset_id) + dataset = bigquery.Dataset(dataset_ref) + dataset.location = "US" + dataset = client.create_dataset(dataset) + to_delete.append(dataset) + + snippets_dir = os.path.abspath(os.path.dirname(__file__)) + filepath = os.path.join( + snippets_dir, "..", "..", "bigquery", "tests", "data", "people.csv" + ) + table_ref = dataset_ref.table("my_table") + old_schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + bigquery.SchemaField("favorite_color", "STRING", mode="REQUIRED"), + ] + table = client.create_table(bigquery.Table(table_ref, schema=old_schema)) + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_ref = client.dataset('my_dataset') + # filepath = 'path/to/your_file.csv' + + # Retrieves the destination table and checks the number of required fields + table_id = "my_table" + table_ref = dataset_ref.table(table_id) + table = client.get_table(table_ref) + original_required_fields = sum(field.mode == "REQUIRED" for field in table.schema) + # In this example, the existing table has 3 required fields. + print("{} fields in the schema are required.".format(original_required_fields)) + + # Configures the load job to append the data to a destination table, + # allowing field relaxation + job_config = bigquery.LoadJobConfig() + job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND + job_config.schema_update_options = [ + bigquery.SchemaUpdateOption.ALLOW_FIELD_RELAXATION + ] + # In this example, the existing table contains three required fields + # ('full_name', 'age', and 'favorite_color'), while the data to load + # contains only the first two fields. + job_config.schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + ] + job_config.source_format = bigquery.SourceFormat.CSV + job_config.skip_leading_rows = 1 + + with open(filepath, "rb") as source_file: + job = client.load_table_from_file( + source_file, + table_ref, + location="US", # Must match the destination dataset location. + job_config=job_config, + ) # API request + + job.result() # Waits for table load to complete. + print( + "Loaded {} rows into {}:{}.".format( + job.output_rows, dataset_id, table_ref.table_id + ) + ) + + # Checks the updated number of required fields + table = client.get_table(table) + current_required_fields = sum(field.mode == "REQUIRED" for field in table.schema) + print("{} fields in the schema are now required.".format(current_required_fields)) + assert original_required_fields - current_required_fields == 1 + assert len(table.schema) == 3 + assert table.schema[2].mode == "NULLABLE" + assert table.num_rows > 0 + + # [END bigquery_relax_column_load_append] \ No newline at end of file diff --git a/bigquery/samples/manage_dataset_labels.py b/bigquery/samples/manage_dataset_labels.py new file mode 100644 index 000000000000..430985d91746 --- /dev/null +++ b/bigquery/samples/manage_dataset_labels.py @@ -0,0 +1,72 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def manage_dataset_labels(client, to_delete): + + # [START bigquery_delete_label_dataset] + dataset_id = "label_dataset_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + dataset = client.create_dataset(dataset) + to_delete.append(dataset) + + # [START bigquery_label_dataset] + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_ref = client.dataset('my_dataset') + # dataset = client.get_dataset(dataset_ref) # API request + + assert dataset.labels == {} + labels = {"color": "green"} + dataset.labels = labels + + dataset = client.update_dataset(dataset, ["labels"]) # API request + + assert dataset.labels == labels + # [END bigquery_label_dataset] + + # [START bigquery_get_dataset_labels] + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'my_dataset' + + dataset_ref = client.dataset(dataset_id) + dataset = client.get_dataset(dataset_ref) # API request + + # View dataset labels + print("Dataset ID: {}".format(dataset_id)) + print("Labels:") + if dataset.labels: + for label, value in dataset.labels.items(): + print("\t{}: {}".format(label, value)) + else: + print("\tDataset has no labels defined.") + # [END bigquery_get_dataset_labels] + assert dataset.labels == labels + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_ref = client.dataset('my_dataset') + # dataset = client.get_dataset(dataset_ref) # API request + + # This example dataset starts with one label + assert dataset.labels == {"color": "green"} + # To delete a label from a dataset, set its value to None + dataset.labels["color"] = None + + dataset = client.update_dataset(dataset, ["labels"]) # API request + + assert dataset.labels == {} + + # [END bigquery_delete_label_dataset] \ No newline at end of file diff --git a/bigquery/samples/manage_job.py b/bigquery/samples/manage_job.py new file mode 100644 index 000000000000..1cfb04c61c3c --- /dev/null +++ b/bigquery/samples/manage_job.py @@ -0,0 +1,54 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def manage_job(client): + + # [START bigquery_get_job] + sql = """ + SELECT corpus + FROM `bigquery-public-data.samples.shakespeare` + GROUP BY corpus; + """ + location = "us" + job = client.query(sql, location=location) + job_id = job.job_id + + # [START bigquery_cancel_job] + # TODO(developer): Uncomment the lines below and replace with your values. + # from google.cloud import bigquery + # client = bigquery.Client() + # job_id = 'bq-job-123x456-123y123z123c' # replace with your job ID + # location = 'us' # replace with your location + + job = client.cancel_job(job_id, location=location) + # [END bigquery_cancel_job] + + # TODO(developer): Uncomment the lines below and replace with your values. + # from google.cloud import bigquery + # client = bigquery.Client() + # job_id = 'bq-job-123x456-123y123z123c' # replace with your job ID + # location = 'us' # replace with your location + + job = client.get_job(job_id, location=location) # API request + + # Print selected job properties + print("Details for job {} running in {}:".format(job_id, location)) + print( + "\tType: {}\n\tState: {}\n\tCreated: {}".format( + job.job_type, job.state, job.created + ) + ) + + # [END bigquery_get_job] \ No newline at end of file diff --git a/bigquery/samples/manage_table_labels.py b/bigquery/samples/manage_table_labels.py new file mode 100644 index 000000000000..18b115295976 --- /dev/null +++ b/bigquery/samples/manage_table_labels.py @@ -0,0 +1,78 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def manage_table_labels(client, to_delete): + + # [START bigquery_delete_label_table] + dataset_id = "label_table_dataset_{}".format(_millis()) + table_id = "label_table_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + client.create_dataset(dataset) + to_delete.append(dataset) + + table = bigquery.Table(dataset.table(table_id), schema=SCHEMA) + table = client.create_table(table) + + # [START bigquery_label_table] + # from google.cloud import bigquery + # client = bigquery.Client() + # table_ref = client.dataset('my_dataset').table('my_table') + # table = client.get_table(table_ref) # API request + + assert table.labels == {} + labels = {"color": "green"} + table.labels = labels + + table = client.update_table(table, ["labels"]) # API request + + assert table.labels == labels + # [END bigquery_label_table] + + # [START bigquery_get_table_labels] + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'my_dataset' + # table_id = 'my_table' + + dataset_ref = client.dataset(dataset_id) + table_ref = dataset_ref.table(table_id) + table = client.get_table(table_ref) # API Request + + # View table labels + print("Table ID: {}".format(table_id)) + print("Labels:") + if table.labels: + for label, value in table.labels.items(): + print("\t{}: {}".format(label, value)) + else: + print("\tTable has no labels defined.") + # [END bigquery_get_table_labels] + assert table.labels == labels + + # from google.cloud import bigquery + # client = bigquery.Client() + # table_ref = client.dataset('my_dataset').table('my_table') + # table = client.get_table(table_ref) # API request + + # This example table starts with one label + assert table.labels == {"color": "green"} + # To delete a label from a table, set its value to None + table.labels["color"] = None + + table = client.update_table(table, ["labels"]) # API request + + assert table.labels == {} + + # [END bigquery_delete_label_table] \ No newline at end of file diff --git a/bigquery/samples/manage_views.py b/bigquery/samples/manage_views.py new file mode 100644 index 000000000000..f6b63ce0853d --- /dev/null +++ b/bigquery/samples/manage_views.py @@ -0,0 +1,134 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def manage_views(client, to_delete): + + # [START bigquery_grant_view_access] + project = client.project + source_dataset_id = "source_dataset_{}".format(_millis()) + source_dataset_ref = client.dataset(source_dataset_id) + source_dataset = bigquery.Dataset(source_dataset_ref) + source_dataset = client.create_dataset(source_dataset) + to_delete.append(source_dataset) + + job_config = bigquery.LoadJobConfig() + job_config.schema = [ + bigquery.SchemaField("name", "STRING"), + bigquery.SchemaField("post_abbr", "STRING"), + ] + job_config.skip_leading_rows = 1 + uri = "gs://cloud-samples-data/bigquery/us-states/us-states.csv" + source_table_id = "us_states" + load_job = client.load_table_from_uri( + uri, source_dataset.table(source_table_id), job_config=job_config + ) + load_job.result() + + shared_dataset_id = "shared_dataset_{}".format(_millis()) + shared_dataset_ref = client.dataset(shared_dataset_id) + shared_dataset = bigquery.Dataset(shared_dataset_ref) + shared_dataset = client.create_dataset(shared_dataset) + to_delete.append(shared_dataset) + + # [START bigquery_create_view] + # from google.cloud import bigquery + # client = bigquery.Client() + # project = 'my-project' + # source_dataset_id = 'my_source_dataset' + # source_table_id = 'us_states' + # shared_dataset_ref = client.dataset('my_shared_dataset') + + # This example shows how to create a shared view of a source table of + # US States. The source table contains all 50 states, while the view will + # contain only states with names starting with 'W'. + view_ref = shared_dataset_ref.table("my_shared_view") + view = bigquery.Table(view_ref) + sql_template = 'SELECT name, post_abbr FROM `{}.{}.{}` WHERE name LIKE "W%"' + view.view_query = sql_template.format(project, source_dataset_id, source_table_id) + view = client.create_table(view) # API request + + print("Successfully created view at {}".format(view.full_table_id)) + # [END bigquery_create_view] + + # [START bigquery_update_view_query] + # from google.cloud import bigquery + # client = bigquery.Client() + # project = 'my-project' + # source_dataset_id = 'my_source_dataset' + # source_table_id = 'us_states' + # shared_dataset_ref = client.dataset('my_shared_dataset') + + # This example shows how to update a shared view of a source table of + # US States. The view's query will be updated to contain only states with + # names starting with 'M'. + view_ref = shared_dataset_ref.table("my_shared_view") + view = bigquery.Table(view_ref) + sql_template = 'SELECT name, post_abbr FROM `{}.{}.{}` WHERE name LIKE "M%"' + view.view_query = sql_template.format(project, source_dataset_id, source_table_id) + view = client.update_table(view, ["view_query"]) # API request + # [END bigquery_update_view_query] + + # [START bigquery_get_view] + # from google.cloud import bigquery + # client = bigquery.Client() + # shared_dataset_id = 'my_shared_dataset' + + view_ref = client.dataset(shared_dataset_id).table("my_shared_view") + view = client.get_table(view_ref) # API Request + + # Display view properties + print("View at {}".format(view.full_table_id)) + print("View Query:\n{}".format(view.view_query)) + # [END bigquery_get_view] + assert view.view_query is not None + + analyst_group_email = "example-analyst-group@google.com" + # from google.cloud import bigquery + # client = bigquery.Client() + + # Assign access controls to the dataset containing the view + # shared_dataset_id = 'my_shared_dataset' + # analyst_group_email = 'data_analysts@example.com' + shared_dataset = client.get_dataset( + client.dataset(shared_dataset_id) + ) # API request + access_entries = shared_dataset.access_entries + access_entries.append( + bigquery.AccessEntry("READER", "groupByEmail", analyst_group_email) + ) + shared_dataset.access_entries = access_entries + shared_dataset = client.update_dataset( + shared_dataset, ["access_entries"] + ) # API request + + # Authorize the view to access the source dataset + # project = 'my-project' + # source_dataset_id = 'my_source_dataset' + source_dataset = client.get_dataset( + client.dataset(source_dataset_id) + ) # API request + view_reference = { + "projectId": project, + "datasetId": shared_dataset_id, + "tableId": "my_shared_view", + } + access_entries = source_dataset.access_entries + access_entries.append(bigquery.AccessEntry(None, "view", view_reference)) + source_dataset.access_entries = access_entries + source_dataset = client.update_dataset( + source_dataset, ["access_entries"] + ) # API request + + # [END bigquery_grant_view_access] \ No newline at end of file diff --git a/bigquery/samples/query_external_gcs_permanent_table.py b/bigquery/samples/query_external_gcs_permanent_table.py new file mode 100644 index 000000000000..bfc81c62922e --- /dev/null +++ b/bigquery/samples/query_external_gcs_permanent_table.py @@ -0,0 +1,55 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def query_external_gcs_permanent_table(client, to_delete): + + # [START bigquery_query_external_gcs_perm] + dataset_id = "query_external_gcs_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + client.create_dataset(dataset) + to_delete.append(dataset) + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'my_dataset' + + # Configure the external data source + dataset_ref = client.dataset(dataset_id) + table_id = "us_states" + schema = [ + bigquery.SchemaField("name", "STRING"), + bigquery.SchemaField("post_abbr", "STRING"), + ] + table = bigquery.Table(dataset_ref.table(table_id), schema=schema) + external_config = bigquery.ExternalConfig("CSV") + external_config.source_uris = [ + "gs://cloud-samples-data/bigquery/us-states/us-states.csv" + ] + external_config.options.skip_leading_rows = 1 # optionally skip header row + table.external_data_configuration = external_config + + # Create a permanent table linked to the GCS file + table = client.create_table(table) # API request + + # Example query to find states starting with 'W' + sql = 'SELECT * FROM `{}.{}` WHERE name LIKE "W%"'.format(dataset_id, table_id) + + query_job = client.query(sql) # API request + + w_states = list(query_job) # Waits for query to finish + print("There are {} states with names starting with W.".format(len(w_states))) + assert len(w_states) == 4 + + # [END bigquery_query_external_gcs_perm] \ No newline at end of file diff --git a/bigquery/samples/query_external_gcs_temporary_table.py b/bigquery/samples/query_external_gcs_temporary_table.py new file mode 100644 index 000000000000..8181fad633fe --- /dev/null +++ b/bigquery/samples/query_external_gcs_temporary_table.py @@ -0,0 +1,45 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def query_external_gcs_temporary_table(client): + + # [START bigquery_query_external_gcs_temp] + # from google.cloud import bigquery + # client = bigquery.Client() + + # Configure the external data source and query job + external_config = bigquery.ExternalConfig("CSV") + external_config.source_uris = [ + "gs://cloud-samples-data/bigquery/us-states/us-states.csv" + ] + external_config.schema = [ + bigquery.SchemaField("name", "STRING"), + bigquery.SchemaField("post_abbr", "STRING"), + ] + external_config.options.skip_leading_rows = 1 # optionally skip header row + table_id = "us_states" + job_config = bigquery.QueryJobConfig() + job_config.table_definitions = {table_id: external_config} + + # Example query to find states starting with 'W' + sql = 'SELECT * FROM `{}` WHERE name LIKE "W%"'.format(table_id) + + query_job = client.query(sql, job_config=job_config) # API request + + w_states = list(query_job) # Waits for query to finish + print("There are {} states with names starting with W.".format(len(w_states))) + assert len(w_states) == 4 + + # [END bigquery_query_external_gcs_temp] \ No newline at end of file diff --git a/bigquery/samples/query_external_sheets_permanent_table.py b/bigquery/samples/query_external_sheets_permanent_table.py new file mode 100644 index 000000000000..8ca9af070a3d --- /dev/null +++ b/bigquery/samples/query_external_sheets_permanent_table.py @@ -0,0 +1,70 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def query_external_sheets_permanent_table(client, to_delete): + + # [START bigquery_query_external_sheets_perm] + dataset_id = "query_external_sheets_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + client.create_dataset(dataset) + to_delete.append(dataset) + + import google.auth + + # from google.cloud import bigquery + # dataset_id = 'my_dataset' + + # Create credentials with Drive & BigQuery API scopes + # Both APIs must be enabled for your project before running this code + credentials, project = google.auth.default( + scopes=[ + "https://www.googleapis.com/auth/drive", + "https://www.googleapis.com/auth/bigquery", + ] + ) + client = bigquery.Client(credentials=credentials, project=project) + + # Configure the external data source + dataset_ref = client.dataset(dataset_id) + table_id = "us_states" + schema = [ + bigquery.SchemaField("name", "STRING"), + bigquery.SchemaField("post_abbr", "STRING"), + ] + table = bigquery.Table(dataset_ref.table(table_id), schema=schema) + external_config = bigquery.ExternalConfig("GOOGLE_SHEETS") + # Use a shareable link or grant viewing access to the email address you + # used to authenticate with BigQuery (this example Sheet is public) + sheet_url = ( + "https://docs.google.com/spreadsheets" + "/d/1i_QCL-7HcSyUZmIbP9E6lO_T5u3HnpLe7dnpHaijg_E/edit?usp=sharing" + ) + external_config.source_uris = [sheet_url] + external_config.options.skip_leading_rows = 1 # optionally skip header row + table.external_data_configuration = external_config + + # Create a permanent table linked to the Sheets file + table = client.create_table(table) # API request + + # Example query to find states starting with 'W' + sql = 'SELECT * FROM `{}.{}` WHERE name LIKE "W%"'.format(dataset_id, table_id) + + query_job = client.query(sql) # API request + + w_states = list(query_job) # Waits for query to finish + print("There are {} states with names starting with W.".format(len(w_states))) + assert len(w_states) == 4 + + # [END bigquery_query_external_sheets_perm] \ No newline at end of file diff --git a/bigquery/samples/query_external_sheets_temporary_table.py b/bigquery/samples/query_external_sheets_temporary_table.py new file mode 100644 index 000000000000..ba6288ac7714 --- /dev/null +++ b/bigquery/samples/query_external_sheets_temporary_table.py @@ -0,0 +1,62 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def query_external_sheets_temporary_table(client): + + # [START bigquery_auth_drive_scope] + # [START bigquery_query_external_sheets_temp] + import google.auth + + # from google.cloud import bigquery + + # Create credentials with Drive & BigQuery API scopes + # Both APIs must be enabled for your project before running this code + credentials, project = google.auth.default( + scopes=[ + "https://www.googleapis.com/auth/drive", + "https://www.googleapis.com/auth/bigquery", + ] + ) + client = bigquery.Client(credentials=credentials, project=project) + + # Configure the external data source and query job + external_config = bigquery.ExternalConfig("GOOGLE_SHEETS") + # Use a shareable link or grant viewing access to the email address you + # used to authenticate with BigQuery (this example Sheet is public) + sheet_url = ( + "https://docs.google.com/spreadsheets" + "/d/1i_QCL-7HcSyUZmIbP9E6lO_T5u3HnpLe7dnpHaijg_E/edit?usp=sharing" + ) + external_config.source_uris = [sheet_url] + external_config.schema = [ + bigquery.SchemaField("name", "STRING"), + bigquery.SchemaField("post_abbr", "STRING"), + ] + external_config.options.skip_leading_rows = 1 # optionally skip header row + table_id = "us_states" + job_config = bigquery.QueryJobConfig() + job_config.table_definitions = {table_id: external_config} + + # Example query to find states starting with 'W' + sql = 'SELECT * FROM `{}` WHERE name LIKE "W%"'.format(table_id) + + query_job = client.query(sql, job_config=job_config) # API request + + w_states = list(query_job) # Waits for query to finish + print("There are {} states with names starting with W.".format(len(w_states))) + # [END bigquery_query_external_sheets_temp] + assert len(w_states) == 4 + + # [END bigquery_auth_drive_scope] \ No newline at end of file diff --git a/bigquery/samples/query_no_cache.py b/bigquery/samples/query_no_cache.py new file mode 100644 index 000000000000..fb4a27a6f886 --- /dev/null +++ b/bigquery/samples/query_no_cache.py @@ -0,0 +1,40 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def query_no_cache(client): + + # [START bigquery_query_no_cache] + # from google.cloud import bigquery + # client = bigquery.Client() + + job_config = bigquery.QueryJobConfig() + job_config.use_query_cache = False + sql = """ + SELECT corpus + FROM `bigquery-public-data.samples.shakespeare` + GROUP BY corpus; + """ + query_job = client.query( + sql, + # Location must match that of the dataset(s) referenced in the query. + location="US", + job_config=job_config, + ) # API request + + # Print the results. + for row in query_job: # API request - fetches results + print(row) + + # [END bigquery_query_no_cache] \ No newline at end of file diff --git a/bigquery/samples/query_results_as_dataframe.py b/bigquery/samples/query_results_as_dataframe.py new file mode 100644 index 000000000000..690afd74c62a --- /dev/null +++ b/bigquery/samples/query_results_as_dataframe.py @@ -0,0 +1,35 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def query_results_as_dataframe(client): + + # [START bigquery_query_results_dataframe] + # from google.cloud import bigquery + # client = bigquery.Client() + + sql = """ + SELECT name, SUM(number) as count + FROM `bigquery-public-data.usa_names.usa_1910_current` + GROUP BY name + ORDER BY count DESC + LIMIT 10 + """ + + df = client.query(sql).to_dataframe() + assert isinstance(df, pandas.DataFrame) + assert len(list(df)) == 2 # verify the number of columns + assert len(df) == 10 # verify the number of rows + + # [END bigquery_query_results_dataframe] \ No newline at end of file diff --git a/bigquery/samples/relax_column.py b/bigquery/samples/relax_column.py new file mode 100644 index 000000000000..16a56127094b --- /dev/null +++ b/bigquery/samples/relax_column.py @@ -0,0 +1,51 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def relax_column(client, to_delete): + + # [START bigquery_relax_column] + """Updates a schema field from required to nullable.""" + dataset_id = "relax_column_dataset_{}".format(_millis()) + table_id = "relax_column_table_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + dataset = client.create_dataset(dataset) + to_delete.append(dataset) + + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'my_dataset' + # table_id = 'my_table' + + original_schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + ] + table_ref = client.dataset(dataset_id).table(table_id) + table = bigquery.Table(table_ref, schema=original_schema) + table = client.create_table(table) + assert all(field.mode == "REQUIRED" for field in table.schema) + + # SchemaField properties cannot be edited after initialization. + # To make changes, construct new SchemaField objects. + relaxed_schema = [ + bigquery.SchemaField("full_name", "STRING", mode="NULLABLE"), + bigquery.SchemaField("age", "INTEGER", mode="NULLABLE"), + ] + table.schema = relaxed_schema + table = client.update_table(table, ["schema"]) + + assert all(field.mode == "NULLABLE" for field in table.schema) + + # [END bigquery_relax_column] \ No newline at end of file diff --git a/bigquery/samples/table_exists.py b/bigquery/samples/table_exists.py new file mode 100644 index 000000000000..66ecd216fb36 --- /dev/null +++ b/bigquery/samples/table_exists.py @@ -0,0 +1,34 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def table_exists(client, to_delete): + + # [START bigquery_table_exists] + """Determine if a table exists.""" + + DATASET_ID = "get_table_dataset_{}".format(_millis()) + TABLE_ID = "get_table_table_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(DATASET_ID)) + dataset = client.create_dataset(dataset) + to_delete.append(dataset) + + table_ref = dataset.table(TABLE_ID) + table = bigquery.Table(table_ref, schema=SCHEMA) + table = client.create_table(table) + + assert table_exists(client, table_ref) + assert not table_exists(client, dataset.table("i_dont_exist")) + + # [END bigquery_table_exists] \ No newline at end of file diff --git a/bigquery/samples/table_insert_rows.py b/bigquery/samples/table_insert_rows.py new file mode 100644 index 000000000000..cdbe2da43d71 --- /dev/null +++ b/bigquery/samples/table_insert_rows.py @@ -0,0 +1,45 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def table_insert_rows(client, to_delete): + + # [START bigquery_table_insert_rows] + """Insert / fetch table data.""" + dataset_id = "table_insert_rows_dataset_{}".format(_millis()) + table_id = "table_insert_rows_table_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + dataset = client.create_dataset(dataset) + dataset.location = "US" + to_delete.append(dataset) + + table = bigquery.Table(dataset.table(table_id), schema=SCHEMA) + table = client.create_table(table) + + # TODO(developer): Uncomment the lines below and replace with your values. + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'my_dataset' # replace with your dataset ID + # For this sample, the table must already exist and have a defined schema + # table_id = 'my_table' # replace with your table ID + # table_ref = client.dataset(dataset_id).table(table_id) + # table = client.get_table(table_ref) # API request + + rows_to_insert = [(u"Phred Phlyntstone", 32), (u"Wylma Phlyntstone", 29)] + + errors = client.insert_rows(table, rows_to_insert) # API request + + assert errors == [] + + # [END bigquery_table_insert_rows] \ No newline at end of file diff --git a/bigquery/samples/tests/test_add_empty_column.py b/bigquery/samples/tests/test_add_empty_column.py new file mode 100644 index 000000000000..8268954bf84c --- /dev/null +++ b/bigquery/samples/tests/test_add_empty_column.py @@ -0,0 +1,27 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import add_empty_column + + +def test_add_empty_column(capsys, client): + + schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + ] + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_browse_table_data.py b/bigquery/samples/tests/test_browse_table_data.py new file mode 100644 index 000000000000..43b7fb16fc42 --- /dev/null +++ b/bigquery/samples/tests/test_browse_table_data.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import browse_table_data + + +def test_browse_table_data(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_client_list_jobs.py b/bigquery/samples/tests/test_client_list_jobs.py new file mode 100644 index 000000000000..265ccbe211ac --- /dev/null +++ b/bigquery/samples/tests/test_client_list_jobs.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import client_list_jobs + + +def test_client_list_jobs(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_client_query.py b/bigquery/samples/tests/test_client_query.py new file mode 100644 index 000000000000..745f8a8bb777 --- /dev/null +++ b/bigquery/samples/tests/test_client_query.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import client_query + + +def test_client_query(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_client_query_add_column.py b/bigquery/samples/tests/test_client_query_add_column.py new file mode 100644 index 000000000000..9f535d93ee87 --- /dev/null +++ b/bigquery/samples/tests/test_client_query_add_column.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import client_query_add_column + + +def test_client_query_add_column(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_client_query_batch.py b/bigquery/samples/tests/test_client_query_batch.py new file mode 100644 index 000000000000..73fb6c4a3fa6 --- /dev/null +++ b/bigquery/samples/tests/test_client_query_batch.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import client_query_batch + + +def test_client_query_batch(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_client_query_destination_table.py b/bigquery/samples/tests/test_client_query_destination_table.py new file mode 100644 index 000000000000..ad25b852b41b --- /dev/null +++ b/bigquery/samples/tests/test_client_query_destination_table.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import client_query_destination_table + + +def test_client_query_destination_table(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_client_query_destination_table_cmek.py b/bigquery/samples/tests/test_client_query_destination_table_cmek.py new file mode 100644 index 000000000000..03dd38fd7473 --- /dev/null +++ b/bigquery/samples/tests/test_client_query_destination_table_cmek.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import client_query_destination_table_cmek + + +def test_client_query_destination_table_cmek(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_client_query_destination_table_legacy.py b/bigquery/samples/tests/test_client_query_destination_table_legacy.py new file mode 100644 index 000000000000..c23471dd3a79 --- /dev/null +++ b/bigquery/samples/tests/test_client_query_destination_table_legacy.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import client_query_destination_table_legacy + + +def test_client_query_destination_table_legacy(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_client_query_dry_run.py b/bigquery/samples/tests/test_client_query_dry_run.py new file mode 100644 index 000000000000..dccf0a1c6ed2 --- /dev/null +++ b/bigquery/samples/tests/test_client_query_dry_run.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import client_query_dry_run + + +def test_client_query_dry_run(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_client_query_legacy_sql.py b/bigquery/samples/tests/test_client_query_legacy_sql.py new file mode 100644 index 000000000000..7a066b2971b9 --- /dev/null +++ b/bigquery/samples/tests/test_client_query_legacy_sql.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import client_query_legacy_sql + + +def test_client_query_legacy_sql(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_client_query_relax_column.py b/bigquery/samples/tests/test_client_query_relax_column.py new file mode 100644 index 000000000000..9f8c3d4fd9ad --- /dev/null +++ b/bigquery/samples/tests/test_client_query_relax_column.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import client_query_relax_column + + +def test_client_query_relax_column(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_client_query_total_rows.py b/bigquery/samples/tests/test_client_query_total_rows.py new file mode 100644 index 000000000000..1ee895949f4f --- /dev/null +++ b/bigquery/samples/tests/test_client_query_total_rows.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import client_query_total_rows + + +def test_client_query_total_rows(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_client_query_w_array_params.py b/bigquery/samples/tests/test_client_query_w_array_params.py new file mode 100644 index 000000000000..51586c6b557d --- /dev/null +++ b/bigquery/samples/tests/test_client_query_w_array_params.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import client_query_w_array_params + + +def test_client_query_w_array_params(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_client_query_w_named_params.py b/bigquery/samples/tests/test_client_query_w_named_params.py new file mode 100644 index 000000000000..370bf09afd16 --- /dev/null +++ b/bigquery/samples/tests/test_client_query_w_named_params.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import client_query_w_named_params + + +def test_client_query_w_named_params(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_client_query_w_positional_params.py b/bigquery/samples/tests/test_client_query_w_positional_params.py new file mode 100644 index 000000000000..5f6cf57a5125 --- /dev/null +++ b/bigquery/samples/tests/test_client_query_w_positional_params.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import client_query_w_positional_params + + +def test_client_query_w_positional_params(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_client_query_w_struct_params.py b/bigquery/samples/tests/test_client_query_w_struct_params.py new file mode 100644 index 000000000000..102538d942dc --- /dev/null +++ b/bigquery/samples/tests/test_client_query_w_struct_params.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import client_query_w_struct_params + + +def test_client_query_w_struct_params(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_client_query_w_timestamp_params.py b/bigquery/samples/tests/test_client_query_w_timestamp_params.py new file mode 100644 index 000000000000..e2ff35dcf79b --- /dev/null +++ b/bigquery/samples/tests/test_client_query_w_timestamp_params.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import client_query_w_timestamp_params + + +def test_client_query_w_timestamp_params(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_copy_table.py b/bigquery/samples/tests/test_copy_table.py new file mode 100644 index 000000000000..ddf157d0ecff --- /dev/null +++ b/bigquery/samples/tests/test_copy_table.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import copy_table + + +def test_copy_table(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_copy_table_cmek.py b/bigquery/samples/tests/test_copy_table_cmek.py new file mode 100644 index 000000000000..cf15f0c0c515 --- /dev/null +++ b/bigquery/samples/tests/test_copy_table_cmek.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import copy_table_cmek + + +def test_copy_table_cmek(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_copy_table_multiple_source.py b/bigquery/samples/tests/test_copy_table_multiple_source.py new file mode 100644 index 000000000000..9c4955194f93 --- /dev/null +++ b/bigquery/samples/tests/test_copy_table_multiple_source.py @@ -0,0 +1,24 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import copy_table_multiple_source + + +def test_copy_table_multiple_source(capsys, client): + + import six + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_create_client_default_credentials.py b/bigquery/samples/tests/test_create_client_default_credentials.py new file mode 100644 index 000000000000..82ab5933a85f --- /dev/null +++ b/bigquery/samples/tests/test_create_client_default_credentials.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import create_client_default_credentials + + +def test_create_client_default_credentials(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_create_partitioned_table.py b/bigquery/samples/tests/test_create_partitioned_table.py new file mode 100644 index 000000000000..b5fd99e783e2 --- /dev/null +++ b/bigquery/samples/tests/test_create_partitioned_table.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import create_partitioned_table + + +def test_create_partitioned_table(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_create_table_cmek.py b/bigquery/samples/tests/test_create_table_cmek.py new file mode 100644 index 000000000000..27454da3c9d8 --- /dev/null +++ b/bigquery/samples/tests/test_create_table_cmek.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import create_table_cmek + + +def test_create_table_cmek(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_create_table_nested_repeated_schema.py b/bigquery/samples/tests/test_create_table_nested_repeated_schema.py new file mode 100644 index 000000000000..66b5cf3e8a90 --- /dev/null +++ b/bigquery/samples/tests/test_create_table_nested_repeated_schema.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import create_table_nested_repeated_schema + + +def test_create_table_nested_repeated_schema(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_dataset_exists.py b/bigquery/samples/tests/test_dataset_exists.py new file mode 100644 index 000000000000..de4ad29de42f --- /dev/null +++ b/bigquery/samples/tests/test_dataset_exists.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import dataset_exists + + +def test_dataset_exists(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_ddl_create_view.py b/bigquery/samples/tests/test_ddl_create_view.py new file mode 100644 index 000000000000..8fddd1792c54 --- /dev/null +++ b/bigquery/samples/tests/test_ddl_create_view.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import ddl_create_view + + +def test_ddl_create_view(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_extract_table.py b/bigquery/samples/tests/test_extract_table.py new file mode 100644 index 000000000000..022437073ad1 --- /dev/null +++ b/bigquery/samples/tests/test_extract_table.py @@ -0,0 +1,30 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import extract_table + + +def test_extract_table(capsys, client): + + from test_utils.retry import RetryErrors + from google.api_core.exceptions import InternalServerError + from google.api_core.exceptions import ServiceUnavailable + from google.api_core.exceptions import TooManyRequests + retry_storage_errors = RetryErrors( + (TooManyRequests, InternalServerError, ServiceUnavailable) + ) + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_extract_table_compressed.py b/bigquery/samples/tests/test_extract_table_compressed.py new file mode 100644 index 000000000000..6e3a691b829e --- /dev/null +++ b/bigquery/samples/tests/test_extract_table_compressed.py @@ -0,0 +1,30 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import extract_table_compressed + + +def test_extract_table_compressed(capsys, client): + + from test_utils.retry import RetryErrors + from google.api_core.exceptions import InternalServerError + from google.api_core.exceptions import ServiceUnavailable + from google.api_core.exceptions import TooManyRequests + retry_storage_errors = RetryErrors( + (TooManyRequests, InternalServerError, ServiceUnavailable) + ) + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_extract_table_json.py b/bigquery/samples/tests/test_extract_table_json.py new file mode 100644 index 000000000000..daad56bcfab5 --- /dev/null +++ b/bigquery/samples/tests/test_extract_table_json.py @@ -0,0 +1,30 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import extract_table_json + + +def test_extract_table_json(capsys, client): + + from test_utils.retry import RetryErrors + from google.api_core.exceptions import InternalServerError + from google.api_core.exceptions import ServiceUnavailable + from google.api_core.exceptions import TooManyRequests + retry_storage_errors = RetryErrors( + (TooManyRequests, InternalServerError, ServiceUnavailable) + ) + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_list_datasets_by_label.py b/bigquery/samples/tests/test_list_datasets_by_label.py new file mode 100644 index 000000000000..dc043c4e06c1 --- /dev/null +++ b/bigquery/samples/tests/test_list_datasets_by_label.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import list_datasets_by_label + + +def test_list_datasets_by_label(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_list_rows_as_dataframe.py b/bigquery/samples/tests/test_list_rows_as_dataframe.py new file mode 100644 index 000000000000..6dd58bd0ef0b --- /dev/null +++ b/bigquery/samples/tests/test_list_rows_as_dataframe.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import list_rows_as_dataframe + + +def test_list_rows_as_dataframe(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_load_and_query_partitioned_table.py b/bigquery/samples/tests/test_load_and_query_partitioned_table.py new file mode 100644 index 000000000000..2c659c5ebf4c --- /dev/null +++ b/bigquery/samples/tests/test_load_and_query_partitioned_table.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import load_and_query_partitioned_table + + +def test_load_and_query_partitioned_table(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_load_table_add_column.py b/bigquery/samples/tests/test_load_table_add_column.py new file mode 100644 index 000000000000..da85c5a9f6a7 --- /dev/null +++ b/bigquery/samples/tests/test_load_table_add_column.py @@ -0,0 +1,24 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import load_table_add_column + + +def test_load_table_add_column(capsys, client): + + import os + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_load_table_from_dataframe.py b/bigquery/samples/tests/test_load_table_from_dataframe.py new file mode 100644 index 000000000000..85dcc1efb6a4 --- /dev/null +++ b/bigquery/samples/tests/test_load_table_from_dataframe.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import load_table_from_dataframe + + +def test_load_table_from_dataframe(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_load_table_from_file.py b/bigquery/samples/tests/test_load_table_from_file.py new file mode 100644 index 000000000000..3bf604801949 --- /dev/null +++ b/bigquery/samples/tests/test_load_table_from_file.py @@ -0,0 +1,24 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import load_table_from_file + + +def test_load_table_from_file(capsys, client): + + import os + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_load_table_from_uri_autodetect.py b/bigquery/samples/tests/test_load_table_from_uri_autodetect.py new file mode 100644 index 000000000000..8b9096657461 --- /dev/null +++ b/bigquery/samples/tests/test_load_table_from_uri_autodetect.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import load_table_from_uri_autodetect + + +def test_load_table_from_uri_autodetect(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_load_table_from_uri_avro.py b/bigquery/samples/tests/test_load_table_from_uri_avro.py new file mode 100644 index 000000000000..9d1785c937cd --- /dev/null +++ b/bigquery/samples/tests/test_load_table_from_uri_avro.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import load_table_from_uri_avro + + +def test_load_table_from_uri_avro(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_load_table_from_uri_cmek.py b/bigquery/samples/tests/test_load_table_from_uri_cmek.py new file mode 100644 index 000000000000..3adf093be3e7 --- /dev/null +++ b/bigquery/samples/tests/test_load_table_from_uri_cmek.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import load_table_from_uri_cmek + + +def test_load_table_from_uri_cmek(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_load_table_from_uri_csv.py b/bigquery/samples/tests/test_load_table_from_uri_csv.py new file mode 100644 index 000000000000..91aadefa4d34 --- /dev/null +++ b/bigquery/samples/tests/test_load_table_from_uri_csv.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import load_table_from_uri_csv + + +def test_load_table_from_uri_csv(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_load_table_from_uri_json.py b/bigquery/samples/tests/test_load_table_from_uri_json.py new file mode 100644 index 000000000000..edfb3debf922 --- /dev/null +++ b/bigquery/samples/tests/test_load_table_from_uri_json.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import load_table_from_uri_json + + +def test_load_table_from_uri_json(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_load_table_from_uri_orc.py b/bigquery/samples/tests/test_load_table_from_uri_orc.py new file mode 100644 index 000000000000..bfa0ecd112d7 --- /dev/null +++ b/bigquery/samples/tests/test_load_table_from_uri_orc.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import load_table_from_uri_orc + + +def test_load_table_from_uri_orc(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_load_table_from_uri_parquet.py b/bigquery/samples/tests/test_load_table_from_uri_parquet.py new file mode 100644 index 000000000000..cd4a6a25b73c --- /dev/null +++ b/bigquery/samples/tests/test_load_table_from_uri_parquet.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import load_table_from_uri_parquet + + +def test_load_table_from_uri_parquet(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_load_table_from_uri_truncate.py b/bigquery/samples/tests/test_load_table_from_uri_truncate.py new file mode 100644 index 000000000000..f91822459c1f --- /dev/null +++ b/bigquery/samples/tests/test_load_table_from_uri_truncate.py @@ -0,0 +1,24 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import load_table_from_uri_truncate + + +def test_load_table_from_uri_truncate(capsys, client): + + import six + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_load_table_relax_column.py b/bigquery/samples/tests/test_load_table_relax_column.py new file mode 100644 index 000000000000..986c5a51004c --- /dev/null +++ b/bigquery/samples/tests/test_load_table_relax_column.py @@ -0,0 +1,24 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import load_table_relax_column + + +def test_load_table_relax_column(capsys, client): + + import os + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_manage_dataset_labels.py b/bigquery/samples/tests/test_manage_dataset_labels.py new file mode 100644 index 000000000000..ee740f3996b6 --- /dev/null +++ b/bigquery/samples/tests/test_manage_dataset_labels.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import manage_dataset_labels + + +def test_manage_dataset_labels(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_manage_job.py b/bigquery/samples/tests/test_manage_job.py new file mode 100644 index 000000000000..840cb4a05ca4 --- /dev/null +++ b/bigquery/samples/tests/test_manage_job.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import manage_job + + +def test_manage_job(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_manage_table_labels.py b/bigquery/samples/tests/test_manage_table_labels.py new file mode 100644 index 000000000000..6c9c97daaa9c --- /dev/null +++ b/bigquery/samples/tests/test_manage_table_labels.py @@ -0,0 +1,27 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import manage_table_labels + + +def test_manage_table_labels(capsys, client): + + schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + ] + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_manage_views.py b/bigquery/samples/tests/test_manage_views.py new file mode 100644 index 000000000000..218a8b773ba6 --- /dev/null +++ b/bigquery/samples/tests/test_manage_views.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import manage_views + + +def test_manage_views(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_query_external_gcs_permanent_table.py b/bigquery/samples/tests/test_query_external_gcs_permanent_table.py new file mode 100644 index 000000000000..0ae97297c220 --- /dev/null +++ b/bigquery/samples/tests/test_query_external_gcs_permanent_table.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import query_external_gcs_permanent_table + + +def test_query_external_gcs_permanent_table(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_query_external_gcs_temporary_table.py b/bigquery/samples/tests/test_query_external_gcs_temporary_table.py new file mode 100644 index 000000000000..1ea95f615925 --- /dev/null +++ b/bigquery/samples/tests/test_query_external_gcs_temporary_table.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import query_external_gcs_temporary_table + + +def test_query_external_gcs_temporary_table(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_query_external_sheets_permanent_table.py b/bigquery/samples/tests/test_query_external_sheets_permanent_table.py new file mode 100644 index 000000000000..52bd62a56e52 --- /dev/null +++ b/bigquery/samples/tests/test_query_external_sheets_permanent_table.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import query_external_sheets_permanent_table + + +def test_query_external_sheets_permanent_table(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_query_external_sheets_temporary_table.py b/bigquery/samples/tests/test_query_external_sheets_temporary_table.py new file mode 100644 index 000000000000..32f9631df9d2 --- /dev/null +++ b/bigquery/samples/tests/test_query_external_sheets_temporary_table.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import query_external_sheets_temporary_table + + +def test_query_external_sheets_temporary_table(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_query_no_cache.py b/bigquery/samples/tests/test_query_no_cache.py new file mode 100644 index 000000000000..9661eb1fb60f --- /dev/null +++ b/bigquery/samples/tests/test_query_no_cache.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import query_no_cache + + +def test_query_no_cache(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_query_results_as_dataframe.py b/bigquery/samples/tests/test_query_results_as_dataframe.py new file mode 100644 index 000000000000..fd9db3c8b5d4 --- /dev/null +++ b/bigquery/samples/tests/test_query_results_as_dataframe.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import query_results_as_dataframe + + +def test_query_results_as_dataframe(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_relax_column.py b/bigquery/samples/tests/test_relax_column.py new file mode 100644 index 000000000000..561c98567102 --- /dev/null +++ b/bigquery/samples/tests/test_relax_column.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import relax_column + + +def test_relax_column(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_table_exists.py b/bigquery/samples/tests/test_table_exists.py new file mode 100644 index 000000000000..e8baeae3a6ea --- /dev/null +++ b/bigquery/samples/tests/test_table_exists.py @@ -0,0 +1,27 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import table_exists + + +def test_table_exists(capsys, client): + + schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + ] + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_table_insert_rows.py b/bigquery/samples/tests/test_table_insert_rows.py new file mode 100644 index 000000000000..cc3b3aa87a30 --- /dev/null +++ b/bigquery/samples/tests/test_table_insert_rows.py @@ -0,0 +1,27 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import table_insert_rows + + +def test_table_insert_rows(capsys, client): + + schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + ] + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_undelete_table.py b/bigquery/samples/tests/test_undelete_table.py new file mode 100644 index 000000000000..03437c033ff4 --- /dev/null +++ b/bigquery/samples/tests/test_undelete_table.py @@ -0,0 +1,27 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import undelete_table + + +def test_undelete_table(capsys, client): + + schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + ] + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_update_table_cmek.py b/bigquery/samples/tests/test_update_table_cmek.py new file mode 100644 index 000000000000..7c8e8ba6249d --- /dev/null +++ b/bigquery/samples/tests/test_update_table_cmek.py @@ -0,0 +1,22 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import update_table_cmek + + +def test_update_table_cmek(capsys, client): + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_update_table_description.py b/bigquery/samples/tests/test_update_table_description.py new file mode 100644 index 000000000000..3c3c6b6989ad --- /dev/null +++ b/bigquery/samples/tests/test_update_table_description.py @@ -0,0 +1,27 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import update_table_description + + +def test_update_table_description(capsys, client): + + schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + ] + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/tests/test_update_table_expiration.py b/bigquery/samples/tests/test_update_table_expiration.py new file mode 100644 index 000000000000..836830fe3874 --- /dev/null +++ b/bigquery/samples/tests/test_update_table_expiration.py @@ -0,0 +1,27 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import update_table_expiration + + +def test_update_table_expiration(capsys, client): + + schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + ] + + out, err = capsys.readouterr() + assert \ No newline at end of file diff --git a/bigquery/samples/undelete_table.py b/bigquery/samples/undelete_table.py new file mode 100644 index 000000000000..340e42b99018 --- /dev/null +++ b/bigquery/samples/undelete_table.py @@ -0,0 +1,77 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def undelete_table(client, to_delete): + + # [START bigquery_undelete_table] + dataset_id = "undelete_table_dataset_{}".format(_millis()) + table_id = "undelete_table_table_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + dataset.location = "US" + dataset = client.create_dataset(dataset) + to_delete.append(dataset) + + table = bigquery.Table(dataset.table(table_id), schema=SCHEMA) + client.create_table(table) + + # TODO(developer): Uncomment the lines below and replace with your values. + # import time + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'my_dataset' # Replace with your dataset ID. + # table_id = 'my_table' # Replace with your table ID. + + table_ref = client.dataset(dataset_id).table(table_id) + + # TODO(developer): Choose an appropriate snapshot point as epoch + # milliseconds. For this example, we choose the current time as we're about + # to delete the table immediately afterwards. + snapshot_epoch = int(time.time() * 1000) + + # Due to very short lifecycle of the table, ensure we're not picking a time + # prior to the table creation due to time drift between backend and client. + table = client.get_table(table_ref) + created_epoch = datetime_helpers.to_microseconds(table.created) + if created_epoch > snapshot_epoch: + snapshot_epoch = created_epoch + + # [START bigquery_undelete_table] + + # "Accidentally" delete the table. + client.delete_table(table_ref) # API request + + # Construct the restore-from table ID using a snapshot decorator. + snapshot_table_id = "{}@{}".format(table_id, snapshot_epoch) + source_table_ref = client.dataset(dataset_id).table(snapshot_table_id) + + # Choose a new table ID for the recovered table data. + recovered_table_id = "{}_recovered".format(table_id) + dest_table_ref = client.dataset(dataset_id).table(recovered_table_id) + + # Construct and run a copy job. + job = client.copy_table( + source_table_ref, + dest_table_ref, + # Location must match that of the source and destination tables. + location="US", + ) # API request + + job.result() # Waits for job to complete. + + print( + "Copied data from deleted table {} to {}".format(table_id, recovered_table_id) + ) + + # [END bigquery_undelete_table] \ No newline at end of file diff --git a/bigquery/samples/update_table_cmek.py b/bigquery/samples/update_table_cmek.py new file mode 100644 index 000000000000..ee4a85561759 --- /dev/null +++ b/bigquery/samples/update_table_cmek.py @@ -0,0 +1,55 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def update_table_cmek(client, to_delete): + + # [START bigquery_update_table_cmek] + """Patch a table's metadata.""" + dataset_id = "update_table_cmek_{}".format(_millis()) + table_id = "update_table_cmek_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + client.create_dataset(dataset) + to_delete.append(dataset) + + table = bigquery.Table(dataset.table(table_id)) + original_kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format( + "cloud-samples-tests", "us-central1", "test", "test" + ) + table.encryption_configuration = bigquery.EncryptionConfiguration( + kms_key_name=original_kms_key_name + ) + table = client.create_table(table) + + # from google.cloud import bigquery + # client = bigquery.Client() + + assert table.encryption_configuration.kms_key_name == original_kms_key_name + + # Set a new encryption key to use for the destination. + # TODO: Replace this key with a key you have created in KMS. + updated_kms_key_name = ( + "projects/cloud-samples-tests/locations/us-central1/" + "keyRings/test/cryptoKeys/otherkey" + ) + table.encryption_configuration = bigquery.EncryptionConfiguration( + kms_key_name=updated_kms_key_name + ) + + table = client.update_table(table, ["encryption_configuration"]) # API request + + assert table.encryption_configuration.kms_key_name == updated_kms_key_name + assert original_kms_key_name != updated_kms_key_name + + # [END bigquery_update_table_cmek] \ No newline at end of file diff --git a/bigquery/samples/update_table_description.py b/bigquery/samples/update_table_description.py new file mode 100644 index 000000000000..d83bb7c1e374 --- /dev/null +++ b/bigquery/samples/update_table_description.py @@ -0,0 +1,42 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def update_table_description(client, to_delete): + + # [START bigquery_update_table_description] + """Update a table's description.""" + dataset_id = "update_table_description_dataset_{}".format(_millis()) + table_id = "update_table_description_table_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + client.create_dataset(dataset) + to_delete.append(dataset) + + table = bigquery.Table(dataset.table(table_id), schema=SCHEMA) + table.description = "Original description." + table = client.create_table(table) + + # from google.cloud import bigquery + # client = bigquery.Client() + # table_ref = client.dataset('my_dataset').table('my_table') + # table = client.get_table(table_ref) # API request + + assert table.description == "Original description." + table.description = "Updated description." + + table = client.update_table(table, ["description"]) # API request + + assert table.description == "Updated description." + + # [END bigquery_update_table_description] \ No newline at end of file diff --git a/bigquery/samples/update_table_expiration.py b/bigquery/samples/update_table_expiration.py new file mode 100644 index 000000000000..7c14c6aa6b46 --- /dev/null +++ b/bigquery/samples/update_table_expiration.py @@ -0,0 +1,48 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def update_table_expiration(client, to_delete): + + # [START bigquery_update_table_expiration] + """Update a table's expiration time.""" + dataset_id = "update_table_expiration_dataset_{}".format(_millis()) + table_id = "update_table_expiration_table_{}".format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + client.create_dataset(dataset) + to_delete.append(dataset) + + table = bigquery.Table(dataset.table(table_id), schema=SCHEMA) + table = client.create_table(table) + + import datetime + import pytz + + # from google.cloud import bigquery + # client = bigquery.Client() + # table_ref = client.dataset('my_dataset').table('my_table') + # table = client.get_table(table_ref) # API request + + assert table.expires is None + + # set table to expire 5 days from now + expiration = datetime.datetime.now(pytz.utc) + datetime.timedelta(days=5) + table.expires = expiration + table = client.update_table(table, ["expires"]) # API request + + # expiration is stored in milliseconds + margin = datetime.timedelta(microseconds=1000) + assert expiration - margin <= table.expires <= expiration + margin + + # [END bigquery_update_table_expiration] \ No newline at end of file From 377310c027c3883f2c8c39a3d758aa1bfed4d3b4 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 9 Aug 2019 17:33:01 +0300 Subject: [PATCH 02/68] list_datasets_by_label *.rst + *.py + test + conf + --- bigquery/docs/usage/datasets.rst | 18 +++++++++--------- bigquery/samples/list_datasets_by_label.py | 16 ++++------------ bigquery/samples/tests/conftest.py | 9 +++++++++ .../tests/test_list_datasets_by_label.py | 6 ++++-- 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/bigquery/docs/usage/datasets.rst b/bigquery/docs/usage/datasets.rst index 6d5cda7d84aa..97348f015c62 100644 --- a/bigquery/docs/usage/datasets.rst +++ b/bigquery/docs/usage/datasets.rst @@ -25,6 +25,15 @@ List datasets for a project with the :start-after: [START bigquery_list_datasets] :end-before: [END bigquery_list_datasets] +List datasets by label for a project with the +:func:`~google.cloud.bigquery.client.Client.list_datasets` method: + +.. literalinclude:: ../samples/samples/list_datasets_by_label.py + :language: python + :dedent: 4 + :start-after: [START bigquery_list_datasets_by_label] + :end-before: [END bigquery_list_datasets_by_label] + Getting a Dataset ^^^^^^^^^^^^^^^^^ @@ -94,15 +103,6 @@ Delete a dataset with the [-REPLACE_COMMENT-] :func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: -.. literalinclude:: ../samples/samples/list_datasets_by_label.py - :language: python - :dedent: 4 - :start-after: [START bigquery_list_datasets_by_label] - :end-before: [END bigquery_list_datasets_by_label] - -[-REPLACE_COMMENT-] -:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: - .. literalinclude:: ../samples/samples/manage_dataset_labels.py :language: python :dedent: 4 diff --git a/bigquery/samples/list_datasets_by_label.py b/bigquery/samples/list_datasets_by_label.py index ce7636f9f0c7..26eb038c59e2 100644 --- a/bigquery/samples/list_datasets_by_label.py +++ b/bigquery/samples/list_datasets_by_label.py @@ -13,20 +13,14 @@ # limitations under the License. -def list_datasets_by_label(client, to_delete): +def list_datasets_by_label(client): # [START bigquery_list_datasets_by_label] - dataset_id = "list_datasets_by_label_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - dataset.labels = {"color": "green"} - dataset = client.create_dataset(dataset) # API request - to_delete.append(dataset) + from google.cloud import bigquery - # from google.cloud import bigquery + # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() - # The following label filter example will find datasets with an - # arbitrary 'color' label set to 'green' label_filter = "labels.color:green" datasets = list(client.list_datasets(filter=label_filter)) @@ -36,7 +30,5 @@ def list_datasets_by_label(client, to_delete): print("\t{}".format(dataset.dataset_id)) else: print("No datasets found with this filter.") - found = set([dataset.dataset_id for dataset in datasets]) - assert dataset_id in found - # [END bigquery_list_datasets_by_label] \ No newline at end of file + # [END bigquery_list_datasets_by_label] diff --git a/bigquery/samples/tests/conftest.py b/bigquery/samples/tests/conftest.py index fe5391ee8a4d..6156226ecb4b 100644 --- a/bigquery/samples/tests/conftest.py +++ b/bigquery/samples/tests/conftest.py @@ -65,6 +65,15 @@ def dataset_id(client): client.delete_dataset(dataset, delete_contents=True, not_found_ok=True) +@pytest.fixture +def dataset_label(client, dataset_id): + dataset = client.get_dataset(dataset_id) + dataset.labels = {"color": "green"} + dataset = client.update_dataset(dataset, ["labels"]) + yield "{}.{}".format(dataset.project, dataset.dataset_id) + client.delete_dataset(dataset, delete_contents=True, not_found_ok=True) + + @pytest.fixture def table_id(client, dataset_id): now = datetime.datetime.now() diff --git a/bigquery/samples/tests/test_list_datasets_by_label.py b/bigquery/samples/tests/test_list_datasets_by_label.py index dc043c4e06c1..80fbf641d409 100644 --- a/bigquery/samples/tests/test_list_datasets_by_label.py +++ b/bigquery/samples/tests/test_list_datasets_by_label.py @@ -16,7 +16,9 @@ from .. import list_datasets_by_label -def test_list_datasets_by_label(capsys, client): +def test_list_datasets_by_label(capsys, client, dataset_id, dataset_label): + label_filter = "labels.color:green" + list_datasets_by_label.list_datasets_by_label(client) out, err = capsys.readouterr() - assert \ No newline at end of file + assert "Datasets filtered by {}:".format(label_filter) in out From a7b2376e681b692c513bd3b94e0a75f3ceb52b17 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Mon, 12 Aug 2019 13:09:22 +0300 Subject: [PATCH 03/68] manage_dataset_labels Methods were divided into 3 files: - add label - get labels - delete labels *.rst - docs updated tests passed successfully --- bigquery/docs/snippets.py | 6 +- bigquery/docs/usage/datasets.rst | 39 +++++++--- bigquery/noxfile.py | 2 +- bigquery/samples/delete_dataset_labels.py | 41 +++++++++++ bigquery/samples/get_dataset_labels.py | 38 ++++++++++ bigquery/samples/label_dataset.py | 42 +++++++++++ bigquery/samples/manage_dataset_labels.py | 72 ------------------- .../tests/test_delete_dataset_labels.py | 23 ++++++ .../samples/tests/test_get_dataset_labels.py | 23 ++++++ ...ataset_labels.py => test_label_dataset.py} | 8 ++- 10 files changed, 206 insertions(+), 88 deletions(-) create mode 100644 bigquery/samples/delete_dataset_labels.py create mode 100644 bigquery/samples/get_dataset_labels.py create mode 100644 bigquery/samples/label_dataset.py delete mode 100644 bigquery/samples/manage_dataset_labels.py create mode 100644 bigquery/samples/tests/test_delete_dataset_labels.py create mode 100644 bigquery/samples/tests/test_get_dataset_labels.py rename bigquery/samples/tests/{test_manage_dataset_labels.py => test_label_dataset.py} (75%) diff --git a/bigquery/docs/snippets.py b/bigquery/docs/snippets.py index b104e79ea789..766ddc238102 100644 --- a/bigquery/docs/snippets.py +++ b/bigquery/docs/snippets.py @@ -23,11 +23,11 @@ need to be deleted during teardown. """ -import os +# import os import time import pytest -import six +# import six try: import fastparquet @@ -42,7 +42,7 @@ except (ImportError, AttributeError): pyarrow = None -from google.api_core import datetime_helpers +# from google.api_core import datetime_helpers from google.api_core.exceptions import InternalServerError from google.api_core.exceptions import ServiceUnavailable from google.api_core.exceptions import TooManyRequests diff --git a/bigquery/docs/usage/datasets.rst b/bigquery/docs/usage/datasets.rst index 97348f015c62..314a593a90b5 100644 --- a/bigquery/docs/usage/datasets.rst +++ b/bigquery/docs/usage/datasets.rst @@ -79,6 +79,36 @@ Modify user permissions on a dataset with the :start-after: [START bigquery_update_dataset_access] :end-before: [END bigquery_update_dataset_access] +Manage Dataset labels +^^^^^^^^^^^^^^^^^^^^^ + +Add labels to a dataset with the +:func:`~google.cloud.bigquery.client.Client.update_dataset` method: + +.. literalinclude:: ../samples/samples/label_dataset.py + :language: python + :dedent: 4 + :start-after: [START bigquery_label_dataset] + :end-before: [END bigquery_label_dataset] + +Get dataset's labels with the +:func:`~google.cloud.bigquery.client.Client.get_dataset` method: + +.. literalinclude:: ../samples/samples/get_dataset_labels.py + :language: python + :dedent: 4 + :start-after: [START bigquery_get_dataset_labels] + :end-before: [END bigquery_get_dataset_labels] + +Delete dataset's labels with the +:func:`~google.cloud.bigquery.client.Client.update_dataset` method: + +.. literalinclude:: ../samples/samples/delete_dataset_labels.py + :language: python + :dedent: 4 + :start-after: [START bigquery_delete_label_dataset] + :end-before: [END bigquery_delete_label_dataset] + Deleting a Dataset ^^^^^^^^^^^^^^^^^^ @@ -99,12 +129,3 @@ Delete a dataset with the :dedent: 4 :start-after: [START bigquery_dataset_exists] :end-before: [END bigquery_dataset_exists] - -[-REPLACE_COMMENT-] -:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: - -.. literalinclude:: ../samples/samples/manage_dataset_labels.py - :language: python - :dedent: 4 - :start-after: [START bigquery_delete_label_dataset] - :end-before: [END bigquery_delete_label_dataset] diff --git a/bigquery/noxfile.py b/bigquery/noxfile.py index 37611a5ce296..ce3249c78cb6 100644 --- a/bigquery/noxfile.py +++ b/bigquery/noxfile.py @@ -136,7 +136,7 @@ def cover(session): session.run("coverage", "erase") -@nox.session(python="3.6") +@nox.session(python="3.7") def lint(session): """Run linters. diff --git a/bigquery/samples/delete_dataset_labels.py b/bigquery/samples/delete_dataset_labels.py new file mode 100644 index 000000000000..432619af7a1c --- /dev/null +++ b/bigquery/samples/delete_dataset_labels.py @@ -0,0 +1,41 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def delete_dataset_labels(client, dataset_id): + + # [START bigquery_delete_label_dataset] + from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + # TODO(developer): Set dataset_id to the ID of the dataset to fetch. + # dataset_id = 'your-project.your_dataset' + + dataset = client.get_dataset(dataset_id) + + # To delete a label from a dataset, set its value to None + dataset.labels["color"] = None + + dataset = client.update_dataset(dataset, ["labels"]) # API request + + print("Dataset ID: {}".format(dataset_id)) + if dataset.labels: + for label, value in dataset.labels.items(): + print("\t{}: {}".format(label, value)) + else: + print("\tLabels deleted.") + + # [END bigquery_delete_label_dataset] diff --git a/bigquery/samples/get_dataset_labels.py b/bigquery/samples/get_dataset_labels.py new file mode 100644 index 000000000000..ee9309ba25a0 --- /dev/null +++ b/bigquery/samples/get_dataset_labels.py @@ -0,0 +1,38 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def get_dataset_labels(client, dataset_id): + + # [START bigquery_get_dataset_labels] + from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + # TODO(developer): Set dataset_id to the ID of the dataset to fetch. + # dataset_id = 'your-project.your_dataset' + + dataset = client.get_dataset(dataset_id) + + # View dataset labels + print("Dataset ID: {}".format(dataset_id)) + print("Labels:") + if dataset.labels: + for label, value in dataset.labels.items(): + print("\t{}: {}".format(label, value)) + else: + print("\tDataset has no labels defined.") + + # [END bigquery_get_dataset_labels] diff --git a/bigquery/samples/label_dataset.py b/bigquery/samples/label_dataset.py new file mode 100644 index 000000000000..57e10ca0a59c --- /dev/null +++ b/bigquery/samples/label_dataset.py @@ -0,0 +1,42 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def label_dataset(client, dataset_id): + + # [START bigquery_label_dataset] + from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + # TODO(developer): Set dataset_id to the ID of the dataset to fetch. + # dataset_id = 'your-project.your_dataset' + + dataset = client.get_dataset(dataset_id) + + labels = {"color": "green"} + dataset.labels = labels + + dataset = client.update_dataset(dataset, ["labels"]) + + print("Dataset ID: {}".format(dataset_id)) + print("Added labels:") + if dataset.labels: + for label, value in dataset.labels.items(): + print("\t{}: {}".format(label, value)) + else: + print("\tNo labels defined.") + + # [END bigquery_label_dataset] diff --git a/bigquery/samples/manage_dataset_labels.py b/bigquery/samples/manage_dataset_labels.py deleted file mode 100644 index 430985d91746..000000000000 --- a/bigquery/samples/manage_dataset_labels.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def manage_dataset_labels(client, to_delete): - - # [START bigquery_delete_label_dataset] - dataset_id = "label_dataset_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - dataset = client.create_dataset(dataset) - to_delete.append(dataset) - - # [START bigquery_label_dataset] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_ref = client.dataset('my_dataset') - # dataset = client.get_dataset(dataset_ref) # API request - - assert dataset.labels == {} - labels = {"color": "green"} - dataset.labels = labels - - dataset = client.update_dataset(dataset, ["labels"]) # API request - - assert dataset.labels == labels - # [END bigquery_label_dataset] - - # [START bigquery_get_dataset_labels] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'my_dataset' - - dataset_ref = client.dataset(dataset_id) - dataset = client.get_dataset(dataset_ref) # API request - - # View dataset labels - print("Dataset ID: {}".format(dataset_id)) - print("Labels:") - if dataset.labels: - for label, value in dataset.labels.items(): - print("\t{}: {}".format(label, value)) - else: - print("\tDataset has no labels defined.") - # [END bigquery_get_dataset_labels] - assert dataset.labels == labels - - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_ref = client.dataset('my_dataset') - # dataset = client.get_dataset(dataset_ref) # API request - - # This example dataset starts with one label - assert dataset.labels == {"color": "green"} - # To delete a label from a dataset, set its value to None - dataset.labels["color"] = None - - dataset = client.update_dataset(dataset, ["labels"]) # API request - - assert dataset.labels == {} - - # [END bigquery_delete_label_dataset] \ No newline at end of file diff --git a/bigquery/samples/tests/test_delete_dataset_labels.py b/bigquery/samples/tests/test_delete_dataset_labels.py new file mode 100644 index 000000000000..b5879746c32a --- /dev/null +++ b/bigquery/samples/tests/test_delete_dataset_labels.py @@ -0,0 +1,23 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import delete_dataset_labels + + +def test_delete_dataset_labels(capsys, client, dataset_id, dataset_label): + + delete_dataset_labels.delete_dataset_labels(client, dataset_id) + out, err = capsys.readouterr() + assert "Labels deleted." in out diff --git a/bigquery/samples/tests/test_get_dataset_labels.py b/bigquery/samples/tests/test_get_dataset_labels.py new file mode 100644 index 000000000000..6f08b4f14979 --- /dev/null +++ b/bigquery/samples/tests/test_get_dataset_labels.py @@ -0,0 +1,23 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import get_dataset_labels + + +def test_get_dataset_labels(capsys, client, dataset_id, dataset_label): + + get_dataset_labels.get_dataset_labels(client, dataset_id) + out, err = capsys.readouterr() + assert "color: green" in out diff --git a/bigquery/samples/tests/test_manage_dataset_labels.py b/bigquery/samples/tests/test_label_dataset.py similarity index 75% rename from bigquery/samples/tests/test_manage_dataset_labels.py rename to bigquery/samples/tests/test_label_dataset.py index ee740f3996b6..d0bd3bbbe38d 100644 --- a/bigquery/samples/tests/test_manage_dataset_labels.py +++ b/bigquery/samples/tests/test_label_dataset.py @@ -13,10 +13,12 @@ # limitations under the License. -from .. import manage_dataset_labels +from .. import label_dataset -def test_manage_dataset_labels(capsys, client): +def test_label_dataset(capsys, client, dataset_id): + labels = {"color": "green"} + label_dataset.label_dataset(client, dataset_id) out, err = capsys.readouterr() - assert \ No newline at end of file + assert "color: green" in out From 0de5d8b50b6d15f504c1290f36710daa94f5b5eb Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Mon, 12 Aug 2019 13:54:31 +0300 Subject: [PATCH 04/68] add_empty_column --- bigquery/docs/usage/tables.rst | 18 +++++------ bigquery/samples/add_empty_column.py | 32 +++++++++---------- .../samples/tests/test_add_empty_column.py | 10 ++---- 3 files changed, 28 insertions(+), 32 deletions(-) diff --git a/bigquery/docs/usage/tables.rst b/bigquery/docs/usage/tables.rst index 4bffe3fc11bb..5b81d97fd1f8 100644 --- a/bigquery/docs/usage/tables.rst +++ b/bigquery/docs/usage/tables.rst @@ -113,6 +113,15 @@ Insert rows into a table's data with the :start-after: [START bigquery_table_insert_rows] :end-before: [END bigquery_table_insert_rows] +Adds an empty column to the existing table with the +:func:`~google.cloud.bigquery.update_table` method: + +.. literalinclude:: ../samples/samples/add_empty_column.py + :language: python + :dedent: 4 + :start-after: [START bigquery_add_empty_column] + :end-before: [END bigquery_add_empty_column] + Copying a Table ^^^^^^^^^^^^^^^ @@ -149,15 +158,6 @@ Delete a table with the [-REPLACE_COMMENT-] :func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: -.. literalinclude:: ../samples/samples/add_empty_column.py - :language: python - :dedent: 4 - :start-after: [START bigquery_add_empty_column] - :end-before: [END bigquery_add_empty_column] - -[-REPLACE_COMMENT-] -:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: - .. literalinclude:: ../samples/samples/client_list_jobs.py :language: python :dedent: 4 diff --git a/bigquery/samples/add_empty_column.py b/bigquery/samples/add_empty_column.py index 29d6241f8591..c7d66a59263b 100644 --- a/bigquery/samples/add_empty_column.py +++ b/bigquery/samples/add_empty_column.py @@ -13,26 +13,23 @@ # limitations under the License. -def add_empty_column(client, to_delete): +def add_empty_column(client, table_id): # [START bigquery_add_empty_column] - """Adds an empty column to an existing table.""" - dataset_id = "add_empty_column_dataset_{}".format(_millis()) - table_id = "add_empty_column_table_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - dataset = client.create_dataset(dataset) - to_delete.append(dataset) + from google.cloud import bigquery - table = bigquery.Table(dataset.table(table_id), schema=SCHEMA) - table = client.create_table(table) + schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + ] - # from google.cloud import bigquery + # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() - # dataset_id = 'my_dataset' - # table_id = 'my_table' - table_ref = client.dataset(dataset_id).table(table_id) - table = client.get_table(table_ref) # API request + # TODO(developer): Set table_id to the ID of the table to add an empty column + # table_id = "your-project.your_dataset.your_table_name" + + table = client.get_table(table_id) original_schema = table.schema new_schema = original_schema[:] # creates a copy of the schema @@ -41,6 +38,9 @@ def add_empty_column(client, to_delete): table.schema = new_schema table = client.update_table(table, ["schema"]) # API request - assert len(table.schema) == len(original_schema) + 1 == len(new_schema) + if len(table.schema) == len(original_schema) + 1 == len(new_schema): + print("A new column has been added.") + else: + print("The column has not been added.") - # [END bigquery_add_empty_column] \ No newline at end of file + # [END bigquery_add_empty_column] diff --git a/bigquery/samples/tests/test_add_empty_column.py b/bigquery/samples/tests/test_add_empty_column.py index 8268954bf84c..e6c56e6cbfbc 100644 --- a/bigquery/samples/tests/test_add_empty_column.py +++ b/bigquery/samples/tests/test_add_empty_column.py @@ -16,12 +16,8 @@ from .. import add_empty_column -def test_add_empty_column(capsys, client): - - schema = [ - bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), - ] +def test_add_empty_column(capsys, client, table_id): + add_empty_column.add_empty_column(client, table_id) out, err = capsys.readouterr() - assert \ No newline at end of file + assert "A new column has been added." in out From 5e05cd81ed6fdfe6ee08e8f50b0f834dfb6edef1 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Mon, 12 Aug 2019 16:33:43 +0300 Subject: [PATCH 05/68] browse_table_data --- bigquery/docs/snippets.py | 6 ++-- bigquery/samples/browse_table_data.py | 34 +++++++++++-------- bigquery/samples/tests/conftest.py | 7 ++++ .../samples/tests/test_browse_table_data.py | 9 +++-- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/bigquery/docs/snippets.py b/bigquery/docs/snippets.py index 766ddc238102..b104e79ea789 100644 --- a/bigquery/docs/snippets.py +++ b/bigquery/docs/snippets.py @@ -23,11 +23,11 @@ need to be deleted during teardown. """ -# import os +import os import time import pytest -# import six +import six try: import fastparquet @@ -42,7 +42,7 @@ except (ImportError, AttributeError): pyarrow = None -# from google.api_core import datetime_helpers +from google.api_core import datetime_helpers from google.api_core.exceptions import InternalServerError from google.api_core.exceptions import ServiceUnavailable from google.api_core.exceptions import TooManyRequests diff --git a/bigquery/samples/browse_table_data.py b/bigquery/samples/browse_table_data.py index 35dda3d55582..0a5f9e4ffc24 100644 --- a/bigquery/samples/browse_table_data.py +++ b/bigquery/samples/browse_table_data.py @@ -13,31 +13,39 @@ # limitations under the License. -def browse_table_data(client, to_delete, capsys): +def browse_table_data(client, table_id): # [START bigquery_browse_table] """Retreive selected row data from a table.""" - # from google.cloud import bigquery + from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() - dataset_ref = client.dataset("samples", project="bigquery-public-data") - table_ref = dataset_ref.table("shakespeare") - table = client.get_table(table_ref) # API call + # TODO(developer): Set table_id to the ID of the table to browse data rows + # table_id = "your-project.your_dataset.your_table_name" + + table = client.get_table(table_id) # API call # Load all rows from a table rows = client.list_rows(table) - assert len(list(rows)) == table.num_rows + if len(list(rows)) == table.num_rows: + print("The amount of rows in the table = {}".format(table.num_rows)) # Load the first 10 rows rows = client.list_rows(table, max_results=10) - assert len(list(rows)) == 10 + number_of_rows = len(list(rows)) + if number_of_rows == 10: + print("First {} rows of the table are loaded".format(number_of_rows)) # Specify selected fields to limit the results to certain columns fields = table.schema[:2] # first two columns rows = client.list_rows(table, selected_fields=fields, max_results=10) - assert len(rows.schema) == 2 - assert len(list(rows)) == 10 + if len(rows.schema) == 2: + print("Fields number set to {}".format(len(rows.schema))) + if len(list(rows)) == 10: + print("{} rows of the table loaded".format(number_of_rows)) # Use the start index to load an arbitrary portion of the table rows = client.list_rows(table, start_index=10, max_results=10) @@ -46,11 +54,9 @@ def browse_table_data(client, to_delete, capsys): format_string = "{!s:<16} " * len(rows.schema) field_names = [field.name for field in rows.schema] print(format_string.format(*field_names)) # prints column headers - for row in rows: + for number_of_rows, row in enumerate(rows, 1): print(format_string.format(*row)) # prints row data - out, err = capsys.readouterr() - out = list(filter(bool, out.split("\n"))) # list of non-blank lines - assert len(out) == 11 + print("Printed data contains {} rows".format(number_of_rows)) - # [END bigquery_browse_table] \ No newline at end of file + # [END bigquery_browse_table] diff --git a/bigquery/samples/tests/conftest.py b/bigquery/samples/tests/conftest.py index 6156226ecb4b..1e5f1b76fda7 100644 --- a/bigquery/samples/tests/conftest.py +++ b/bigquery/samples/tests/conftest.py @@ -87,6 +87,13 @@ def table_id(client, dataset_id): client.delete_table(table, not_found_ok=True) +@pytest.fixture +def table_w_data(client): + dataset = client.get_dataset("bigquery-public-data.samples") + table = dataset.table("shakespeare") + return table + + @pytest.fixture def routine_id(client, dataset_id): now = datetime.datetime.now() diff --git a/bigquery/samples/tests/test_browse_table_data.py b/bigquery/samples/tests/test_browse_table_data.py index 43b7fb16fc42..2571bff057bc 100644 --- a/bigquery/samples/tests/test_browse_table_data.py +++ b/bigquery/samples/tests/test_browse_table_data.py @@ -16,7 +16,12 @@ from .. import browse_table_data -def test_browse_table_data(capsys, client): +def test_browse_table_data(capsys, client, table_w_data): + browse_table_data.browse_table_data(client, table_w_data) out, err = capsys.readouterr() - assert \ No newline at end of file + assert "The amount of rows in the table = " in out + assert "First 10 rows of the table are loaded" in out + assert "Fields number set to 2" + assert "List of non-blank lines:" + assert "Printed data contains 11 rows" From f6b60a080fa7c11e89f53a122970f04dd4e6f736 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Mon, 12 Aug 2019 18:34:00 +0300 Subject: [PATCH 06/68] dataset_exists --- bigquery/docs/usage/datasets.rst | 18 +++++------ bigquery/noxfile.py | 2 +- bigquery/samples/dataset_exists.py | 30 +++++++++++++------ .../samples/tests/test_browse_table_data.py | 6 ++-- bigquery/samples/tests/test_dataset_exists.py | 8 +++-- 5 files changed, 40 insertions(+), 24 deletions(-) diff --git a/bigquery/docs/usage/datasets.rst b/bigquery/docs/usage/datasets.rst index 314a593a90b5..5617105cfe2a 100644 --- a/bigquery/docs/usage/datasets.rst +++ b/bigquery/docs/usage/datasets.rst @@ -58,6 +58,15 @@ Create a new dataset with the :start-after: [START bigquery_create_dataset] :end-before: [END bigquery_create_dataset] +Determine if a dataset exists with the +:func:`~google.cloud.bigquery.client.Client.create_dataset` method: + +.. literalinclude:: ../samples/samples/dataset_exists.py + :language: python + :dedent: 4 + :start-after: [START bigquery_dataset_exists] + :end-before: [END bigquery_dataset_exists] + Updating a Dataset ^^^^^^^^^^^^^^^^^^ @@ -120,12 +129,3 @@ Delete a dataset with the :dedent: 4 :start-after: [START bigquery_delete_dataset] :end-before: [END bigquery_delete_dataset] - -[-REPLACE_COMMENT-] -:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: - -.. literalinclude:: ../samples/samples/dataset_exists.py - :language: python - :dedent: 4 - :start-after: [START bigquery_dataset_exists] - :end-before: [END bigquery_dataset_exists] diff --git a/bigquery/noxfile.py b/bigquery/noxfile.py index ce3249c78cb6..37611a5ce296 100644 --- a/bigquery/noxfile.py +++ b/bigquery/noxfile.py @@ -136,7 +136,7 @@ def cover(session): session.run("coverage", "erase") -@nox.session(python="3.7") +@nox.session(python="3.6") def lint(session): """Run linters. diff --git a/bigquery/samples/dataset_exists.py b/bigquery/samples/dataset_exists.py index ed09173ca89a..5fc82bdc3358 100644 --- a/bigquery/samples/dataset_exists.py +++ b/bigquery/samples/dataset_exists.py @@ -13,17 +13,29 @@ # limitations under the License. -def dataset_exists(client, to_delete): +def dataset_exists(client, dataset_id): # [START bigquery_dataset_exists] - """Determine if a dataset exists.""" - DATASET_ID = "get_table_dataset_{}".format(_millis()) - dataset_ref = client.dataset(DATASET_ID) - dataset = bigquery.Dataset(dataset_ref) + from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + # TODO(developer): Set dataset_id to the ID of the dataset to determine its existence. + # dataset_id = "{}.your_dataset".format(client.project) + + # Construct a full Dataset object to send to the API. + dataset = bigquery.Dataset(dataset_id) + + # TODO(developer): Specify the geographic location where the dataset should reside. + dataset.location = "US" + dataset = client.create_dataset(dataset) - to_delete.append(dataset) + print("Created dataset {}.{}".format(client.project, dataset.dataset_id)) - assert dataset_exists(client, dataset_ref) - assert not dataset_exists(client, client.dataset("i_dont_exist")) + try: + dataset = client.create_dataset(dataset) + except Exception as exp: + print(exp) - # [END bigquery_dataset_exists] \ No newline at end of file + # [END bigquery_dataset_exists] diff --git a/bigquery/samples/tests/test_browse_table_data.py b/bigquery/samples/tests/test_browse_table_data.py index 2571bff057bc..9c2bb6713348 100644 --- a/bigquery/samples/tests/test_browse_table_data.py +++ b/bigquery/samples/tests/test_browse_table_data.py @@ -22,6 +22,6 @@ def test_browse_table_data(capsys, client, table_w_data): out, err = capsys.readouterr() assert "The amount of rows in the table = " in out assert "First 10 rows of the table are loaded" in out - assert "Fields number set to 2" - assert "List of non-blank lines:" - assert "Printed data contains 11 rows" + assert "Fields number set to 2" in out + assert "List of non-blank lines:" in out + assert "Printed data contains 11 rows" in out diff --git a/bigquery/samples/tests/test_dataset_exists.py b/bigquery/samples/tests/test_dataset_exists.py index de4ad29de42f..9c54c863b6c4 100644 --- a/bigquery/samples/tests/test_dataset_exists.py +++ b/bigquery/samples/tests/test_dataset_exists.py @@ -16,7 +16,11 @@ from .. import dataset_exists -def test_dataset_exists(capsys, client): +def test_dataset_exists(capsys, client, random_dataset_id): + dataset_exists.dataset_exists(client, random_dataset_id) out, err = capsys.readouterr() - assert \ No newline at end of file + # print(out) + # raise ValueError + assert "Created dataset {}".format(random_dataset_id) in out + assert "Already Exists:" in out From a7732513bc0e37fdf7ee28e814b14a978ce68dc7 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Mon, 12 Aug 2019 19:28:49 +0300 Subject: [PATCH 07/68] complete change of dataset_exists --- bigquery/docs/usage/datasets.rst | 2 +- bigquery/samples/dataset_exists.py | 25 ++++++------------- bigquery/samples/tests/test_dataset_exists.py | 10 +++++--- 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/bigquery/docs/usage/datasets.rst b/bigquery/docs/usage/datasets.rst index 5617105cfe2a..27340e3543de 100644 --- a/bigquery/docs/usage/datasets.rst +++ b/bigquery/docs/usage/datasets.rst @@ -59,7 +59,7 @@ Create a new dataset with the :end-before: [END bigquery_create_dataset] Determine if a dataset exists with the -:func:`~google.cloud.bigquery.client.Client.create_dataset` method: +:func:`~google.cloud.bigquery.client.Client.get_dataset` method: .. literalinclude:: ../samples/samples/dataset_exists.py :language: python diff --git a/bigquery/samples/dataset_exists.py b/bigquery/samples/dataset_exists.py index 5fc82bdc3358..803e25020c32 100644 --- a/bigquery/samples/dataset_exists.py +++ b/bigquery/samples/dataset_exists.py @@ -16,26 +16,15 @@ def dataset_exists(client, dataset_id): # [START bigquery_dataset_exists] - from google.cloud import bigquery + from google.cloud.exceptions import NotFound - # TODO(developer): Construct a BigQuery client object. - # client = bigquery.Client() - - # TODO(developer): Set dataset_id to the ID of the dataset to determine its existence. - # dataset_id = "{}.your_dataset".format(client.project) - - # Construct a full Dataset object to send to the API. - dataset = bigquery.Dataset(dataset_id) - - # TODO(developer): Specify the geographic location where the dataset should reside. - dataset.location = "US" - - dataset = client.create_dataset(dataset) - print("Created dataset {}.{}".format(client.project, dataset.dataset_id)) + # TODO(developer): Set dataset_id to the ID of the dataset to determine existence. + # dataset_id = 'your-project.your_dataset' try: - dataset = client.create_dataset(dataset) - except Exception as exp: - print(exp) + client.get_dataset(dataset_id) + print("Dataset {} already exists".format(dataset_id)) + except NotFound: + print("Dataset {} not found".format(dataset_id)) # [END bigquery_dataset_exists] diff --git a/bigquery/samples/tests/test_dataset_exists.py b/bigquery/samples/tests/test_dataset_exists.py index 9c54c863b6c4..659aebb8a995 100644 --- a/bigquery/samples/tests/test_dataset_exists.py +++ b/bigquery/samples/tests/test_dataset_exists.py @@ -13,14 +13,16 @@ # limitations under the License. +from google.cloud import bigquery from .. import dataset_exists def test_dataset_exists(capsys, client, random_dataset_id): + dataset_exists.dataset_exists(client, random_dataset_id) + dataset = bigquery.Dataset(random_dataset_id) + dataset = client.create_dataset(dataset) dataset_exists.dataset_exists(client, random_dataset_id) out, err = capsys.readouterr() - # print(out) - # raise ValueError - assert "Created dataset {}".format(random_dataset_id) in out - assert "Already Exists:" in out + assert "Dataset {} not found".format(random_dataset_id) in out + assert "Dataset {} already exists".format(random_dataset_id) in out From 95dc8f8cab68273eb9c57db1a1fc318e24c03a40 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 13 Aug 2019 12:10:40 +0300 Subject: [PATCH 08/68] Update datasets.rst minor corrections, 'dataset_exists' moved to the 'Getting a Dataset' section --- bigquery/docs/usage/datasets.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bigquery/docs/usage/datasets.rst b/bigquery/docs/usage/datasets.rst index 27340e3543de..52c5d430919b 100644 --- a/bigquery/docs/usage/datasets.rst +++ b/bigquery/docs/usage/datasets.rst @@ -46,6 +46,15 @@ Get a dataset resource (to pick up changes made by another client) with the :start-after: [START bigquery_get_dataset] :end-before: [END bigquery_get_dataset] +Determine if a dataset exists with the +:func:`~google.cloud.bigquery.client.Client.get_dataset` method: + +.. literalinclude:: ../samples/samples/dataset_exists.py + :language: python + :dedent: 4 + :start-after: [START bigquery_dataset_exists] + :end-before: [END bigquery_dataset_exists] + Creating a Dataset ^^^^^^^^^^^^^^^^^^ @@ -58,15 +67,6 @@ Create a new dataset with the :start-after: [START bigquery_create_dataset] :end-before: [END bigquery_create_dataset] -Determine if a dataset exists with the -:func:`~google.cloud.bigquery.client.Client.get_dataset` method: - -.. literalinclude:: ../samples/samples/dataset_exists.py - :language: python - :dedent: 4 - :start-after: [START bigquery_dataset_exists] - :end-before: [END bigquery_dataset_exists] - Updating a Dataset ^^^^^^^^^^^^^^^^^^ From cf7576e33b78bfddea398be4ed890e138b676034 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 13 Aug 2019 13:02:02 +0300 Subject: [PATCH 09/68] cosmetic chgs --- bigquery/docs/usage/datasets.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery/docs/usage/datasets.rst b/bigquery/docs/usage/datasets.rst index 52c5d430919b..82d6d5425afd 100644 --- a/bigquery/docs/usage/datasets.rst +++ b/bigquery/docs/usage/datasets.rst @@ -28,7 +28,7 @@ List datasets for a project with the List datasets by label for a project with the :func:`~google.cloud.bigquery.client.Client.list_datasets` method: -.. literalinclude:: ../samples/samples/list_datasets_by_label.py +.. literalinclude:: ../samples/list_datasets_by_label.py :language: python :dedent: 4 :start-after: [START bigquery_list_datasets_by_label] From 0645e33a051b12ec7ce8dc3458c19a95933a7153 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 13 Aug 2019 13:31:59 +0300 Subject: [PATCH 10/68] client_list_jobs --- bigquery/docs/usage/jobs.rst | 11 ++++--- bigquery/samples/client_list_jobs.py | 30 ++++++++++--------- .../samples/tests/test_client_list_jobs.py | 6 +++- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/bigquery/docs/usage/jobs.rst b/bigquery/docs/usage/jobs.rst index 914d1d459ee7..c3dd71031bfc 100644 --- a/bigquery/docs/usage/jobs.rst +++ b/bigquery/docs/usage/jobs.rst @@ -1,9 +1,6 @@ Managing Jobs ~~~~~~~~~~~~~ -List jobs for a project -^^^^^^^^^^^^^^^^^^^^^^^ - Jobs describe actions performed on data in BigQuery tables: - Load data into a table @@ -11,7 +8,13 @@ Jobs describe actions performed on data in BigQuery tables: - Extract data from a table - Copy a table -.. literalinclude:: ../snippets.py +Listing jobs +^^^^^^^^^^^^ + +List jobs for a project with the +:func:`~google.cloud.bigquery.client.Client.list_jobs` method: + +.. literalinclude:: ../samples/client_list_jobs.py :language: python :dedent: 4 :start-after: [START bigquery_list_jobs] diff --git a/bigquery/samples/client_list_jobs.py b/bigquery/samples/client_list_jobs.py index d63c6a7c9b6c..d60c947cf099 100644 --- a/bigquery/samples/client_list_jobs.py +++ b/bigquery/samples/client_list_jobs.py @@ -16,36 +16,38 @@ def client_list_jobs(client): # [START bigquery_list_jobs] - """List jobs for a project.""" + from google.cloud import bigquery - # TODO(developer): Uncomment the lines below and replace with your values. - # from google.cloud import bigquery + # TODO(developer): Construct a BigQuery client object. # project = 'my_project' # replace with your project ID # client = bigquery.Client(project=project) - import datetime # List the 10 most recent jobs in reverse chronological order. # Omit the max_results parameter to list jobs from the past 6 months. print("Last 10 jobs:") - for job in client.list_jobs(max_results=10): # API request(s) - print(job.job_id) + for number, job in enumerate(client.list_jobs(max_results=10), 1): # API request(s) + print("{} - {}".format(number, job.job_id)) # The following are examples of additional optional parameters: + import datetime + # Use min_creation_time and/or max_creation_time to specify a time window. print("Jobs from the last ten minutes:") ten_mins_ago = datetime.datetime.utcnow() - datetime.timedelta(minutes=10) - for job in client.list_jobs(min_creation_time=ten_mins_ago): - print(job.job_id) + for number, job in enumerate(client.list_jobs(min_creation_time=ten_mins_ago), 1): + print("{} - {}".format(number, job.job_id)) # Use all_users to include jobs run by all users in the project. print("Last 10 jobs run by all users:") - for job in client.list_jobs(max_results=10, all_users=True): - print("{} run by user: {}".format(job.job_id, job.user_email)) + for number, job in enumerate(client.list_jobs(max_results=10, all_users=True), 1): + print("{} - {} run by user: {}".format(number, job.job_id, job.user_email)) # Use state_filter to filter by job state. - print("Jobs currently running:") - for job in client.list_jobs(state_filter="RUNNING"): - print(job.job_id) + print("Last 10 jobs done:") + for number, job in enumerate( + client.list_jobs(max_results=10, state_filter="DONE"), 1 + ): + print("{} - {}".format(number, job.job_id)) - # [END bigquery_list_jobs] \ No newline at end of file + # [END bigquery_list_jobs] diff --git a/bigquery/samples/tests/test_client_list_jobs.py b/bigquery/samples/tests/test_client_list_jobs.py index 265ccbe211ac..c31ab22b2058 100644 --- a/bigquery/samples/tests/test_client_list_jobs.py +++ b/bigquery/samples/tests/test_client_list_jobs.py @@ -14,9 +14,13 @@ from .. import client_list_jobs +from .. import create_job def test_client_list_jobs(capsys, client): + job = create_job.create_job(client) + client_list_jobs.client_list_jobs(client) + client.cancel_job(job.job_id, location="US") out, err = capsys.readouterr() - assert \ No newline at end of file + assert "1 - {}".format(job.job_id) in out From 92efd59dc108fa8a98e16a86a1fdf5e6f963cc3a Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 13 Aug 2019 15:13:51 +0300 Subject: [PATCH 11/68] client_query --- bigquery/docs/usage/queries.rst | 5 +++-- bigquery/samples/client_query.py | 10 +++++----- bigquery/samples/tests/test_client_query.py | 3 ++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/bigquery/docs/usage/queries.rst b/bigquery/docs/usage/queries.rst index fc77bb5b80cd..1f0720e47f1a 100644 --- a/bigquery/docs/usage/queries.rst +++ b/bigquery/docs/usage/queries.rst @@ -4,9 +4,10 @@ Running Queries Querying data ^^^^^^^^^^^^^ -Run a query and wait for it to finish: +Run a query and wait for it to finish with the +:func:`~google.cloud.bigquery.client.Client.query` method: -.. literalinclude:: ../snippets.py +.. literalinclude:: ../samples/client_query.py :language: python :dedent: 4 :start-after: [START bigquery_query] diff --git a/bigquery/samples/client_query.py b/bigquery/samples/client_query.py index b18e9c6e16f4..66ca39cea6f7 100644 --- a/bigquery/samples/client_query.py +++ b/bigquery/samples/client_query.py @@ -16,9 +16,9 @@ def client_query(client): # [START bigquery_query] - """Run a simple query.""" + from google.cloud import bigquery - # from google.cloud import bigquery + # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() query = ( @@ -34,7 +34,7 @@ def client_query(client): for row in query_job: # API request - fetches results # Row values can be accessed by field name or index - assert row[0] == row.name == row["name"] - print(row) + if row[0] == row.name == row["name"]: + print(row) - # [END bigquery_query] \ No newline at end of file + # [END bigquery_query] diff --git a/bigquery/samples/tests/test_client_query.py b/bigquery/samples/tests/test_client_query.py index 745f8a8bb777..dd7b424a795a 100644 --- a/bigquery/samples/tests/test_client_query.py +++ b/bigquery/samples/tests/test_client_query.py @@ -18,5 +18,6 @@ def test_client_query(capsys, client): + client_query.client_query(client) out, err = capsys.readouterr() - assert \ No newline at end of file + assert "Row" in out From 18148217bacd23e7dfbccd6bac6cc8d3715e897a Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 13 Aug 2019 17:12:43 +0300 Subject: [PATCH 12/68] copy_table --- bigquery/samples/copy_table.py | 40 +++++++++++++---------- bigquery/samples/tests/test_copy_table.py | 8 +++-- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/bigquery/samples/copy_table.py b/bigquery/samples/copy_table.py index 1e12b43a43cc..0956a80f9b31 100644 --- a/bigquery/samples/copy_table.py +++ b/bigquery/samples/copy_table.py @@ -13,35 +13,39 @@ # limitations under the License. -def copy_table(client, to_delete): +def copy_table(client, dataset_id, table_id): # [START bigquery_copy_table] - dataset_id = "copy_table_dataset_{}".format(_millis()) - dest_dataset = bigquery.Dataset(client.dataset(dataset_id)) - dest_dataset.location = "US" - dest_dataset = client.create_dataset(dest_dataset) - to_delete.append(dest_dataset) + from google.cloud import bigquery - # from google.cloud import bigquery + # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() - source_dataset = client.dataset("samples", project="bigquery-public-data") - source_table_ref = source_dataset.table("shakespeare") + # TODO(developer): Set dataset_id to the ID of the dataset where to copy a table. + # dataset_id = 'your-project.your_dataset' - # dataset_id = 'my_dataset' - dest_table_ref = client.dataset(dataset_id).table("destination_table") + # TODO(developer): Set table_id to the ID of the original table + # table_id = "your-project.your_dataset.your_table_name" + + dataset = client.get_dataset(dataset_id) + + orig_table = client.get_table(table_id) + + dest_table = dataset.table("destination_table") job = client.copy_table( - source_table_ref, - dest_table_ref, + orig_table, + dest_table, # Location must match that of the source and destination tables. location="US", - ) # API request + ) job.result() # Waits for job to complete. - assert job.state == "DONE" - dest_table = client.get_table(dest_table_ref) # API request - assert dest_table.num_rows > 0 + if job.state == "DONE": + print("Process complete") + dest_table = client.get_table(dest_table) + if dest_table.num_rows == orig_table.num_rows: + print("Copy of the table created") - # [END bigquery_copy_table] \ No newline at end of file + # [END bigquery_copy_table] diff --git a/bigquery/samples/tests/test_copy_table.py b/bigquery/samples/tests/test_copy_table.py index ddf157d0ecff..69b3ce90f450 100644 --- a/bigquery/samples/tests/test_copy_table.py +++ b/bigquery/samples/tests/test_copy_table.py @@ -16,7 +16,11 @@ from .. import copy_table -def test_copy_table(capsys, client): +def test_copy_table(capsys, client, dataset_id, table_w_data): + copy_table.copy_table(client, dataset_id, table_w_data) out, err = capsys.readouterr() - assert \ No newline at end of file + # print(out) + # raise ValueError + assert "Process complete" in out + assert "Copy of the table created" in out From a31bc1f2ee39f5d228e7a7a2d26b56c4d20808a9 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 13 Aug 2019 13:02:02 +0300 Subject: [PATCH 13/68] cosmetic chgs --- bigquery/docs/usage/datasets.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery/docs/usage/datasets.rst b/bigquery/docs/usage/datasets.rst index 27340e3543de..c07c581074b6 100644 --- a/bigquery/docs/usage/datasets.rst +++ b/bigquery/docs/usage/datasets.rst @@ -28,7 +28,7 @@ List datasets for a project with the List datasets by label for a project with the :func:`~google.cloud.bigquery.client.Client.list_datasets` method: -.. literalinclude:: ../samples/samples/list_datasets_by_label.py +.. literalinclude:: ../samples/list_datasets_by_label.py :language: python :dedent: 4 :start-after: [START bigquery_list_datasets_by_label] From 725c5158e952495924e4b8e926f7f78dbc91ed27 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 13 Aug 2019 12:10:40 +0300 Subject: [PATCH 14/68] Update datasets.rst minor corrections, 'dataset_exists' moved to the 'Getting a Dataset' section --- bigquery/docs/usage/datasets.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bigquery/docs/usage/datasets.rst b/bigquery/docs/usage/datasets.rst index c07c581074b6..82d6d5425afd 100644 --- a/bigquery/docs/usage/datasets.rst +++ b/bigquery/docs/usage/datasets.rst @@ -46,6 +46,15 @@ Get a dataset resource (to pick up changes made by another client) with the :start-after: [START bigquery_get_dataset] :end-before: [END bigquery_get_dataset] +Determine if a dataset exists with the +:func:`~google.cloud.bigquery.client.Client.get_dataset` method: + +.. literalinclude:: ../samples/samples/dataset_exists.py + :language: python + :dedent: 4 + :start-after: [START bigquery_dataset_exists] + :end-before: [END bigquery_dataset_exists] + Creating a Dataset ^^^^^^^^^^^^^^^^^^ @@ -58,15 +67,6 @@ Create a new dataset with the :start-after: [START bigquery_create_dataset] :end-before: [END bigquery_create_dataset] -Determine if a dataset exists with the -:func:`~google.cloud.bigquery.client.Client.get_dataset` method: - -.. literalinclude:: ../samples/samples/dataset_exists.py - :language: python - :dedent: 4 - :start-after: [START bigquery_dataset_exists] - :end-before: [END bigquery_dataset_exists] - Updating a Dataset ^^^^^^^^^^^^^^^^^^ From 6ffc123f2019a382cb43279f572b89b4a40eed8f Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 13 Aug 2019 17:38:31 +0300 Subject: [PATCH 15/68] cosmetic chgs grammar fix --- bigquery/samples/dataset_exists.py | 2 +- bigquery/samples/tests/test_dataset_exists.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bigquery/samples/dataset_exists.py b/bigquery/samples/dataset_exists.py index 803e25020c32..02703bb2f768 100644 --- a/bigquery/samples/dataset_exists.py +++ b/bigquery/samples/dataset_exists.py @@ -25,6 +25,6 @@ def dataset_exists(client, dataset_id): client.get_dataset(dataset_id) print("Dataset {} already exists".format(dataset_id)) except NotFound: - print("Dataset {} not found".format(dataset_id)) + print("Dataset {} is not found".format(dataset_id)) # [END bigquery_dataset_exists] diff --git a/bigquery/samples/tests/test_dataset_exists.py b/bigquery/samples/tests/test_dataset_exists.py index 659aebb8a995..591e462daa4b 100644 --- a/bigquery/samples/tests/test_dataset_exists.py +++ b/bigquery/samples/tests/test_dataset_exists.py @@ -24,5 +24,5 @@ def test_dataset_exists(capsys, client, random_dataset_id): dataset = client.create_dataset(dataset) dataset_exists.dataset_exists(client, random_dataset_id) out, err = capsys.readouterr() - assert "Dataset {} not found".format(random_dataset_id) in out + assert "Dataset {} is not found".format(random_dataset_id) in out assert "Dataset {} already exists".format(random_dataset_id) in out From 5654dcd4f780b123fe5b206a56aa3d8538e9d7c9 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 13 Aug 2019 17:38:31 +0300 Subject: [PATCH 16/68] cosmetic chgs grammar fix --- bigquery/samples/dataset_exists.py | 2 +- bigquery/samples/tests/test_dataset_exists.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bigquery/samples/dataset_exists.py b/bigquery/samples/dataset_exists.py index 803e25020c32..02703bb2f768 100644 --- a/bigquery/samples/dataset_exists.py +++ b/bigquery/samples/dataset_exists.py @@ -25,6 +25,6 @@ def dataset_exists(client, dataset_id): client.get_dataset(dataset_id) print("Dataset {} already exists".format(dataset_id)) except NotFound: - print("Dataset {} not found".format(dataset_id)) + print("Dataset {} is not found".format(dataset_id)) # [END bigquery_dataset_exists] diff --git a/bigquery/samples/tests/test_dataset_exists.py b/bigquery/samples/tests/test_dataset_exists.py index 659aebb8a995..591e462daa4b 100644 --- a/bigquery/samples/tests/test_dataset_exists.py +++ b/bigquery/samples/tests/test_dataset_exists.py @@ -24,5 +24,5 @@ def test_dataset_exists(capsys, client, random_dataset_id): dataset = client.create_dataset(dataset) dataset_exists.dataset_exists(client, random_dataset_id) out, err = capsys.readouterr() - assert "Dataset {} not found".format(random_dataset_id) in out + assert "Dataset {} is not found".format(random_dataset_id) in out assert "Dataset {} already exists".format(random_dataset_id) in out From 2ff376cb6c1a8b833fc07b1b84b00ccfb276c594 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 13 Aug 2019 18:08:41 +0300 Subject: [PATCH 17/68] table_exists --- bigquery/docs/usage/tables.rst | 18 ++++++++-------- bigquery/samples/table_exists.py | 24 +++++++++------------ bigquery/samples/tests/test_table_exists.py | 12 +++++++++-- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/bigquery/docs/usage/tables.rst b/bigquery/docs/usage/tables.rst index 5b81d97fd1f8..9ade46ec3595 100644 --- a/bigquery/docs/usage/tables.rst +++ b/bigquery/docs/usage/tables.rst @@ -28,6 +28,15 @@ Get a table resource with the :start-after: [START bigquery_get_table] :end-before: [END bigquery_get_table] +Determine if a table exists with the +:func:`~google.cloud.bigquery.client.Client.get_table` method: + +.. literalinclude:: ../samples/samples/table_exists.py + :language: python + :dedent: 4 + :start-after: [START bigquery_table_exists] + :end-before: [END bigquery_table_exists] + Browse data rows in a table with the :func:`~google.cloud.bigquery.client.Client.list_rows` method: @@ -545,15 +554,6 @@ Delete a table with the [-REPLACE_COMMENT-] :func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: -.. literalinclude:: ../samples/samples/table_exists.py - :language: python - :dedent: 4 - :start-after: [START bigquery_table_exists] - :end-before: [END bigquery_table_exists] - -[-REPLACE_COMMENT-] -:func:`~google.cloud.bigquery.[-REPLACE_METHOD-]` method: - .. literalinclude:: ../samples/samples/undelete_table.py :language: python :dedent: 4 diff --git a/bigquery/samples/table_exists.py b/bigquery/samples/table_exists.py index 66ecd216fb36..14051de42fa1 100644 --- a/bigquery/samples/table_exists.py +++ b/bigquery/samples/table_exists.py @@ -13,22 +13,18 @@ # limitations under the License. -def table_exists(client, to_delete): +def table_exists(client, table_id): # [START bigquery_table_exists] - """Determine if a table exists.""" + from google.cloud.exceptions import NotFound - DATASET_ID = "get_table_dataset_{}".format(_millis()) - TABLE_ID = "get_table_table_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(DATASET_ID)) - dataset = client.create_dataset(dataset) - to_delete.append(dataset) + # TODO(developer): Set table_id to the ID of the table to determine existence. + # table_id = 'your-project.your_dataset.your_table' - table_ref = dataset.table(TABLE_ID) - table = bigquery.Table(table_ref, schema=SCHEMA) - table = client.create_table(table) + try: + client.get_table(table_id) + print("Table {} already exists".format(table_id)) + except NotFound: + print("Table {} is not found".format(table_id)) - assert table_exists(client, table_ref) - assert not table_exists(client, dataset.table("i_dont_exist")) - - # [END bigquery_table_exists] \ No newline at end of file + # [END bigquery_table_exists] diff --git a/bigquery/samples/tests/test_table_exists.py b/bigquery/samples/tests/test_table_exists.py index e8baeae3a6ea..4872db5ae01f 100644 --- a/bigquery/samples/tests/test_table_exists.py +++ b/bigquery/samples/tests/test_table_exists.py @@ -13,15 +13,23 @@ # limitations under the License. +from google.cloud import bigquery from .. import table_exists -def test_table_exists(capsys, client): +def test_table_exists(capsys, client, random_table_id): schema = [ bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), ] + table_exists.table_exists(client, random_table_id) + table = bigquery.Table(random_table_id, schema=schema) + table = client.create_table(table) + table_exists.table_exists(client, random_table_id) out, err = capsys.readouterr() - assert \ No newline at end of file + # print(out) + # raise ValueError + assert "Table {} is not found".format(random_table_id) in out + assert "Table {} already exists".format(random_table_id) in out From 948ddb5ee31a3a287dc3167123c4aa5453f4e3d3 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 13 Aug 2019 18:09:19 +0300 Subject: [PATCH 18/68] Update test_copy_table.py minor corrections, removed extra comments --- bigquery/samples/tests/test_copy_table.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bigquery/samples/tests/test_copy_table.py b/bigquery/samples/tests/test_copy_table.py index 69b3ce90f450..ba71276596b5 100644 --- a/bigquery/samples/tests/test_copy_table.py +++ b/bigquery/samples/tests/test_copy_table.py @@ -20,7 +20,5 @@ def test_copy_table(capsys, client, dataset_id, table_w_data): copy_table.copy_table(client, dataset_id, table_w_data) out, err = capsys.readouterr() - # print(out) - # raise ValueError assert "Process complete" in out assert "Copy of the table created" in out From e8d62c4742453ea308cbe3ee3811d3fa1766928d Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 13 Aug 2019 18:10:25 +0300 Subject: [PATCH 19/68] Update test_table_exists.py minor corrections, removed extra comments --- bigquery/samples/tests/test_table_exists.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bigquery/samples/tests/test_table_exists.py b/bigquery/samples/tests/test_table_exists.py index 4872db5ae01f..32a02e5bbcbc 100644 --- a/bigquery/samples/tests/test_table_exists.py +++ b/bigquery/samples/tests/test_table_exists.py @@ -29,7 +29,5 @@ def test_table_exists(capsys, client, random_table_id): table = client.create_table(table) table_exists.table_exists(client, random_table_id) out, err = capsys.readouterr() - # print(out) - # raise ValueError assert "Table {} is not found".format(random_table_id) in out assert "Table {} already exists".format(random_table_id) in out From 28251ecb0bec416264c62d3f49f0cfc3309bf704 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 13 Aug 2019 18:13:35 +0300 Subject: [PATCH 20/68] Update snippets.py deleted 'dataset_exists' and 'table_exists' methods --- bigquery/docs/snippets.py | 42 --------------------------------------- 1 file changed, 42 deletions(-) diff --git a/bigquery/docs/snippets.py b/bigquery/docs/snippets.py index b104e79ea789..d05a383b7908 100644 --- a/bigquery/docs/snippets.py +++ b/bigquery/docs/snippets.py @@ -87,47 +87,5 @@ def delete(self): self._wrapped.close() -def dataset_exists(client, dataset_reference): - """Return if a dataset exists. - - Args: - client (google.cloud.bigquery.client.Client): - A client to connect to the BigQuery API. - dataset_reference (google.cloud.bigquery.dataset.DatasetReference): - A reference to the dataset to look for. - - Returns: - bool: ``True`` if the dataset exists, ``False`` otherwise. - """ - from google.cloud.exceptions import NotFound - - try: - client.get_dataset(dataset_reference) - return True - except NotFound: - return False - - -def table_exists(client, table_reference): - """Return if a table exists. - - Args: - client (google.cloud.bigquery.client.Client): - A client to connect to the BigQuery API. - table_reference (google.cloud.bigquery.table.TableReference): - A reference to the table to look for. - - Returns: - bool: ``True`` if the table exists, ``False`` otherwise. - """ - from google.cloud.exceptions import NotFound - - try: - client.get_table(table_reference) - return True - except NotFound: - return False - - if __name__ == "__main__": pytest.main() From ad623baf9007198607517dc61e7518f37ab7df26 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 14 Aug 2019 12:10:36 +0300 Subject: [PATCH 21/68] update client_list_jobs Added additional asserts into the test + refactoring of the main file. --- bigquery/samples/client_list_jobs.py | 18 ++++++++---------- .../samples/tests/test_client_list_jobs.py | 5 ++++- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/bigquery/samples/client_list_jobs.py b/bigquery/samples/client_list_jobs.py index d60c947cf099..ed9d9c7fbb8a 100644 --- a/bigquery/samples/client_list_jobs.py +++ b/bigquery/samples/client_list_jobs.py @@ -25,8 +25,8 @@ def client_list_jobs(client): # List the 10 most recent jobs in reverse chronological order. # Omit the max_results parameter to list jobs from the past 6 months. print("Last 10 jobs:") - for number, job in enumerate(client.list_jobs(max_results=10), 1): # API request(s) - print("{} - {}".format(number, job.job_id)) + for job in client.list_jobs(max_results=10): # API request(s) + print("{}".format(job.job_id)) # The following are examples of additional optional parameters: @@ -35,19 +35,17 @@ def client_list_jobs(client): # Use min_creation_time and/or max_creation_time to specify a time window. print("Jobs from the last ten minutes:") ten_mins_ago = datetime.datetime.utcnow() - datetime.timedelta(minutes=10) - for number, job in enumerate(client.list_jobs(min_creation_time=ten_mins_ago), 1): - print("{} - {}".format(number, job.job_id)) + for job in client.list_jobs(min_creation_time=ten_mins_ago): + print("{}".format(job.job_id)) # Use all_users to include jobs run by all users in the project. print("Last 10 jobs run by all users:") - for number, job in enumerate(client.list_jobs(max_results=10, all_users=True), 1): - print("{} - {} run by user: {}".format(number, job.job_id, job.user_email)) + for job in client.list_jobs(max_results=10, all_users=True): + print("{} run by user: {}".format(job.job_id, job.user_email)) # Use state_filter to filter by job state. print("Last 10 jobs done:") - for number, job in enumerate( - client.list_jobs(max_results=10, state_filter="DONE"), 1 - ): - print("{} - {}".format(number, job.job_id)) + for job in client.list_jobs(max_results=10, state_filter="DONE"): + print("{}".format(job.job_id)) # [END bigquery_list_jobs] diff --git a/bigquery/samples/tests/test_client_list_jobs.py b/bigquery/samples/tests/test_client_list_jobs.py index c31ab22b2058..8f558ed4cfc4 100644 --- a/bigquery/samples/tests/test_client_list_jobs.py +++ b/bigquery/samples/tests/test_client_list_jobs.py @@ -23,4 +23,7 @@ def test_client_list_jobs(capsys, client): client_list_jobs.client_list_jobs(client) client.cancel_job(job.job_id, location="US") out, err = capsys.readouterr() - assert "1 - {}".format(job.job_id) in out + assert "Last 10 jobs:\n{}".format(job.job_id) in out + assert "Jobs from the last ten minutes:\n{}".format(job.job_id) in out + assert "Last 10 jobs run by all users:\n{}".format(job.job_id) in out + assert "Last 10 jobs done:\n{}".format(job.job_id) in out From ec67aee26d55df43641cb151a7f864364a92d30c Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 14 Aug 2019 12:34:10 +0300 Subject: [PATCH 22/68] table_insert_rows --- bigquery/samples/table_insert_rows.py | 36 +++++++------------ .../samples/tests/test_table_insert_rows.py | 9 +++-- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/bigquery/samples/table_insert_rows.py b/bigquery/samples/table_insert_rows.py index cdbe2da43d71..072b947d1f0e 100644 --- a/bigquery/samples/table_insert_rows.py +++ b/bigquery/samples/table_insert_rows.py @@ -13,33 +13,23 @@ # limitations under the License. -def table_insert_rows(client, to_delete): +def table_insert_rows(client, table_id): # [START bigquery_table_insert_rows] - """Insert / fetch table data.""" - dataset_id = "table_insert_rows_dataset_{}".format(_millis()) - table_id = "table_insert_rows_table_{}".format(_millis()) - dataset = bigquery.Dataset(client.dataset(dataset_id)) - dataset = client.create_dataset(dataset) - dataset.location = "US" - to_delete.append(dataset) - - table = bigquery.Table(dataset.table(table_id), schema=SCHEMA) - table = client.create_table(table) - - # TODO(developer): Uncomment the lines below and replace with your values. - # from google.cloud import bigquery + from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() - # dataset_id = 'my_dataset' # replace with your dataset ID - # For this sample, the table must already exist and have a defined schema - # table_id = 'my_table' # replace with your table ID - # table_ref = client.dataset(dataset_id).table(table_id) - # table = client.get_table(table_ref) # API request - rows_to_insert = [(u"Phred Phlyntstone", 32), (u"Wylma Phlyntstone", 29)] + # TODO(developer): Set table_id to the ID of the model to fetch. + # table_id = 'your-project.your_dataset.your_table' - errors = client.insert_rows(table, rows_to_insert) # API request + table = client.get_table(table_id) + + rows_to_insert = [(u"Phred Phlyntstone", 32), (u"Wylma Phlyntstone", 29)] - assert errors == [] + errors = client.insert_rows(table, rows_to_insert) + if errors == []: + print("New rows have been added") - # [END bigquery_table_insert_rows] \ No newline at end of file + # [END bigquery_table_insert_rows] diff --git a/bigquery/samples/tests/test_table_insert_rows.py b/bigquery/samples/tests/test_table_insert_rows.py index cc3b3aa87a30..b92e6df71074 100644 --- a/bigquery/samples/tests/test_table_insert_rows.py +++ b/bigquery/samples/tests/test_table_insert_rows.py @@ -13,15 +13,20 @@ # limitations under the License. +from google.cloud import bigquery from .. import table_insert_rows -def test_table_insert_rows(capsys, client): +def test_table_insert_rows(capsys, client, random_table_id): schema = [ bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), ] + table = bigquery.Table(random_table_id, schema=schema) + table = client.create_table(table) + + table_insert_rows.table_insert_rows(client, random_table_id) out, err = capsys.readouterr() - assert \ No newline at end of file + assert "New rows have been added" in out From c8c0f70da945d4dcbdbbe598e62e9fc07a5e23e9 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 14 Aug 2019 12:36:27 +0300 Subject: [PATCH 23/68] Update client_query.py cosmetic chgs by 'black' --- bigquery/samples/client_query.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bigquery/samples/client_query.py b/bigquery/samples/client_query.py index 66ca39cea6f7..66e9138923ac 100644 --- a/bigquery/samples/client_query.py +++ b/bigquery/samples/client_query.py @@ -26,6 +26,7 @@ def client_query(client): 'WHERE state = "TX" ' "LIMIT 100" ) + query_job = client.query( query, # Location must match that of the dataset(s) referenced in the query. From febfa113415b6c901429fae4b14519cc2a1f0a72 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 14 Aug 2019 13:10:22 +0300 Subject: [PATCH 24/68] update client_query Chged an assertion parameter --- bigquery/samples/client_query.py | 3 +++ bigquery/samples/tests/test_client_query.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bigquery/samples/client_query.py b/bigquery/samples/client_query.py index 66e9138923ac..3af46fecbd17 100644 --- a/bigquery/samples/client_query.py +++ b/bigquery/samples/client_query.py @@ -33,6 +33,9 @@ def client_query(client): location="US", ) # API request - starts the query + if query_job: + print("The query completed") + for row in query_job: # API request - fetches results # Row values can be accessed by field name or index if row[0] == row.name == row["name"]: diff --git a/bigquery/samples/tests/test_client_query.py b/bigquery/samples/tests/test_client_query.py index dd7b424a795a..2e71e7dfb2e0 100644 --- a/bigquery/samples/tests/test_client_query.py +++ b/bigquery/samples/tests/test_client_query.py @@ -20,4 +20,4 @@ def test_client_query(capsys, client): client_query.client_query(client) out, err = capsys.readouterr() - assert "Row" in out + assert "The query completed" in out From 0e7ccf53341a086d07a4a4952be5b8a6d0abcb35 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 14 Aug 2019 13:51:55 +0300 Subject: [PATCH 25/68] *.rst files correction Deleted extra 'samples/' --- bigquery/docs/usage/datasets.rst | 8 ++++---- bigquery/docs/usage/tables.rst | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bigquery/docs/usage/datasets.rst b/bigquery/docs/usage/datasets.rst index 82d6d5425afd..2daee77f36d2 100644 --- a/bigquery/docs/usage/datasets.rst +++ b/bigquery/docs/usage/datasets.rst @@ -49,7 +49,7 @@ Get a dataset resource (to pick up changes made by another client) with the Determine if a dataset exists with the :func:`~google.cloud.bigquery.client.Client.get_dataset` method: -.. literalinclude:: ../samples/samples/dataset_exists.py +.. literalinclude:: ../samples/dataset_exists.py :language: python :dedent: 4 :start-after: [START bigquery_dataset_exists] @@ -94,7 +94,7 @@ Manage Dataset labels Add labels to a dataset with the :func:`~google.cloud.bigquery.client.Client.update_dataset` method: -.. literalinclude:: ../samples/samples/label_dataset.py +.. literalinclude:: ../samples/label_dataset.py :language: python :dedent: 4 :start-after: [START bigquery_label_dataset] @@ -103,7 +103,7 @@ Add labels to a dataset with the Get dataset's labels with the :func:`~google.cloud.bigquery.client.Client.get_dataset` method: -.. literalinclude:: ../samples/samples/get_dataset_labels.py +.. literalinclude:: ../samples/get_dataset_labels.py :language: python :dedent: 4 :start-after: [START bigquery_get_dataset_labels] @@ -112,7 +112,7 @@ Get dataset's labels with the Delete dataset's labels with the :func:`~google.cloud.bigquery.client.Client.update_dataset` method: -.. literalinclude:: ../samples/samples/delete_dataset_labels.py +.. literalinclude:: ../samples/delete_dataset_labels.py :language: python :dedent: 4 :start-after: [START bigquery_delete_label_dataset] diff --git a/bigquery/docs/usage/tables.rst b/bigquery/docs/usage/tables.rst index d59900585306..27fbb728ee73 100644 --- a/bigquery/docs/usage/tables.rst +++ b/bigquery/docs/usage/tables.rst @@ -116,7 +116,7 @@ Insert rows into a table's data with the Adds an empty column to the existing table with the :func:`~google.cloud.bigquery.update_table` method: -.. literalinclude:: ../samples/samples/add_empty_column.py +.. literalinclude:: ../samples/add_empty_column.py :language: python :dedent: 4 :start-after: [START bigquery_add_empty_column] From d5ab15d5754b3bf3b4b0d1dea3b0ccc312992dc3 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 14 Aug 2019 13:54:10 +0300 Subject: [PATCH 26/68] Update tables.rst Deleted extra 'samples/' --- bigquery/docs/usage/tables.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery/docs/usage/tables.rst b/bigquery/docs/usage/tables.rst index 28cab11cff03..b0d7f02e30e2 100644 --- a/bigquery/docs/usage/tables.rst +++ b/bigquery/docs/usage/tables.rst @@ -31,7 +31,7 @@ Get a table resource with the Determine if a table exists with the :func:`~google.cloud.bigquery.client.Client.get_table` method: -.. literalinclude:: ../samples/samples/table_exists.py +.. literalinclude:: ../samples/table_exists.py :language: python :dedent: 4 :start-after: [START bigquery_table_exists] From 7704ad3e8e42554511df4aef9a8e009bc372db29 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 14 Aug 2019 14:28:39 +0300 Subject: [PATCH 27/68] update copy_table chged assert implementation --- bigquery/samples/copy_table.py | 4 ++-- bigquery/samples/tests/test_copy_table.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bigquery/samples/copy_table.py b/bigquery/samples/copy_table.py index 0956a80f9b31..9d598c3c06b7 100644 --- a/bigquery/samples/copy_table.py +++ b/bigquery/samples/copy_table.py @@ -43,9 +43,9 @@ def copy_table(client, dataset_id, table_id): job.result() # Waits for job to complete. if job.state == "DONE": - print("Process complete") + print("The process completed") dest_table = client.get_table(dest_table) if dest_table.num_rows == orig_table.num_rows: - print("Copy of the table created") + print("A copy of the table created") # [END bigquery_copy_table] diff --git a/bigquery/samples/tests/test_copy_table.py b/bigquery/samples/tests/test_copy_table.py index ba71276596b5..d685e9f510fa 100644 --- a/bigquery/samples/tests/test_copy_table.py +++ b/bigquery/samples/tests/test_copy_table.py @@ -20,5 +20,5 @@ def test_copy_table(capsys, client, dataset_id, table_w_data): copy_table.copy_table(client, dataset_id, table_w_data) out, err = capsys.readouterr() - assert "Process complete" in out - assert "Copy of the table created" in out + assert "The process completed" in out + assert "A copy of the table created" in out From 6dd1174090acd052895f97f6409b0054d7d331ef Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 14 Aug 2019 15:33:06 +0300 Subject: [PATCH 28/68] update first-five chged quotes + added dots --- bigquery/samples/add_empty_column.py | 2 +- bigquery/samples/browse_table_data.py | 2 +- bigquery/samples/dataset_exists.py | 2 +- bigquery/samples/delete_dataset_labels.py | 2 +- bigquery/samples/get_dataset_labels.py | 2 +- bigquery/samples/label_dataset.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bigquery/samples/add_empty_column.py b/bigquery/samples/add_empty_column.py index c7d66a59263b..b53de62a1f87 100644 --- a/bigquery/samples/add_empty_column.py +++ b/bigquery/samples/add_empty_column.py @@ -26,7 +26,7 @@ def add_empty_column(client, table_id): # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() - # TODO(developer): Set table_id to the ID of the table to add an empty column + # TODO(developer): Set table_id to the ID of the table to add an empty column. # table_id = "your-project.your_dataset.your_table_name" table = client.get_table(table_id) diff --git a/bigquery/samples/browse_table_data.py b/bigquery/samples/browse_table_data.py index 0a5f9e4ffc24..1885fc7bf405 100644 --- a/bigquery/samples/browse_table_data.py +++ b/bigquery/samples/browse_table_data.py @@ -23,7 +23,7 @@ def browse_table_data(client, table_id): # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() - # TODO(developer): Set table_id to the ID of the table to browse data rows + # TODO(developer): Set table_id to the ID of the table to browse data rows. # table_id = "your-project.your_dataset.your_table_name" table = client.get_table(table_id) # API call diff --git a/bigquery/samples/dataset_exists.py b/bigquery/samples/dataset_exists.py index 02703bb2f768..fba822564c1b 100644 --- a/bigquery/samples/dataset_exists.py +++ b/bigquery/samples/dataset_exists.py @@ -19,7 +19,7 @@ def dataset_exists(client, dataset_id): from google.cloud.exceptions import NotFound # TODO(developer): Set dataset_id to the ID of the dataset to determine existence. - # dataset_id = 'your-project.your_dataset' + # dataset_id = "your-project.your_dataset" try: client.get_dataset(dataset_id) diff --git a/bigquery/samples/delete_dataset_labels.py b/bigquery/samples/delete_dataset_labels.py index 432619af7a1c..a049cca92507 100644 --- a/bigquery/samples/delete_dataset_labels.py +++ b/bigquery/samples/delete_dataset_labels.py @@ -22,7 +22,7 @@ def delete_dataset_labels(client, dataset_id): # client = bigquery.Client() # TODO(developer): Set dataset_id to the ID of the dataset to fetch. - # dataset_id = 'your-project.your_dataset' + # dataset_id = "your-project.your_dataset" dataset = client.get_dataset(dataset_id) diff --git a/bigquery/samples/get_dataset_labels.py b/bigquery/samples/get_dataset_labels.py index ee9309ba25a0..8517d9a2417c 100644 --- a/bigquery/samples/get_dataset_labels.py +++ b/bigquery/samples/get_dataset_labels.py @@ -22,7 +22,7 @@ def get_dataset_labels(client, dataset_id): # client = bigquery.Client() # TODO(developer): Set dataset_id to the ID of the dataset to fetch. - # dataset_id = 'your-project.your_dataset' + # dataset_id = "your-project.your_dataset" dataset = client.get_dataset(dataset_id) diff --git a/bigquery/samples/label_dataset.py b/bigquery/samples/label_dataset.py index 57e10ca0a59c..19aaa333ab94 100644 --- a/bigquery/samples/label_dataset.py +++ b/bigquery/samples/label_dataset.py @@ -22,7 +22,7 @@ def label_dataset(client, dataset_id): # client = bigquery.Client() # TODO(developer): Set dataset_id to the ID of the dataset to fetch. - # dataset_id = 'your-project.your_dataset' + # dataset_id = "your-project.your_dataset" dataset = client.get_dataset(dataset_id) From 50c4aedbdeb485fc49ce684e8f4317bae7211b3c Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 14 Aug 2019 15:36:38 +0300 Subject: [PATCH 29/68] update second-five chged quotes + added dots --- bigquery/samples/client_list_jobs.py | 2 +- bigquery/samples/copy_table.py | 4 ++-- bigquery/samples/table_exists.py | 2 +- bigquery/samples/table_insert_rows.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bigquery/samples/client_list_jobs.py b/bigquery/samples/client_list_jobs.py index ed9d9c7fbb8a..048a37ef2170 100644 --- a/bigquery/samples/client_list_jobs.py +++ b/bigquery/samples/client_list_jobs.py @@ -19,7 +19,7 @@ def client_list_jobs(client): from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. - # project = 'my_project' # replace with your project ID + # project = "my_project" # replace with your project ID # client = bigquery.Client(project=project) # List the 10 most recent jobs in reverse chronological order. diff --git a/bigquery/samples/copy_table.py b/bigquery/samples/copy_table.py index 9d598c3c06b7..4b721c2b68ca 100644 --- a/bigquery/samples/copy_table.py +++ b/bigquery/samples/copy_table.py @@ -22,9 +22,9 @@ def copy_table(client, dataset_id, table_id): # client = bigquery.Client() # TODO(developer): Set dataset_id to the ID of the dataset where to copy a table. - # dataset_id = 'your-project.your_dataset' + # dataset_id = "your-project.your_dataset" - # TODO(developer): Set table_id to the ID of the original table + # TODO(developer): Set table_id to the ID of the original table. # table_id = "your-project.your_dataset.your_table_name" dataset = client.get_dataset(dataset_id) diff --git a/bigquery/samples/table_exists.py b/bigquery/samples/table_exists.py index 14051de42fa1..3b3d9b461f08 100644 --- a/bigquery/samples/table_exists.py +++ b/bigquery/samples/table_exists.py @@ -19,7 +19,7 @@ def table_exists(client, table_id): from google.cloud.exceptions import NotFound # TODO(developer): Set table_id to the ID of the table to determine existence. - # table_id = 'your-project.your_dataset.your_table' + # table_id = "your-project.your_dataset.your_table" try: client.get_table(table_id) diff --git a/bigquery/samples/table_insert_rows.py b/bigquery/samples/table_insert_rows.py index 072b947d1f0e..ccc14c5520b9 100644 --- a/bigquery/samples/table_insert_rows.py +++ b/bigquery/samples/table_insert_rows.py @@ -22,7 +22,7 @@ def table_insert_rows(client, table_id): # client = bigquery.Client() # TODO(developer): Set table_id to the ID of the model to fetch. - # table_id = 'your-project.your_dataset.your_table' + # table_id = "your-project.your_dataset.your_table" table = client.get_table(table_id) From 7a654c31e47f49cbcd05d18d77b8cfe955993d11 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 14 Aug 2019 17:04:39 +0300 Subject: [PATCH 30/68] Update test_dataset_exists.py import updates --- bigquery/samples/tests/test_dataset_exists.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bigquery/samples/tests/test_dataset_exists.py b/bigquery/samples/tests/test_dataset_exists.py index 591e462daa4b..314c1db9b29c 100644 --- a/bigquery/samples/tests/test_dataset_exists.py +++ b/bigquery/samples/tests/test_dataset_exists.py @@ -14,6 +14,7 @@ from google.cloud import bigquery + from .. import dataset_exists From 8e261157a1b667fda4f4553838fb375996c99d39 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 14 Aug 2019 17:06:22 +0300 Subject: [PATCH 31/68] update imports --- bigquery/samples/tests/test_table_exists.py | 1 + bigquery/samples/tests/test_table_insert_rows.py | 1 + 2 files changed, 2 insertions(+) diff --git a/bigquery/samples/tests/test_table_exists.py b/bigquery/samples/tests/test_table_exists.py index 32a02e5bbcbc..77ecf050d5aa 100644 --- a/bigquery/samples/tests/test_table_exists.py +++ b/bigquery/samples/tests/test_table_exists.py @@ -14,6 +14,7 @@ from google.cloud import bigquery + from .. import table_exists diff --git a/bigquery/samples/tests/test_table_insert_rows.py b/bigquery/samples/tests/test_table_insert_rows.py index b92e6df71074..f9e04bccb37f 100644 --- a/bigquery/samples/tests/test_table_insert_rows.py +++ b/bigquery/samples/tests/test_table_insert_rows.py @@ -14,6 +14,7 @@ from google.cloud import bigquery + from .. import table_insert_rows From 0cf0071b1e0c845196aeaae89278959c1ca09960 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 14 Aug 2019 18:40:18 +0300 Subject: [PATCH 32/68] update list_datasets_by_label Chged assertion unit --- bigquery/samples/list_datasets_by_label.py | 2 +- bigquery/samples/tests/test_list_datasets_by_label.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/bigquery/samples/list_datasets_by_label.py b/bigquery/samples/list_datasets_by_label.py index 26eb038c59e2..d9aa38368f5a 100644 --- a/bigquery/samples/list_datasets_by_label.py +++ b/bigquery/samples/list_datasets_by_label.py @@ -27,7 +27,7 @@ def list_datasets_by_label(client): if datasets: print("Datasets filtered by {}:".format(label_filter)) for dataset in datasets: # API request(s) - print("\t{}".format(dataset.dataset_id)) + print("\t{}.{}".format(dataset.project, dataset.dataset_id)) else: print("No datasets found with this filter.") diff --git a/bigquery/samples/tests/test_list_datasets_by_label.py b/bigquery/samples/tests/test_list_datasets_by_label.py index 80fbf641d409..a5f1aea068a4 100644 --- a/bigquery/samples/tests/test_list_datasets_by_label.py +++ b/bigquery/samples/tests/test_list_datasets_by_label.py @@ -16,9 +16,8 @@ from .. import list_datasets_by_label -def test_list_datasets_by_label(capsys, client, dataset_id, dataset_label): +def test_list_datasets_by_label(capsys, client, dataset_label): - label_filter = "labels.color:green" list_datasets_by_label.list_datasets_by_label(client) out, err = capsys.readouterr() - assert "Datasets filtered by {}:".format(label_filter) in out + assert "{}".format(dataset_label) in out From f602e3f50047ea3fabfeb74834fc461b467c8012 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Thu, 15 Aug 2019 12:14:23 +0300 Subject: [PATCH 33/68] Update test_browse_table_data.py corrected the test asserts --- bigquery/samples/tests/test_browse_table_data.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bigquery/samples/tests/test_browse_table_data.py b/bigquery/samples/tests/test_browse_table_data.py index 9c2bb6713348..7931f62c35aa 100644 --- a/bigquery/samples/tests/test_browse_table_data.py +++ b/bigquery/samples/tests/test_browse_table_data.py @@ -23,5 +23,4 @@ def test_browse_table_data(capsys, client, table_w_data): assert "The amount of rows in the table = " in out assert "First 10 rows of the table are loaded" in out assert "Fields number set to 2" in out - assert "List of non-blank lines:" in out - assert "Printed data contains 11 rows" in out + assert "Printed data contains 10 rows" in out From 0bf9aab831d1be1b23ed0702b956d3fb8d5aff2e Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Thu, 15 Aug 2019 16:34:05 +0300 Subject: [PATCH 34/68] The update of the first-five according to the requested chgs * deleted unnecessary schema (add_empty_column) * added 'get_dataset' method to check that dataset actually was updated (label_dataset & delete_dataset_labels) * chged name of the fixture on more suitable (conftest & test_delete_dataset_labels & test_get_dataset_labels & list_datasets_by_label) * deleted unnecessary 'labels' variable (test_label_dataset) --- bigquery/samples/add_empty_column.py | 5 ----- bigquery/samples/delete_dataset_labels.py | 4 +++- bigquery/samples/label_dataset.py | 5 +++-- bigquery/samples/tests/conftest.py | 2 +- bigquery/samples/tests/test_delete_dataset_labels.py | 2 +- bigquery/samples/tests/test_get_dataset_labels.py | 2 +- bigquery/samples/tests/test_label_dataset.py | 1 - bigquery/samples/tests/test_list_datasets_by_label.py | 4 ++-- 8 files changed, 11 insertions(+), 14 deletions(-) diff --git a/bigquery/samples/add_empty_column.py b/bigquery/samples/add_empty_column.py index b53de62a1f87..68237ff68c93 100644 --- a/bigquery/samples/add_empty_column.py +++ b/bigquery/samples/add_empty_column.py @@ -18,11 +18,6 @@ def add_empty_column(client, table_id): # [START bigquery_add_empty_column] from google.cloud import bigquery - schema = [ - bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), - ] - # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() diff --git a/bigquery/samples/delete_dataset_labels.py b/bigquery/samples/delete_dataset_labels.py index a049cca92507..1c118e3bec2f 100644 --- a/bigquery/samples/delete_dataset_labels.py +++ b/bigquery/samples/delete_dataset_labels.py @@ -29,7 +29,9 @@ def delete_dataset_labels(client, dataset_id): # To delete a label from a dataset, set its value to None dataset.labels["color"] = None - dataset = client.update_dataset(dataset, ["labels"]) # API request + dataset = client.update_dataset(dataset, ["labels"]) + + dataset = client.get_dataset(dataset_id) print("Dataset ID: {}".format(dataset_id)) if dataset.labels: diff --git a/bigquery/samples/label_dataset.py b/bigquery/samples/label_dataset.py index 19aaa333ab94..38d5e654304f 100644 --- a/bigquery/samples/label_dataset.py +++ b/bigquery/samples/label_dataset.py @@ -26,11 +26,12 @@ def label_dataset(client, dataset_id): dataset = client.get_dataset(dataset_id) - labels = {"color": "green"} - dataset.labels = labels + dataset.labels = {"color": "green"} dataset = client.update_dataset(dataset, ["labels"]) + dataset = client.get_dataset(dataset_id) + print("Dataset ID: {}".format(dataset_id)) print("Added labels:") if dataset.labels: diff --git a/bigquery/samples/tests/conftest.py b/bigquery/samples/tests/conftest.py index 1e5f1b76fda7..93725c0a26dc 100644 --- a/bigquery/samples/tests/conftest.py +++ b/bigquery/samples/tests/conftest.py @@ -66,7 +66,7 @@ def dataset_id(client): @pytest.fixture -def dataset_label(client, dataset_id): +def dataset_with_labels_id(client, dataset_id): dataset = client.get_dataset(dataset_id) dataset.labels = {"color": "green"} dataset = client.update_dataset(dataset, ["labels"]) diff --git a/bigquery/samples/tests/test_delete_dataset_labels.py b/bigquery/samples/tests/test_delete_dataset_labels.py index b5879746c32a..45ce61e1d96c 100644 --- a/bigquery/samples/tests/test_delete_dataset_labels.py +++ b/bigquery/samples/tests/test_delete_dataset_labels.py @@ -16,7 +16,7 @@ from .. import delete_dataset_labels -def test_delete_dataset_labels(capsys, client, dataset_id, dataset_label): +def test_delete_dataset_labels(capsys, client, dataset_id, dataset_with_labels_id): delete_dataset_labels.delete_dataset_labels(client, dataset_id) out, err = capsys.readouterr() diff --git a/bigquery/samples/tests/test_get_dataset_labels.py b/bigquery/samples/tests/test_get_dataset_labels.py index 6f08b4f14979..bd55b7c8a884 100644 --- a/bigquery/samples/tests/test_get_dataset_labels.py +++ b/bigquery/samples/tests/test_get_dataset_labels.py @@ -16,7 +16,7 @@ from .. import get_dataset_labels -def test_get_dataset_labels(capsys, client, dataset_id, dataset_label): +def test_get_dataset_labels(capsys, client, dataset_id, dataset_with_labels_id): get_dataset_labels.get_dataset_labels(client, dataset_id) out, err = capsys.readouterr() diff --git a/bigquery/samples/tests/test_label_dataset.py b/bigquery/samples/tests/test_label_dataset.py index d0bd3bbbe38d..1ecc6a975f4c 100644 --- a/bigquery/samples/tests/test_label_dataset.py +++ b/bigquery/samples/tests/test_label_dataset.py @@ -18,7 +18,6 @@ def test_label_dataset(capsys, client, dataset_id): - labels = {"color": "green"} label_dataset.label_dataset(client, dataset_id) out, err = capsys.readouterr() assert "color: green" in out diff --git a/bigquery/samples/tests/test_list_datasets_by_label.py b/bigquery/samples/tests/test_list_datasets_by_label.py index a5f1aea068a4..ed6b6236d699 100644 --- a/bigquery/samples/tests/test_list_datasets_by_label.py +++ b/bigquery/samples/tests/test_list_datasets_by_label.py @@ -16,8 +16,8 @@ from .. import list_datasets_by_label -def test_list_datasets_by_label(capsys, client, dataset_label): +def test_list_datasets_by_label(capsys, client, dataset_with_labels_id): list_datasets_by_label.list_datasets_by_label(client) out, err = capsys.readouterr() - assert "{}".format(dataset_label) in out + assert "{}".format(dataset_with_labels_id) in out From 2e50a814835987b9c602995b8472d865c51dd9ef Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Thu, 15 Aug 2019 19:10:46 +0300 Subject: [PATCH 35/68] snippets file reformatted --- bigquery/docs/snippets.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/bigquery/docs/snippets.py b/bigquery/docs/snippets.py index 9dd339165c0c..123e1fd29386 100644 --- a/bigquery/docs/snippets.py +++ b/bigquery/docs/snippets.py @@ -485,14 +485,6 @@ def test_update_table_expiration(client, to_delete): # [END bigquery_update_table_expiration] -@pytest.mark.skip( - reason=( - "update_table() is flaky " - "https://github.com/GoogleCloudPlatform/google-cloud-python/issues/5589" - ) -) - - @pytest.mark.skip( reason=( "update_table() is flaky " @@ -581,8 +573,6 @@ def test_update_table_cmek(client, to_delete): # [END bigquery_update_table_cmek] - - @pytest.mark.skip( reason=( "update_table() is flaky " From 9e77624ac37ee616ef310d466122593b8029cac1 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 16 Aug 2019 13:42:39 +0300 Subject: [PATCH 36/68] Update tables.rst --- bigquery/docs/usage/tables.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery/docs/usage/tables.rst b/bigquery/docs/usage/tables.rst index 27fbb728ee73..458c5b0009ba 100644 --- a/bigquery/docs/usage/tables.rst +++ b/bigquery/docs/usage/tables.rst @@ -113,7 +113,7 @@ Insert rows into a table's data with the :start-after: [START bigquery_table_insert_rows] :end-before: [END bigquery_table_insert_rows] -Adds an empty column to the existing table with the +Add an empty column to the existing table with the :func:`~google.cloud.bigquery.update_table` method: .. literalinclude:: ../samples/add_empty_column.py From 7d996735c0226e92421c2bf54b01774582030143 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Mon, 19 Aug 2019 13:55:17 +0300 Subject: [PATCH 37/68] update_client_query * refigured the condition when printing results --- bigquery/samples/client_query.py | 13 ++++++------- bigquery/samples/tests/test_client_query.py | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/bigquery/samples/client_query.py b/bigquery/samples/client_query.py index 3af46fecbd17..f394e3a2ba1e 100644 --- a/bigquery/samples/client_query.py +++ b/bigquery/samples/client_query.py @@ -33,12 +33,11 @@ def client_query(client): location="US", ) # API request - starts the query - if query_job: - print("The query completed") - - for row in query_job: # API request - fetches results - # Row values can be accessed by field name or index - if row[0] == row.name == row["name"]: - print(row) + if query_job.state == "RUNNING": + print("The query data:") + for row in query_job: # API request - fetches results + # Row values can be accessed by field name or index + if row[0] == row.name == row["name"]: + print(row) # [END bigquery_query] diff --git a/bigquery/samples/tests/test_client_query.py b/bigquery/samples/tests/test_client_query.py index 2e71e7dfb2e0..fc53ca73e6e2 100644 --- a/bigquery/samples/tests/test_client_query.py +++ b/bigquery/samples/tests/test_client_query.py @@ -20,4 +20,4 @@ def test_client_query(capsys, client): client_query.client_query(client) out, err = capsys.readouterr() - assert "The query completed" in out + assert "The query data:" in out From e37f5a2e30c3b31d3820f66d5da41fe3978bb6ae Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 23 Aug 2019 14:03:44 +0300 Subject: [PATCH 38/68] update manage_dataset_labels --- bigquery/samples/delete_dataset_labels.py | 13 ++--------- bigquery/samples/get_dataset_labels.py | 2 +- bigquery/samples/label_dataset.py | 15 ++---------- bigquery/samples/tests/conftest.py | 9 -------- ...abels.py => test_dataset_label_samples.py} | 13 +++++++++-- .../tests/test_delete_dataset_labels.py | 23 ------------------- bigquery/samples/tests/test_label_dataset.py | 23 ------------------- 7 files changed, 16 insertions(+), 82 deletions(-) rename bigquery/samples/tests/{test_get_dataset_labels.py => test_dataset_label_samples.py} (62%) delete mode 100644 bigquery/samples/tests/test_delete_dataset_labels.py delete mode 100644 bigquery/samples/tests/test_label_dataset.py diff --git a/bigquery/samples/delete_dataset_labels.py b/bigquery/samples/delete_dataset_labels.py index 1c118e3bec2f..e0f8492ee944 100644 --- a/bigquery/samples/delete_dataset_labels.py +++ b/bigquery/samples/delete_dataset_labels.py @@ -16,7 +16,7 @@ def delete_dataset_labels(client, dataset_id): # [START bigquery_delete_label_dataset] - from google.cloud import bigquery + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() @@ -30,14 +30,5 @@ def delete_dataset_labels(client, dataset_id): dataset.labels["color"] = None dataset = client.update_dataset(dataset, ["labels"]) - - dataset = client.get_dataset(dataset_id) - - print("Dataset ID: {}".format(dataset_id)) - if dataset.labels: - for label, value in dataset.labels.items(): - print("\t{}: {}".format(label, value)) - else: - print("\tLabels deleted.") - + print("Labels deleted from {}".format(dataset_id)) # [END bigquery_delete_label_dataset] diff --git a/bigquery/samples/get_dataset_labels.py b/bigquery/samples/get_dataset_labels.py index 8517d9a2417c..94c7c203501e 100644 --- a/bigquery/samples/get_dataset_labels.py +++ b/bigquery/samples/get_dataset_labels.py @@ -16,7 +16,7 @@ def get_dataset_labels(client, dataset_id): # [START bigquery_get_dataset_labels] - from google.cloud import bigquery + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() diff --git a/bigquery/samples/label_dataset.py b/bigquery/samples/label_dataset.py index 38d5e654304f..5d48001791f2 100644 --- a/bigquery/samples/label_dataset.py +++ b/bigquery/samples/label_dataset.py @@ -16,7 +16,7 @@ def label_dataset(client, dataset_id): # [START bigquery_label_dataset] - from google.cloud import bigquery + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() @@ -25,19 +25,8 @@ def label_dataset(client, dataset_id): # dataset_id = "your-project.your_dataset" dataset = client.get_dataset(dataset_id) - dataset.labels = {"color": "green"} - dataset = client.update_dataset(dataset, ["labels"]) - dataset = client.get_dataset(dataset_id) - - print("Dataset ID: {}".format(dataset_id)) - print("Added labels:") - if dataset.labels: - for label, value in dataset.labels.items(): - print("\t{}: {}".format(label, value)) - else: - print("\tNo labels defined.") - + print("Labels added to {}".format(dataset_id)) # [END bigquery_label_dataset] diff --git a/bigquery/samples/tests/conftest.py b/bigquery/samples/tests/conftest.py index 93725c0a26dc..034607b03e87 100644 --- a/bigquery/samples/tests/conftest.py +++ b/bigquery/samples/tests/conftest.py @@ -65,15 +65,6 @@ def dataset_id(client): client.delete_dataset(dataset, delete_contents=True, not_found_ok=True) -@pytest.fixture -def dataset_with_labels_id(client, dataset_id): - dataset = client.get_dataset(dataset_id) - dataset.labels = {"color": "green"} - dataset = client.update_dataset(dataset, ["labels"]) - yield "{}.{}".format(dataset.project, dataset.dataset_id) - client.delete_dataset(dataset, delete_contents=True, not_found_ok=True) - - @pytest.fixture def table_id(client, dataset_id): now = datetime.datetime.now() diff --git a/bigquery/samples/tests/test_get_dataset_labels.py b/bigquery/samples/tests/test_dataset_label_samples.py similarity index 62% rename from bigquery/samples/tests/test_get_dataset_labels.py rename to bigquery/samples/tests/test_dataset_label_samples.py index bd55b7c8a884..9e8091302b70 100644 --- a/bigquery/samples/tests/test_get_dataset_labels.py +++ b/bigquery/samples/tests/test_dataset_label_samples.py @@ -12,12 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. - +from .. import delete_dataset_labels from .. import get_dataset_labels +from .. import label_dataset + +def test_dataset_label_samples(capsys, client, dataset_id): -def test_get_dataset_labels(capsys, client, dataset_id, dataset_with_labels_id): + label_dataset.label_dataset(client, dataset_id) + out, err = capsys.readouterr() + assert "Labels added to {}".format(dataset_id) in out get_dataset_labels.get_dataset_labels(client, dataset_id) out, err = capsys.readouterr() assert "color: green" in out + + delete_dataset_labels.delete_dataset_labels(client, dataset_id) + out, err = capsys.readouterr() + assert "Labels deleted from {}".format(dataset_id) in out diff --git a/bigquery/samples/tests/test_delete_dataset_labels.py b/bigquery/samples/tests/test_delete_dataset_labels.py deleted file mode 100644 index 45ce61e1d96c..000000000000 --- a/bigquery/samples/tests/test_delete_dataset_labels.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from .. import delete_dataset_labels - - -def test_delete_dataset_labels(capsys, client, dataset_id, dataset_with_labels_id): - - delete_dataset_labels.delete_dataset_labels(client, dataset_id) - out, err = capsys.readouterr() - assert "Labels deleted." in out diff --git a/bigquery/samples/tests/test_label_dataset.py b/bigquery/samples/tests/test_label_dataset.py deleted file mode 100644 index 1ecc6a975f4c..000000000000 --- a/bigquery/samples/tests/test_label_dataset.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from .. import label_dataset - - -def test_label_dataset(capsys, client, dataset_id): - - label_dataset.label_dataset(client, dataset_id) - out, err = capsys.readouterr() - assert "color: green" in out From af27c29a50f67c5ab40eb5691cdbec8c931cddd8 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 23 Aug 2019 14:18:42 +0300 Subject: [PATCH 39/68] update browse_table_data --- bigquery/samples/browse_table_data.py | 48 +++++++------------ bigquery/samples/tests/conftest.py | 4 +- .../samples/tests/test_browse_table_data.py | 12 ++--- 3 files changed, 24 insertions(+), 40 deletions(-) diff --git a/bigquery/samples/browse_table_data.py b/bigquery/samples/browse_table_data.py index 1885fc7bf405..46525f353bd3 100644 --- a/bigquery/samples/browse_table_data.py +++ b/bigquery/samples/browse_table_data.py @@ -16,9 +16,7 @@ def browse_table_data(client, table_id): # [START bigquery_browse_table] - """Retreive selected row data from a table.""" - - from google.cloud import bigquery + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() @@ -26,37 +24,23 @@ def browse_table_data(client, table_id): # TODO(developer): Set table_id to the ID of the table to browse data rows. # table_id = "your-project.your_dataset.your_table_name" - table = client.get_table(table_id) # API call + # Download all rows from a table. + rows_iter = client.list_rows(table_id) - # Load all rows from a table - rows = client.list_rows(table) - if len(list(rows)) == table.num_rows: - print("The amount of rows in the table = {}".format(table.num_rows)) + # Iterate over rows to make the API requests to fetch row data. + rows = list(rows_iter) + print("Downloaded {} rows from table {}".format(len(rows), table_id)) - # Load the first 10 rows - rows = client.list_rows(table, max_results=10) - number_of_rows = len(list(rows)) - if number_of_rows == 10: - print("First {} rows of the table are loaded".format(number_of_rows)) + # Download at most 10 rows. + rows_iter = client.list_rows(table_id, max_results=10) + rows = list(rows_iter) + print("Downloaded {} rows from table {}".format(len(rows), table_id)) - # Specify selected fields to limit the results to certain columns + # Specify selected fields to limit the results to certain columns. + table = client.get_table(table_id) fields = table.schema[:2] # first two columns - rows = client.list_rows(table, selected_fields=fields, max_results=10) - if len(rows.schema) == 2: - print("Fields number set to {}".format(len(rows.schema))) - if len(list(rows)) == 10: - print("{} rows of the table loaded".format(number_of_rows)) - - # Use the start index to load an arbitrary portion of the table - rows = client.list_rows(table, start_index=10, max_results=10) - - # Print row data in tabular format - format_string = "{!s:<16} " * len(rows.schema) - field_names = [field.name for field in rows.schema] - print(format_string.format(*field_names)) # prints column headers - for number_of_rows, row in enumerate(rows, 1): - print(format_string.format(*row)) # prints row data - - print("Printed data contains {} rows".format(number_of_rows)) - + rows_iter = client.list_rows(table_id, selected_fields=fields, max_results=10) + rows = list(rows_iter) + print("Selected {} columns from table {}.".format(len(rows_iter.schema), table_id)) + print("Downloaded {} rows from table {}".format(len(rows), table_id)) # [END bigquery_browse_table] diff --git a/bigquery/samples/tests/conftest.py b/bigquery/samples/tests/conftest.py index 034607b03e87..a075757abe4e 100644 --- a/bigquery/samples/tests/conftest.py +++ b/bigquery/samples/tests/conftest.py @@ -79,10 +79,10 @@ def table_id(client, dataset_id): @pytest.fixture -def table_w_data(client): +def table_with_data_id(client): dataset = client.get_dataset("bigquery-public-data.samples") table = dataset.table("shakespeare") - return table + return "{}.{}.{}".format(table.project, table.dataset_id, table.table_id) @pytest.fixture diff --git a/bigquery/samples/tests/test_browse_table_data.py b/bigquery/samples/tests/test_browse_table_data.py index 7931f62c35aa..f777bf91ca00 100644 --- a/bigquery/samples/tests/test_browse_table_data.py +++ b/bigquery/samples/tests/test_browse_table_data.py @@ -16,11 +16,11 @@ from .. import browse_table_data -def test_browse_table_data(capsys, client, table_w_data): +def test_browse_table_data(capsys, client, table_with_data_id): - browse_table_data.browse_table_data(client, table_w_data) + browse_table_data.browse_table_data(client, table_with_data_id) out, err = capsys.readouterr() - assert "The amount of rows in the table = " in out - assert "First 10 rows of the table are loaded" in out - assert "Fields number set to 2" in out - assert "Printed data contains 10 rows" in out + assert "Downloaded 164656 rows from table {}".format(table_with_data_id) in out + assert "Downloaded 10 rows from table {}".format(table_with_data_id) in out + assert "Selected 2 columns from table {}".format(table_with_data_id) in out + assert "Downloaded 10 rows from table {}".format(table_with_data_id) in out From 47ee1560a595f3e1562e2a74469ff9bd52a83024 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 23 Aug 2019 14:22:33 +0300 Subject: [PATCH 40/68] deleted extra spaces --- bigquery/samples/add_empty_column.py | 1 - bigquery/samples/dataset_exists.py | 1 - bigquery/samples/list_datasets_by_label.py | 1 - bigquery/samples/tests/test_dataset_exists.py | 3 ++- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/bigquery/samples/add_empty_column.py b/bigquery/samples/add_empty_column.py index 68237ff68c93..eb84037598d3 100644 --- a/bigquery/samples/add_empty_column.py +++ b/bigquery/samples/add_empty_column.py @@ -37,5 +37,4 @@ def add_empty_column(client, table_id): print("A new column has been added.") else: print("The column has not been added.") - # [END bigquery_add_empty_column] diff --git a/bigquery/samples/dataset_exists.py b/bigquery/samples/dataset_exists.py index fba822564c1b..46cf26a623bf 100644 --- a/bigquery/samples/dataset_exists.py +++ b/bigquery/samples/dataset_exists.py @@ -26,5 +26,4 @@ def dataset_exists(client, dataset_id): print("Dataset {} already exists".format(dataset_id)) except NotFound: print("Dataset {} is not found".format(dataset_id)) - # [END bigquery_dataset_exists] diff --git a/bigquery/samples/list_datasets_by_label.py b/bigquery/samples/list_datasets_by_label.py index d9aa38368f5a..5ca079317ab0 100644 --- a/bigquery/samples/list_datasets_by_label.py +++ b/bigquery/samples/list_datasets_by_label.py @@ -30,5 +30,4 @@ def list_datasets_by_label(client): print("\t{}.{}".format(dataset.project, dataset.dataset_id)) else: print("No datasets found with this filter.") - # [END bigquery_list_datasets_by_label] diff --git a/bigquery/samples/tests/test_dataset_exists.py b/bigquery/samples/tests/test_dataset_exists.py index 314c1db9b29c..203c4b884dc4 100644 --- a/bigquery/samples/tests/test_dataset_exists.py +++ b/bigquery/samples/tests/test_dataset_exists.py @@ -21,9 +21,10 @@ def test_dataset_exists(capsys, client, random_dataset_id): dataset_exists.dataset_exists(client, random_dataset_id) + out, err = capsys.readouterr() + assert "Dataset {} is not found".format(random_dataset_id) in out dataset = bigquery.Dataset(random_dataset_id) dataset = client.create_dataset(dataset) dataset_exists.dataset_exists(client, random_dataset_id) out, err = capsys.readouterr() - assert "Dataset {} is not found".format(random_dataset_id) in out assert "Dataset {} already exists".format(random_dataset_id) in out From b78f6c93e07fd225d05a132b424afbe894c111ea Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 23 Aug 2019 16:02:47 +0300 Subject: [PATCH 41/68] update import settings --- bigquery/samples/browse_table_data.py | 1 + bigquery/samples/delete_dataset_labels.py | 1 + bigquery/samples/get_dataset_labels.py | 1 + bigquery/samples/label_dataset.py | 1 + bigquery/samples/list_datasets_by_label.py | 3 ++- 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/bigquery/samples/browse_table_data.py b/bigquery/samples/browse_table_data.py index 46525f353bd3..dd6c572cab6d 100644 --- a/bigquery/samples/browse_table_data.py +++ b/bigquery/samples/browse_table_data.py @@ -16,6 +16,7 @@ def browse_table_data(client, table_id): # [START bigquery_browse_table] + # TODO(developer): Import the client library. # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. diff --git a/bigquery/samples/delete_dataset_labels.py b/bigquery/samples/delete_dataset_labels.py index e0f8492ee944..77c8a0011a66 100644 --- a/bigquery/samples/delete_dataset_labels.py +++ b/bigquery/samples/delete_dataset_labels.py @@ -16,6 +16,7 @@ def delete_dataset_labels(client, dataset_id): # [START bigquery_delete_label_dataset] + # TODO(developer): Import the client library. # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. diff --git a/bigquery/samples/get_dataset_labels.py b/bigquery/samples/get_dataset_labels.py index 94c7c203501e..31c1710c04cd 100644 --- a/bigquery/samples/get_dataset_labels.py +++ b/bigquery/samples/get_dataset_labels.py @@ -16,6 +16,7 @@ def get_dataset_labels(client, dataset_id): # [START bigquery_get_dataset_labels] + # TODO(developer): Import the client library. # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. diff --git a/bigquery/samples/label_dataset.py b/bigquery/samples/label_dataset.py index 5d48001791f2..7840ea25a63f 100644 --- a/bigquery/samples/label_dataset.py +++ b/bigquery/samples/label_dataset.py @@ -16,6 +16,7 @@ def label_dataset(client, dataset_id): # [START bigquery_label_dataset] + # TODO(developer): Import the client library. # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. diff --git a/bigquery/samples/list_datasets_by_label.py b/bigquery/samples/list_datasets_by_label.py index 5ca079317ab0..2254e2476975 100644 --- a/bigquery/samples/list_datasets_by_label.py +++ b/bigquery/samples/list_datasets_by_label.py @@ -16,7 +16,8 @@ def list_datasets_by_label(client): # [START bigquery_list_datasets_by_label] - from google.cloud import bigquery + # TODO(developer): Import the client library. + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() From 6ee351ece2858df0064d44b8ec37fa732b5a27f6 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 23 Aug 2019 15:41:08 +0300 Subject: [PATCH 42/68] reformate samples/ --- bigquery/samples/create_job.py | 1 + bigquery/samples/create_routine.py | 3 ++- bigquery/samples/create_routine_ddl.py | 3 ++- bigquery/samples/delete_dataset.py | 3 ++- bigquery/samples/delete_model.py | 3 ++- bigquery/samples/delete_routine.py | 3 ++- bigquery/samples/delete_table.py | 3 ++- bigquery/samples/get_dataset.py | 3 ++- bigquery/samples/get_model.py | 3 ++- bigquery/samples/get_routine.py | 3 ++- bigquery/samples/get_table.py | 3 ++- bigquery/samples/list_datasets.py | 3 ++- bigquery/samples/list_models.py | 3 ++- bigquery/samples/list_routines.py | 2 +- bigquery/samples/list_tables.py | 3 ++- bigquery/samples/query_to_arrow.py | 9 ++------- bigquery/samples/tests/test_query_to_arrow.py | 4 ++-- bigquery/samples/tests/test_routine_samples.py | 10 +++++----- .../samples/update_dataset_default_table_expiration.py | 3 ++- bigquery/samples/update_dataset_description.py | 3 ++- bigquery/samples/update_model.py | 3 ++- bigquery/samples/update_routine.py | 3 ++- 22 files changed, 45 insertions(+), 32 deletions(-) diff --git a/bigquery/samples/create_job.py b/bigquery/samples/create_job.py index 7570dc49f1d2..24bb85510598 100644 --- a/bigquery/samples/create_job.py +++ b/bigquery/samples/create_job.py @@ -14,6 +14,7 @@ def create_job(client): + # [START bigquery_create_job] from google.cloud import bigquery diff --git a/bigquery/samples/create_routine.py b/bigquery/samples/create_routine.py index 18b999980d72..c08ec4799a3e 100644 --- a/bigquery/samples/create_routine.py +++ b/bigquery/samples/create_routine.py @@ -13,7 +13,8 @@ # limitations under the License. -def main(client, routine_id): +def create_routine(client, routine_id): + # [START bigquery_create_routine] from google.cloud import bigquery from google.cloud import bigquery_v2 diff --git a/bigquery/samples/create_routine_ddl.py b/bigquery/samples/create_routine_ddl.py index aa6254b1139a..a4ae3318e7b4 100644 --- a/bigquery/samples/create_routine_ddl.py +++ b/bigquery/samples/create_routine_ddl.py @@ -13,7 +13,8 @@ # limitations under the License. -def main(client, routine_id): +def create_routine_ddl(client, routine_id): + # [START bigquery_create_routine_ddl] # TODO(developer): Import the client library. # from google.cloud import bigquery diff --git a/bigquery/samples/delete_dataset.py b/bigquery/samples/delete_dataset.py index 58851f1e2120..6cde1b6b2d27 100644 --- a/bigquery/samples/delete_dataset.py +++ b/bigquery/samples/delete_dataset.py @@ -16,7 +16,8 @@ def delete_dataset(client, dataset_id): # [START bigquery_delete_dataset] - from google.cloud import bigquery + # TODO(developer): Import the client library. + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() diff --git a/bigquery/samples/delete_model.py b/bigquery/samples/delete_model.py index 371f9003576b..05b65e3078e6 100644 --- a/bigquery/samples/delete_model.py +++ b/bigquery/samples/delete_model.py @@ -17,7 +17,8 @@ def delete_model(client, model_id): """Sample ID: go/samples-tracker/1534""" # [START bigquery_delete_model] - from google.cloud import bigquery + # TODO(developer): Import the client library. + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() diff --git a/bigquery/samples/delete_routine.py b/bigquery/samples/delete_routine.py index 505faa4780f3..4001836fbb48 100644 --- a/bigquery/samples/delete_routine.py +++ b/bigquery/samples/delete_routine.py @@ -13,7 +13,8 @@ # limitations under the License. -def main(client, routine_id): +def delete_routine(client, routine_id): + # [START bigquery_delete_routine] # TODO(developer): Import the client library. # from google.cloud import bigquery diff --git a/bigquery/samples/delete_table.py b/bigquery/samples/delete_table.py index 3eb7dc918da7..dcdd3d855b2e 100644 --- a/bigquery/samples/delete_table.py +++ b/bigquery/samples/delete_table.py @@ -16,7 +16,8 @@ def delete_table(client, table_id): # [START bigquery_delete_table] - from google.cloud import bigquery + # TODO(developer): Import the client library. + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() diff --git a/bigquery/samples/get_dataset.py b/bigquery/samples/get_dataset.py index eeab2e088d2f..d1dc9fd4c16b 100644 --- a/bigquery/samples/get_dataset.py +++ b/bigquery/samples/get_dataset.py @@ -16,7 +16,8 @@ def get_dataset(client, dataset_id): # [START bigquery_get_dataset] - from google.cloud import bigquery + # TODO(developer): Import the client library. + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() diff --git a/bigquery/samples/get_model.py b/bigquery/samples/get_model.py index 8e43e53ec450..69986733c50b 100644 --- a/bigquery/samples/get_model.py +++ b/bigquery/samples/get_model.py @@ -17,7 +17,8 @@ def get_model(client, model_id): """Sample ID: go/samples-tracker/1510""" # [START bigquery_get_model] - from google.cloud import bigquery + # TODO(developer): Import the client library. + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() diff --git a/bigquery/samples/get_routine.py b/bigquery/samples/get_routine.py index 5850d8d06477..536c59ef170f 100644 --- a/bigquery/samples/get_routine.py +++ b/bigquery/samples/get_routine.py @@ -13,7 +13,8 @@ # limitations under the License. -def main(client, routine_id): +def get_routine(client, routine_id): + # [START bigquery_get_routine] # TODO(developer): Import the client library. # from google.cloud import bigquery diff --git a/bigquery/samples/get_table.py b/bigquery/samples/get_table.py index e6a5c502e2b3..e49e032f6e23 100644 --- a/bigquery/samples/get_table.py +++ b/bigquery/samples/get_table.py @@ -16,7 +16,8 @@ def get_table(client, table_id): # [START bigquery_get_table] - from google.cloud import bigquery + # TODO(developer): Import the client library. + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() diff --git a/bigquery/samples/list_datasets.py b/bigquery/samples/list_datasets.py index c9ddf4f2523c..b57aad1b5e7b 100644 --- a/bigquery/samples/list_datasets.py +++ b/bigquery/samples/list_datasets.py @@ -16,7 +16,8 @@ def list_datasets(client): # [START bigquery_list_datasets] - from google.cloud import bigquery + # TODO(developer): Import the client library. + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() diff --git a/bigquery/samples/list_models.py b/bigquery/samples/list_models.py index cb6e4fb5569f..5b4d21799b28 100644 --- a/bigquery/samples/list_models.py +++ b/bigquery/samples/list_models.py @@ -17,7 +17,8 @@ def list_models(client, dataset_id): """Sample ID: go/samples-tracker/1512""" # [START bigquery_list_models] - from google.cloud import bigquery + # TODO(developer): Import the client library. + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() diff --git a/bigquery/samples/list_routines.py b/bigquery/samples/list_routines.py index 9e90c87a3d9c..1ae4f441cde1 100644 --- a/bigquery/samples/list_routines.py +++ b/bigquery/samples/list_routines.py @@ -13,7 +13,7 @@ # limitations under the License. -def main(client, dataset_id): +def list_routines(client, dataset_id): # [START bigquery_list_routines] # TODO(developer): Import the client library. diff --git a/bigquery/samples/list_tables.py b/bigquery/samples/list_tables.py index 33ed408906b0..2057f2d73891 100644 --- a/bigquery/samples/list_tables.py +++ b/bigquery/samples/list_tables.py @@ -16,7 +16,8 @@ def list_tables(client, dataset_id): # [START bigquery_list_tables] - from google.cloud import bigquery + # TODO(developer): Import the client library. + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() diff --git a/bigquery/samples/query_to_arrow.py b/bigquery/samples/query_to_arrow.py index e3ddc23f889a..b13dcf3e1413 100644 --- a/bigquery/samples/query_to_arrow.py +++ b/bigquery/samples/query_to_arrow.py @@ -13,7 +13,8 @@ # limitations under the License. -def main(client): +def query_to_arrow(client): + # [START bigquery_query_to_arrow] # TODO(developer): Import the client library. # from google.cloud import bigquery @@ -50,9 +51,3 @@ def main(client): print("\nSchema:\n{}".format(repr(arrow_table.schema))) # [END bigquery_query_to_arrow] return arrow_table - - -if __name__ == "__main__": - from google.cloud import bigquery - - main(bigquery.Client()) diff --git a/bigquery/samples/tests/test_query_to_arrow.py b/bigquery/samples/tests/test_query_to_arrow.py index 9e36bcee346f..f70bd49fe565 100644 --- a/bigquery/samples/tests/test_query_to_arrow.py +++ b/bigquery/samples/tests/test_query_to_arrow.py @@ -17,9 +17,9 @@ from .. import query_to_arrow -def test_main(capsys, client): +def test_query_to_arrow(capsys, client): - arrow_table = query_to_arrow.main(client) + arrow_table = query_to_arrow.query_to_arrow(client) out, err = capsys.readouterr() assert "Downloaded 8 rows, 2 columns." in out diff --git a/bigquery/samples/tests/test_routine_samples.py b/bigquery/samples/tests/test_routine_samples.py index 5905d2e69439..5a1c69c7f60f 100644 --- a/bigquery/samples/tests/test_routine_samples.py +++ b/bigquery/samples/tests/test_routine_samples.py @@ -19,7 +19,7 @@ def test_create_routine(capsys, client, random_routine_id): from .. import create_routine - create_routine.main(client, random_routine_id) + create_routine.create_routine(client, random_routine_id) out, err = capsys.readouterr() assert "Created routine {}".format(random_routine_id) in out @@ -27,7 +27,7 @@ def test_create_routine(capsys, client, random_routine_id): def test_create_routine_ddl(capsys, client, random_routine_id): from .. import create_routine_ddl - create_routine_ddl.main(client, random_routine_id) + create_routine_ddl.create_routine_ddl(client, random_routine_id) routine = client.get_routine(random_routine_id) out, err = capsys.readouterr() @@ -68,7 +68,7 @@ def test_create_routine_ddl(capsys, client, random_routine_id): def test_list_routines(capsys, client, dataset_id, routine_id): from .. import list_routines - list_routines.main(client, dataset_id) + list_routines.list_routines(client, dataset_id) out, err = capsys.readouterr() assert "Routines contained in dataset {}:".format(dataset_id) in out assert routine_id in out @@ -77,7 +77,7 @@ def test_list_routines(capsys, client, dataset_id, routine_id): def test_delete_routine(capsys, client, routine_id): from .. import delete_routine - delete_routine.main(client, routine_id) + delete_routine.delete_routine(client, routine_id) out, err = capsys.readouterr() assert "Deleted routine {}.".format(routine_id) in out @@ -85,5 +85,5 @@ def test_delete_routine(capsys, client, routine_id): def test_update_routine(client, routine_id): from .. import update_routine - routine = update_routine.main(client, routine_id) + routine = update_routine.update_routine(client, routine_id) assert routine.body == "x * 4" diff --git a/bigquery/samples/update_dataset_default_table_expiration.py b/bigquery/samples/update_dataset_default_table_expiration.py index a5ac38c01a99..4534bb2011eb 100644 --- a/bigquery/samples/update_dataset_default_table_expiration.py +++ b/bigquery/samples/update_dataset_default_table_expiration.py @@ -16,7 +16,8 @@ def update_dataset_default_table_expiration(client, dataset_id): # [START bigquery_update_dataset_expiration] - from google.cloud import bigquery + # TODO(developer): Import the client library. + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() diff --git a/bigquery/samples/update_dataset_description.py b/bigquery/samples/update_dataset_description.py index 70be80b7507e..f3afb7fa68ce 100644 --- a/bigquery/samples/update_dataset_description.py +++ b/bigquery/samples/update_dataset_description.py @@ -16,7 +16,8 @@ def update_dataset_description(client, dataset_id): # [START bigquery_update_dataset_description] - from google.cloud import bigquery + # TODO(developer): Import the client library. + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() diff --git a/bigquery/samples/update_model.py b/bigquery/samples/update_model.py index 2440066ae1ec..5df4ada886ed 100644 --- a/bigquery/samples/update_model.py +++ b/bigquery/samples/update_model.py @@ -17,7 +17,8 @@ def update_model(client, model_id): """Sample ID: go/samples-tracker/1533""" # [START bigquery_update_model_description] - from google.cloud import bigquery + # TODO(developer): Import the client library. + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() diff --git a/bigquery/samples/update_routine.py b/bigquery/samples/update_routine.py index 8683e761562f..4d491d42e168 100644 --- a/bigquery/samples/update_routine.py +++ b/bigquery/samples/update_routine.py @@ -13,7 +13,8 @@ # limitations under the License. -def main(client, routine_id): +def update_routine(client, routine_id): + # [START bigquery_update_routine] # TODO(developer): Import the client library. # from google.cloud import bigquery From 9a65dae0de72d9a8cbfee7811156c2e9681d7f7d Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 23 Aug 2019 17:02:50 +0300 Subject: [PATCH 43/68] general update second five --- bigquery/samples/client_list_jobs.py | 4 ++-- bigquery/samples/client_query.py | 17 ++++++++--------- bigquery/samples/copy_table.py | 8 ++------ bigquery/samples/table_exists.py | 1 - bigquery/samples/table_insert_rows.py | 4 ++-- bigquery/samples/tests/conftest.py | 7 +++++++ bigquery/samples/tests/test_copy_table.py | 5 ++--- bigquery/samples/tests/test_table_exists.py | 3 ++- 8 files changed, 25 insertions(+), 24 deletions(-) diff --git a/bigquery/samples/client_list_jobs.py b/bigquery/samples/client_list_jobs.py index 048a37ef2170..4919b5150b6f 100644 --- a/bigquery/samples/client_list_jobs.py +++ b/bigquery/samples/client_list_jobs.py @@ -16,7 +16,8 @@ def client_list_jobs(client): # [START bigquery_list_jobs] - from google.cloud import bigquery + # TODO(developer): Import the client library. + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # project = "my_project" # replace with your project ID @@ -47,5 +48,4 @@ def client_list_jobs(client): print("Last 10 jobs done:") for job in client.list_jobs(max_results=10, state_filter="DONE"): print("{}".format(job.job_id)) - # [END bigquery_list_jobs] diff --git a/bigquery/samples/client_query.py b/bigquery/samples/client_query.py index f394e3a2ba1e..cdd371c0066f 100644 --- a/bigquery/samples/client_query.py +++ b/bigquery/samples/client_query.py @@ -16,7 +16,8 @@ def client_query(client): # [START bigquery_query] - from google.cloud import bigquery + # TODO(developer): Import the client library. + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() @@ -31,13 +32,11 @@ def client_query(client): query, # Location must match that of the dataset(s) referenced in the query. location="US", - ) # API request - starts the query - - if query_job.state == "RUNNING": - print("The query data:") - for row in query_job: # API request - fetches results - # Row values can be accessed by field name or index - if row[0] == row.name == row["name"]: - print(row) + ) + print("The query data:") + for row in query_job: # API request - fetches results + # Row values can be accessed by field name or index + if row[0] == row.name == row["name"]: + print(row) # [END bigquery_query] diff --git a/bigquery/samples/copy_table.py b/bigquery/samples/copy_table.py index 4b721c2b68ca..181e58eeeda7 100644 --- a/bigquery/samples/copy_table.py +++ b/bigquery/samples/copy_table.py @@ -16,7 +16,8 @@ def copy_table(client, dataset_id, table_id): # [START bigquery_copy_table] - from google.cloud import bigquery + # TODO(developer): Import the client library. + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() @@ -28,9 +29,7 @@ def copy_table(client, dataset_id, table_id): # table_id = "your-project.your_dataset.your_table_name" dataset = client.get_dataset(dataset_id) - orig_table = client.get_table(table_id) - dest_table = dataset.table("destination_table") job = client.copy_table( @@ -42,10 +41,7 @@ def copy_table(client, dataset_id, table_id): job.result() # Waits for job to complete. - if job.state == "DONE": - print("The process completed") dest_table = client.get_table(dest_table) if dest_table.num_rows == orig_table.num_rows: print("A copy of the table created") - # [END bigquery_copy_table] diff --git a/bigquery/samples/table_exists.py b/bigquery/samples/table_exists.py index 3b3d9b461f08..314addd0d214 100644 --- a/bigquery/samples/table_exists.py +++ b/bigquery/samples/table_exists.py @@ -26,5 +26,4 @@ def table_exists(client, table_id): print("Table {} already exists".format(table_id)) except NotFound: print("Table {} is not found".format(table_id)) - # [END bigquery_table_exists] diff --git a/bigquery/samples/table_insert_rows.py b/bigquery/samples/table_insert_rows.py index ccc14c5520b9..244a83426a67 100644 --- a/bigquery/samples/table_insert_rows.py +++ b/bigquery/samples/table_insert_rows.py @@ -16,7 +16,8 @@ def table_insert_rows(client, table_id): # [START bigquery_table_insert_rows] - from google.cloud import bigquery + # TODO(developer): Import the client library. + # from google.cloud import bigquery # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() @@ -31,5 +32,4 @@ def table_insert_rows(client, table_id): errors = client.insert_rows(table, rows_to_insert) if errors == []: print("New rows have been added") - # [END bigquery_table_insert_rows] diff --git a/bigquery/samples/tests/conftest.py b/bigquery/samples/tests/conftest.py index fe5391ee8a4d..a075757abe4e 100644 --- a/bigquery/samples/tests/conftest.py +++ b/bigquery/samples/tests/conftest.py @@ -78,6 +78,13 @@ def table_id(client, dataset_id): client.delete_table(table, not_found_ok=True) +@pytest.fixture +def table_with_data_id(client): + dataset = client.get_dataset("bigquery-public-data.samples") + table = dataset.table("shakespeare") + return "{}.{}.{}".format(table.project, table.dataset_id, table.table_id) + + @pytest.fixture def routine_id(client, dataset_id): now = datetime.datetime.now() diff --git a/bigquery/samples/tests/test_copy_table.py b/bigquery/samples/tests/test_copy_table.py index d685e9f510fa..765847b0aa48 100644 --- a/bigquery/samples/tests/test_copy_table.py +++ b/bigquery/samples/tests/test_copy_table.py @@ -16,9 +16,8 @@ from .. import copy_table -def test_copy_table(capsys, client, dataset_id, table_w_data): +def test_copy_table(capsys, client, dataset_id, table_with_data_id): - copy_table.copy_table(client, dataset_id, table_w_data) + copy_table.copy_table(client, dataset_id, table_with_data_id) out, err = capsys.readouterr() - assert "The process completed" in out assert "A copy of the table created" in out diff --git a/bigquery/samples/tests/test_table_exists.py b/bigquery/samples/tests/test_table_exists.py index 77ecf050d5aa..bf5e059b022e 100644 --- a/bigquery/samples/tests/test_table_exists.py +++ b/bigquery/samples/tests/test_table_exists.py @@ -26,9 +26,10 @@ def test_table_exists(capsys, client, random_table_id): ] table_exists.table_exists(client, random_table_id) + out, err = capsys.readouterr() + assert "Table {} is not found".format(random_table_id) in out table = bigquery.Table(random_table_id, schema=schema) table = client.create_table(table) table_exists.table_exists(client, random_table_id) out, err = capsys.readouterr() - assert "Table {} is not found".format(random_table_id) in out assert "Table {} already exists".format(random_table_id) in out From 14757f6fa6cf403ab715b7dc166a5e57494e57ab Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Mon, 26 Aug 2019 10:22:26 +0300 Subject: [PATCH 44/68] minor update * rewrote table_with_data_id fixture * removed extra "# API request" --- bigquery/samples/list_datasets_by_label.py | 2 +- bigquery/samples/tests/conftest.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/bigquery/samples/list_datasets_by_label.py b/bigquery/samples/list_datasets_by_label.py index 2254e2476975..8b574b1110eb 100644 --- a/bigquery/samples/list_datasets_by_label.py +++ b/bigquery/samples/list_datasets_by_label.py @@ -27,7 +27,7 @@ def list_datasets_by_label(client): if datasets: print("Datasets filtered by {}:".format(label_filter)) - for dataset in datasets: # API request(s) + for dataset in datasets: print("\t{}.{}".format(dataset.project, dataset.dataset_id)) else: print("No datasets found with this filter.") diff --git a/bigquery/samples/tests/conftest.py b/bigquery/samples/tests/conftest.py index a075757abe4e..f2bb93112a22 100644 --- a/bigquery/samples/tests/conftest.py +++ b/bigquery/samples/tests/conftest.py @@ -80,9 +80,7 @@ def table_id(client, dataset_id): @pytest.fixture def table_with_data_id(client): - dataset = client.get_dataset("bigquery-public-data.samples") - table = dataset.table("shakespeare") - return "{}.{}.{}".format(table.project, table.dataset_id, table.table_id) + return "bigquery-public-data.samples.shakespeare" @pytest.fixture From 9941957f0724ef030de1dc2a68ff164966bb67d3 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Mon, 26 Aug 2019 10:55:16 +0300 Subject: [PATCH 45/68] cosmetic chgs --- bigquery/samples/create_table.py | 10 +++++----- bigquery/samples/delete_model.py | 1 + bigquery/samples/delete_routine.py | 2 +- bigquery/samples/get_dataset.py | 1 - bigquery/samples/get_dataset_labels.py | 1 - bigquery/samples/get_routine.py | 1 - bigquery/samples/tests/test_get_table.py | 1 + 7 files changed, 8 insertions(+), 9 deletions(-) diff --git a/bigquery/samples/create_table.py b/bigquery/samples/create_table.py index 5e2e34d41d99..2a6e98fc72f6 100644 --- a/bigquery/samples/create_table.py +++ b/bigquery/samples/create_table.py @@ -18,17 +18,17 @@ def create_table(client, table_id): # [START bigquery_create_table] from google.cloud import bigquery - schema = [ - bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), - ] - # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() # TODO(developer): Set table_id to the ID of the table to create # table_id = "your-project.your_dataset.your_table_name" + schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + ] + table = bigquery.Table(table_id, schema=schema) table = client.create_table(table) # API request print( diff --git a/bigquery/samples/delete_model.py b/bigquery/samples/delete_model.py index 05b65e3078e6..5ac4305bc97e 100644 --- a/bigquery/samples/delete_model.py +++ b/bigquery/samples/delete_model.py @@ -27,5 +27,6 @@ def delete_model(client, model_id): # model_id = 'your-project.your_dataset.your_model' client.delete_model(model_id) + print("Deleted model '{}'.".format(model_id)) # [END bigquery_delete_model] diff --git a/bigquery/samples/delete_routine.py b/bigquery/samples/delete_routine.py index 4001836fbb48..c0164b415008 100644 --- a/bigquery/samples/delete_routine.py +++ b/bigquery/samples/delete_routine.py @@ -26,6 +26,6 @@ def delete_routine(client, routine_id): # routine_id = "my-project.my_dataset.my_routine" client.delete_routine(routine_id) - # [END bigquery_delete_routine] print("Deleted routine {}.".format(routine_id)) + # [END bigquery_delete_routine] diff --git a/bigquery/samples/get_dataset.py b/bigquery/samples/get_dataset.py index d1dc9fd4c16b..5586c2b95ebb 100644 --- a/bigquery/samples/get_dataset.py +++ b/bigquery/samples/get_dataset.py @@ -53,5 +53,4 @@ def get_dataset(client, dataset_id): print("\t{}".format(table.table_id)) else: print("\tThis dataset does not contain any tables.") - # [END bigquery_get_dataset] diff --git a/bigquery/samples/get_dataset_labels.py b/bigquery/samples/get_dataset_labels.py index 31c1710c04cd..2f21723a550b 100644 --- a/bigquery/samples/get_dataset_labels.py +++ b/bigquery/samples/get_dataset_labels.py @@ -35,5 +35,4 @@ def get_dataset_labels(client, dataset_id): print("\t{}: {}".format(label, value)) else: print("\tDataset has no labels defined.") - # [END bigquery_get_dataset_labels] diff --git a/bigquery/samples/get_routine.py b/bigquery/samples/get_routine.py index 536c59ef170f..d9035c282438 100644 --- a/bigquery/samples/get_routine.py +++ b/bigquery/samples/get_routine.py @@ -35,6 +35,5 @@ def get_routine(client, routine_id): for argument in routine.arguments: print(" Name: '{}'".format(argument.name)) print(" Type: '{}'".format(argument.type_)) - # [END bigquery_get_routine] return routine diff --git a/bigquery/samples/tests/test_get_table.py b/bigquery/samples/tests/test_get_table.py index debf1b63a3fc..b811ccecad1f 100644 --- a/bigquery/samples/tests/test_get_table.py +++ b/bigquery/samples/tests/test_get_table.py @@ -17,6 +17,7 @@ def test_get_table(capsys, client, random_table_id): + schema = [ bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), From 056f3575ae9ccb664d65a92dca480195ddaa174e Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Mon, 26 Aug 2019 14:48:43 +0300 Subject: [PATCH 46/68] Update tables.rst --- bigquery/docs/usage/tables.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery/docs/usage/tables.rst b/bigquery/docs/usage/tables.rst index 1205892b7841..6a6cbd356639 100644 --- a/bigquery/docs/usage/tables.rst +++ b/bigquery/docs/usage/tables.rst @@ -40,7 +40,7 @@ Determine if a table exists with the Browse data rows in a table with the :func:`~google.cloud.bigquery.client.Client.list_rows` method: -.. literalinclude:: ../snippets.py +.. literalinclude:: ../samples/browse_table_data.py :language: python :dedent: 4 :start-after: [START bigquery_browse_table] From 70c032fff9722945a813448069dae4eb7507b5a3 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Mon, 26 Aug 2019 15:43:28 +0300 Subject: [PATCH 47/68] Update test_list_datasets_by_label.py --- bigquery/samples/tests/test_list_datasets_by_label.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bigquery/samples/tests/test_list_datasets_by_label.py b/bigquery/samples/tests/test_list_datasets_by_label.py index ed6b6236d699..346cbf1a982d 100644 --- a/bigquery/samples/tests/test_list_datasets_by_label.py +++ b/bigquery/samples/tests/test_list_datasets_by_label.py @@ -16,8 +16,11 @@ from .. import list_datasets_by_label -def test_list_datasets_by_label(capsys, client, dataset_with_labels_id): +def test_list_datasets_by_label(capsys, client, dataset_id): + dataset = client.get_dataset(dataset_id) + dataset.labels = {"color": "green"} + dataset = client.update_dataset(dataset, ["labels"]) list_datasets_by_label.list_datasets_by_label(client) out, err = capsys.readouterr() - assert "{}".format(dataset_with_labels_id) in out + assert "{}".format(dataset_id) in out From cde3e62e2ce8581565a2d6cc8e6bbf9437f34a8c Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 27 Aug 2019 10:53:24 +0300 Subject: [PATCH 48/68] update delete_dataset_labels --- bigquery/samples/delete_dataset_labels.py | 1 + bigquery/samples/tests/test_dataset_label_samples.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bigquery/samples/delete_dataset_labels.py b/bigquery/samples/delete_dataset_labels.py index 77c8a0011a66..33ff5c0f2620 100644 --- a/bigquery/samples/delete_dataset_labels.py +++ b/bigquery/samples/delete_dataset_labels.py @@ -33,3 +33,4 @@ def delete_dataset_labels(client, dataset_id): dataset = client.update_dataset(dataset, ["labels"]) print("Labels deleted from {}".format(dataset_id)) # [END bigquery_delete_label_dataset] + return dataset diff --git a/bigquery/samples/tests/test_dataset_label_samples.py b/bigquery/samples/tests/test_dataset_label_samples.py index 9e8091302b70..1d78c33b445d 100644 --- a/bigquery/samples/tests/test_dataset_label_samples.py +++ b/bigquery/samples/tests/test_dataset_label_samples.py @@ -27,6 +27,7 @@ def test_dataset_label_samples(capsys, client, dataset_id): out, err = capsys.readouterr() assert "color: green" in out - delete_dataset_labels.delete_dataset_labels(client, dataset_id) + dataset = delete_dataset_labels.delete_dataset_labels(client, dataset_id) out, err = capsys.readouterr() assert "Labels deleted from {}".format(dataset_id) in out + assert dataset.labels.get("color") == None From 9d3c1aa0698c9cdc3cd374bd561a581b8d9dd208 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 27 Aug 2019 10:57:57 +0300 Subject: [PATCH 49/68] Update test_dataset_label_samples.py --- bigquery/samples/tests/test_dataset_label_samples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery/samples/tests/test_dataset_label_samples.py b/bigquery/samples/tests/test_dataset_label_samples.py index 1d78c33b445d..94a2092407b0 100644 --- a/bigquery/samples/tests/test_dataset_label_samples.py +++ b/bigquery/samples/tests/test_dataset_label_samples.py @@ -30,4 +30,4 @@ def test_dataset_label_samples(capsys, client, dataset_id): dataset = delete_dataset_labels.delete_dataset_labels(client, dataset_id) out, err = capsys.readouterr() assert "Labels deleted from {}".format(dataset_id) in out - assert dataset.labels.get("color") == None + assert dataset.labels.get("color") is None From f3607c1eb432538413efc39fb5757db86059689e Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 28 Aug 2019 15:15:48 +0300 Subject: [PATCH 50/68] Update conftest.py --- bigquery/samples/tests/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bigquery/samples/tests/conftest.py b/bigquery/samples/tests/conftest.py index f2bb93112a22..32b23931aa91 100644 --- a/bigquery/samples/tests/conftest.py +++ b/bigquery/samples/tests/conftest.py @@ -57,7 +57,7 @@ def random_routine_id(client, dataset_id): @pytest.fixture def dataset_id(client): now = datetime.datetime.now() - dataset_id = "python_samples_{}_{}".format( + dataset_id = "python_dataset_sample_{}_{}".format( now.strftime("%Y%m%d%H%M%S"), uuid.uuid4().hex[:8] ) dataset = client.create_dataset(dataset_id) @@ -68,7 +68,7 @@ def dataset_id(client): @pytest.fixture def table_id(client, dataset_id): now = datetime.datetime.now() - table_id = "python_samples_{}_{}".format( + table_id = "python_table_sample_{}_{}".format( now.strftime("%Y%m%d%H%M%S"), uuid.uuid4().hex[:8] ) @@ -86,7 +86,7 @@ def table_with_data_id(client): @pytest.fixture def routine_id(client, dataset_id): now = datetime.datetime.now() - routine_id = "python_samples_{}_{}".format( + routine_id = "python_routine_sample_{}_{}".format( now.strftime("%Y%m%d%H%M%S"), uuid.uuid4().hex[:8] ) From 1040fba9ca35d3b8aa8750e19c0051375013b70c Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 28 Aug 2019 17:39:12 +0300 Subject: [PATCH 51/68] update test files --- bigquery/samples/tests/test_create_dataset.py | 1 + bigquery/samples/tests/test_create_job.py | 1 + bigquery/samples/tests/test_create_table.py | 1 + bigquery/samples/tests/test_dataset_label_samples.py | 1 + bigquery/samples/tests/test_delete_dataset.py | 1 + bigquery/samples/tests/test_delete_table.py | 1 + bigquery/samples/tests/test_get_dataset.py | 1 + bigquery/samples/tests/test_get_table.py | 2 ++ bigquery/samples/tests/test_list_datasets.py | 1 + bigquery/samples/tests/test_list_tables.py | 1 + bigquery/samples/tests/test_model_samples.py | 1 + bigquery/samples/tests/test_query_to_arrow.py | 1 + bigquery/samples/tests/test_routine_samples.py | 1 + bigquery/samples/tests/test_update_dataset_access.py | 1 + .../tests/test_update_dataset_default_table_expiration.py | 1 + bigquery/samples/tests/test_update_dataset_description.py | 1 + 16 files changed, 17 insertions(+) diff --git a/bigquery/samples/tests/test_create_dataset.py b/bigquery/samples/tests/test_create_dataset.py index dfadc67d8468..e52e9ddfdced 100644 --- a/bigquery/samples/tests/test_create_dataset.py +++ b/bigquery/samples/tests/test_create_dataset.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + from .. import create_dataset diff --git a/bigquery/samples/tests/test_create_job.py b/bigquery/samples/tests/test_create_job.py index fce005ae8236..5ead51156606 100644 --- a/bigquery/samples/tests/test_create_job.py +++ b/bigquery/samples/tests/test_create_job.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + from .. import create_job diff --git a/bigquery/samples/tests/test_create_table.py b/bigquery/samples/tests/test_create_table.py index 093ee6e94277..f9ebc0e5d70d 100644 --- a/bigquery/samples/tests/test_create_table.py +++ b/bigquery/samples/tests/test_create_table.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + from .. import create_table diff --git a/bigquery/samples/tests/test_dataset_label_samples.py b/bigquery/samples/tests/test_dataset_label_samples.py index 94a2092407b0..1e526f2339ac 100644 --- a/bigquery/samples/tests/test_dataset_label_samples.py +++ b/bigquery/samples/tests/test_dataset_label_samples.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + from .. import delete_dataset_labels from .. import get_dataset_labels from .. import label_dataset diff --git a/bigquery/samples/tests/test_delete_dataset.py b/bigquery/samples/tests/test_delete_dataset.py index 2b1b6ad06195..836b3aebb272 100644 --- a/bigquery/samples/tests/test_delete_dataset.py +++ b/bigquery/samples/tests/test_delete_dataset.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + from .. import delete_dataset diff --git a/bigquery/samples/tests/test_delete_table.py b/bigquery/samples/tests/test_delete_table.py index 8f4796623a83..f76ad8624cc6 100644 --- a/bigquery/samples/tests/test_delete_table.py +++ b/bigquery/samples/tests/test_delete_table.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + from .. import delete_table diff --git a/bigquery/samples/tests/test_get_dataset.py b/bigquery/samples/tests/test_get_dataset.py index 374f8835211a..37ad0165bd86 100644 --- a/bigquery/samples/tests/test_get_dataset.py +++ b/bigquery/samples/tests/test_get_dataset.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + from .. import get_dataset diff --git a/bigquery/samples/tests/test_get_table.py b/bigquery/samples/tests/test_get_table.py index b811ccecad1f..c6eb03ad6633 100644 --- a/bigquery/samples/tests/test_get_table.py +++ b/bigquery/samples/tests/test_get_table.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. + from google.cloud import bigquery + from .. import get_table diff --git a/bigquery/samples/tests/test_list_datasets.py b/bigquery/samples/tests/test_list_datasets.py index 4c66a24f9b1a..d8c32e91ee20 100644 --- a/bigquery/samples/tests/test_list_datasets.py +++ b/bigquery/samples/tests/test_list_datasets.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + from .. import list_datasets diff --git a/bigquery/samples/tests/test_list_tables.py b/bigquery/samples/tests/test_list_tables.py index ec1621ac7579..61ac04ea26ce 100644 --- a/bigquery/samples/tests/test_list_tables.py +++ b/bigquery/samples/tests/test_list_tables.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + from .. import list_tables diff --git a/bigquery/samples/tests/test_model_samples.py b/bigquery/samples/tests/test_model_samples.py index d7b06a92a3e1..99d838533917 100644 --- a/bigquery/samples/tests/test_model_samples.py +++ b/bigquery/samples/tests/test_model_samples.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + from .. import delete_model from .. import get_model from .. import list_models diff --git a/bigquery/samples/tests/test_query_to_arrow.py b/bigquery/samples/tests/test_query_to_arrow.py index f70bd49fe565..2e8d05478a27 100644 --- a/bigquery/samples/tests/test_query_to_arrow.py +++ b/bigquery/samples/tests/test_query_to_arrow.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + import pyarrow from .. import query_to_arrow diff --git a/bigquery/samples/tests/test_routine_samples.py b/bigquery/samples/tests/test_routine_samples.py index 5a1c69c7f60f..d51f7b8cfb5c 100644 --- a/bigquery/samples/tests/test_routine_samples.py +++ b/bigquery/samples/tests/test_routine_samples.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + from google.cloud import bigquery from google.cloud import bigquery_v2 diff --git a/bigquery/samples/tests/test_update_dataset_access.py b/bigquery/samples/tests/test_update_dataset_access.py index ae33dbfe4a4c..679b700731e3 100644 --- a/bigquery/samples/tests/test_update_dataset_access.py +++ b/bigquery/samples/tests/test_update_dataset_access.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + from .. import update_dataset_access diff --git a/bigquery/samples/tests/test_update_dataset_default_table_expiration.py b/bigquery/samples/tests/test_update_dataset_default_table_expiration.py index 46e9654209ed..a97de11a2f1a 100644 --- a/bigquery/samples/tests/test_update_dataset_default_table_expiration.py +++ b/bigquery/samples/tests/test_update_dataset_default_table_expiration.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + from .. import update_dataset_default_table_expiration diff --git a/bigquery/samples/tests/test_update_dataset_description.py b/bigquery/samples/tests/test_update_dataset_description.py index c6f8889f50da..63826077b976 100644 --- a/bigquery/samples/tests/test_update_dataset_description.py +++ b/bigquery/samples/tests/test_update_dataset_description.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + from .. import update_dataset_description From edac824ccf262a9810ee6967b0eb810d85c66460 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 28 Aug 2019 18:15:34 +0300 Subject: [PATCH 52/68] cosmetic chgs --- bigquery/samples/client_list_jobs.py | 4 ++-- bigquery/samples/table_insert_rows.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bigquery/samples/client_list_jobs.py b/bigquery/samples/client_list_jobs.py index 4919b5150b6f..9067ac613efc 100644 --- a/bigquery/samples/client_list_jobs.py +++ b/bigquery/samples/client_list_jobs.py @@ -23,6 +23,8 @@ def client_list_jobs(client): # project = "my_project" # replace with your project ID # client = bigquery.Client(project=project) + import datetime + # List the 10 most recent jobs in reverse chronological order. # Omit the max_results parameter to list jobs from the past 6 months. print("Last 10 jobs:") @@ -31,8 +33,6 @@ def client_list_jobs(client): # The following are examples of additional optional parameters: - import datetime - # Use min_creation_time and/or max_creation_time to specify a time window. print("Jobs from the last ten minutes:") ten_mins_ago = datetime.datetime.utcnow() - datetime.timedelta(minutes=10) diff --git a/bigquery/samples/table_insert_rows.py b/bigquery/samples/table_insert_rows.py index 244a83426a67..43fa6617ea4c 100644 --- a/bigquery/samples/table_insert_rows.py +++ b/bigquery/samples/table_insert_rows.py @@ -26,7 +26,6 @@ def table_insert_rows(client, table_id): # table_id = "your-project.your_dataset.your_table" table = client.get_table(table_id) - rows_to_insert = [(u"Phred Phlyntstone", 32), (u"Wylma Phlyntstone", 29)] errors = client.insert_rows(table, rows_to_insert) From ea949151d34533447bac40979c3548b3a29fee53 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Thu, 29 Aug 2019 16:24:57 +0300 Subject: [PATCH 53/68] Update test_client_list_jobs.py --- bigquery/samples/tests/test_client_list_jobs.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/bigquery/samples/tests/test_client_list_jobs.py b/bigquery/samples/tests/test_client_list_jobs.py index 8f558ed4cfc4..959124c5177b 100644 --- a/bigquery/samples/tests/test_client_list_jobs.py +++ b/bigquery/samples/tests/test_client_list_jobs.py @@ -20,10 +20,14 @@ def test_client_list_jobs(capsys, client): job = create_job.create_job(client) + client.cancel_job(job.job_id) + job.cancel() client_list_jobs.client_list_jobs(client) - client.cancel_job(job.job_id, location="US") out, err = capsys.readouterr() - assert "Last 10 jobs:\n{}".format(job.job_id) in out - assert "Jobs from the last ten minutes:\n{}".format(job.job_id) in out - assert "Last 10 jobs run by all users:\n{}".format(job.job_id) in out - assert "Last 10 jobs done:\n{}".format(job.job_id) in out + print(out) + raise ValueError + # assert "Started job: {}".format(job.job_id) in out + # assert "Last 10 jobs:\n{}".format(job.job_id) in out + # assert "Jobs from the last ten minutes:\n{}".format(job.job_id) in out + # assert "Last 10 jobs run by all users:\n{}".format(job.job_id) in out + # assert "Last 10 jobs done:\n{}".format(job.job_id) in out From 92fca34a9101b3418d5bb776743a8e4effe07aa1 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Thu, 29 Aug 2019 16:25:01 +0300 Subject: [PATCH 54/68] Update test_table_exists.py --- bigquery/samples/tests/test_table_exists.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/bigquery/samples/tests/test_table_exists.py b/bigquery/samples/tests/test_table_exists.py index bf5e059b022e..232d77fbcb60 100644 --- a/bigquery/samples/tests/test_table_exists.py +++ b/bigquery/samples/tests/test_table_exists.py @@ -20,16 +20,11 @@ def test_table_exists(capsys, client, random_table_id): - schema = [ - bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), - ] - table_exists.table_exists(client, random_table_id) out, err = capsys.readouterr() - assert "Table {} is not found".format(random_table_id) in out - table = bigquery.Table(random_table_id, schema=schema) + assert "Table {} is not found.".format(random_table_id) in out + table = bigquery.Table(random_table_id) table = client.create_table(table) table_exists.table_exists(client, random_table_id) out, err = capsys.readouterr() - assert "Table {} already exists".format(random_table_id) in out + assert "Table {} already exists.".format(random_table_id) in out From 17fb64489ada717bf73fd96191a79e19f302b3fc Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Thu, 29 Aug 2019 16:25:23 +0300 Subject: [PATCH 55/68] cosmetic chgs --- bigquery/samples/client_query.py | 5 ++--- bigquery/samples/copy_table.py | 5 ++--- bigquery/samples/table_exists.py | 4 ++-- bigquery/samples/table_insert_rows.py | 2 +- bigquery/samples/tests/test_copy_table.py | 2 +- bigquery/samples/tests/test_table_insert_rows.py | 2 +- 6 files changed, 9 insertions(+), 11 deletions(-) diff --git a/bigquery/samples/client_query.py b/bigquery/samples/client_query.py index cdd371c0066f..8bcaf4d517e3 100644 --- a/bigquery/samples/client_query.py +++ b/bigquery/samples/client_query.py @@ -35,8 +35,7 @@ def client_query(client): ) print("The query data:") - for row in query_job: # API request - fetches results + for row in query_job: # Row values can be accessed by field name or index - if row[0] == row.name == row["name"]: - print(row) + print(row) # [END bigquery_query] diff --git a/bigquery/samples/copy_table.py b/bigquery/samples/copy_table.py index 181e58eeeda7..f25194d80b7f 100644 --- a/bigquery/samples/copy_table.py +++ b/bigquery/samples/copy_table.py @@ -38,10 +38,9 @@ def copy_table(client, dataset_id, table_id): # Location must match that of the source and destination tables. location="US", ) - - job.result() # Waits for job to complete. + job.result() dest_table = client.get_table(dest_table) if dest_table.num_rows == orig_table.num_rows: - print("A copy of the table created") + print("A copy of the table created.") # [END bigquery_copy_table] diff --git a/bigquery/samples/table_exists.py b/bigquery/samples/table_exists.py index 314addd0d214..e8aa1c433da4 100644 --- a/bigquery/samples/table_exists.py +++ b/bigquery/samples/table_exists.py @@ -23,7 +23,7 @@ def table_exists(client, table_id): try: client.get_table(table_id) - print("Table {} already exists".format(table_id)) + print("Table {} already exists.".format(table_id)) except NotFound: - print("Table {} is not found".format(table_id)) + print("Table {} is not found.".format(table_id)) # [END bigquery_table_exists] diff --git a/bigquery/samples/table_insert_rows.py b/bigquery/samples/table_insert_rows.py index 43fa6617ea4c..d203ab651139 100644 --- a/bigquery/samples/table_insert_rows.py +++ b/bigquery/samples/table_insert_rows.py @@ -30,5 +30,5 @@ def table_insert_rows(client, table_id): errors = client.insert_rows(table, rows_to_insert) if errors == []: - print("New rows have been added") + print("New rows have been added.") # [END bigquery_table_insert_rows] diff --git a/bigquery/samples/tests/test_copy_table.py b/bigquery/samples/tests/test_copy_table.py index 765847b0aa48..7826e8d8cfac 100644 --- a/bigquery/samples/tests/test_copy_table.py +++ b/bigquery/samples/tests/test_copy_table.py @@ -20,4 +20,4 @@ def test_copy_table(capsys, client, dataset_id, table_with_data_id): copy_table.copy_table(client, dataset_id, table_with_data_id) out, err = capsys.readouterr() - assert "A copy of the table created" in out + assert "A copy of the table created." in out diff --git a/bigquery/samples/tests/test_table_insert_rows.py b/bigquery/samples/tests/test_table_insert_rows.py index f9e04bccb37f..95d119dbdc93 100644 --- a/bigquery/samples/tests/test_table_insert_rows.py +++ b/bigquery/samples/tests/test_table_insert_rows.py @@ -30,4 +30,4 @@ def test_table_insert_rows(capsys, client, random_table_id): table_insert_rows.table_insert_rows(client, random_table_id) out, err = capsys.readouterr() - assert "New rows have been added" in out + assert "New rows have been added." in out From 2dfcc3d2fed4321f184aae361910db110c716e73 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Thu, 29 Aug 2019 16:38:01 +0300 Subject: [PATCH 56/68] Update test_client_list_jobs.py --- bigquery/samples/tests/test_client_list_jobs.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/bigquery/samples/tests/test_client_list_jobs.py b/bigquery/samples/tests/test_client_list_jobs.py index 959124c5177b..825c984971ac 100644 --- a/bigquery/samples/tests/test_client_list_jobs.py +++ b/bigquery/samples/tests/test_client_list_jobs.py @@ -24,10 +24,8 @@ def test_client_list_jobs(capsys, client): job.cancel() client_list_jobs.client_list_jobs(client) out, err = capsys.readouterr() - print(out) - raise ValueError - # assert "Started job: {}".format(job.job_id) in out - # assert "Last 10 jobs:\n{}".format(job.job_id) in out - # assert "Jobs from the last ten minutes:\n{}".format(job.job_id) in out - # assert "Last 10 jobs run by all users:\n{}".format(job.job_id) in out - # assert "Last 10 jobs done:\n{}".format(job.job_id) in out + assert "Started job: {}".format(job.job_id) in out + assert "Last 10 jobs:\n{}".format(job.job_id) in out + assert "Jobs from the last ten minutes:\n{}".format(job.job_id) in out + assert "Last 10 jobs run by all users:\n{}".format(job.job_id) in out + assert "Last 10 jobs done:\n{}".format(job.job_id) in out From 902fdd9a1e748d5189ca4261306934324fa2b264 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 30 Aug 2019 11:00:00 +0300 Subject: [PATCH 57/68] cosmetic chgs --- bigquery/samples/client_list_jobs.py | 4 ++-- bigquery/samples/copy_table.py | 2 +- bigquery/samples/tests/test_client_query.py | 1 + bigquery/samples/tests/test_table_insert_rows.py | 1 - 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bigquery/samples/client_list_jobs.py b/bigquery/samples/client_list_jobs.py index 9067ac613efc..a9741f52bb2e 100644 --- a/bigquery/samples/client_list_jobs.py +++ b/bigquery/samples/client_list_jobs.py @@ -19,12 +19,12 @@ def client_list_jobs(client): # TODO(developer): Import the client library. # from google.cloud import bigquery + import datetime + # TODO(developer): Construct a BigQuery client object. # project = "my_project" # replace with your project ID # client = bigquery.Client(project=project) - import datetime - # List the 10 most recent jobs in reverse chronological order. # Omit the max_results parameter to list jobs from the past 6 months. print("Last 10 jobs:") diff --git a/bigquery/samples/copy_table.py b/bigquery/samples/copy_table.py index f25194d80b7f..97b0299e3214 100644 --- a/bigquery/samples/copy_table.py +++ b/bigquery/samples/copy_table.py @@ -28,8 +28,8 @@ def copy_table(client, dataset_id, table_id): # TODO(developer): Set table_id to the ID of the original table. # table_id = "your-project.your_dataset.your_table_name" - dataset = client.get_dataset(dataset_id) orig_table = client.get_table(table_id) + dataset = client.get_dataset(dataset_id) dest_table = dataset.table("destination_table") job = client.copy_table( diff --git a/bigquery/samples/tests/test_client_query.py b/bigquery/samples/tests/test_client_query.py index fc53ca73e6e2..47969b880180 100644 --- a/bigquery/samples/tests/test_client_query.py +++ b/bigquery/samples/tests/test_client_query.py @@ -21,3 +21,4 @@ def test_client_query(capsys, client): client_query.client_query(client) out, err = capsys.readouterr() assert "The query data:" in out + assert "Row(" in out diff --git a/bigquery/samples/tests/test_table_insert_rows.py b/bigquery/samples/tests/test_table_insert_rows.py index 95d119dbdc93..913d2a670fdd 100644 --- a/bigquery/samples/tests/test_table_insert_rows.py +++ b/bigquery/samples/tests/test_table_insert_rows.py @@ -27,7 +27,6 @@ def test_table_insert_rows(capsys, client, random_table_id): table = bigquery.Table(random_table_id, schema=schema) table = client.create_table(table) - table_insert_rows.table_insert_rows(client, random_table_id) out, err = capsys.readouterr() assert "New rows have been added." in out From df4f6c17de002e114e3921dd548cdd00f7bc00dd Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 3 Sep 2019 10:20:31 +0300 Subject: [PATCH 58/68] Merge branch 'master' into second-five-v2 --- bigquery/docs/snippets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bigquery/docs/snippets.py b/bigquery/docs/snippets.py index cc25db138d35..1ec0bc42c51a 100644 --- a/bigquery/docs/snippets.py +++ b/bigquery/docs/snippets.py @@ -1284,6 +1284,7 @@ def test_copy_table_multiple_source(client, to_delete): assert dest_table.num_rows == 2 +@pytest.mark.skip(reason="Backend responds with a 500 internal error.") def test_copy_table_cmek(client, to_delete): dataset_id = "copy_table_cmek_{}".format(_millis()) dest_dataset = bigquery.Dataset(client.dataset(dataset_id)) From ccf3dcdaef84be4bfea8ea85bc0b2cad8436db9b Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 3 Sep 2019 10:27:41 +0300 Subject: [PATCH 59/68] update client_list_jobs * fixed assertion error by removing incorrect assertion statements. --- bigquery/samples/client_list_jobs.py | 3 +-- bigquery/samples/tests/test_client_list_jobs.py | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/bigquery/samples/client_list_jobs.py b/bigquery/samples/client_list_jobs.py index a9741f52bb2e..08eb4fbd99ef 100644 --- a/bigquery/samples/client_list_jobs.py +++ b/bigquery/samples/client_list_jobs.py @@ -22,8 +22,7 @@ def client_list_jobs(client): import datetime # TODO(developer): Construct a BigQuery client object. - # project = "my_project" # replace with your project ID - # client = bigquery.Client(project=project) + # client = bigquery.Client() # List the 10 most recent jobs in reverse chronological order. # Omit the max_results parameter to list jobs from the past 6 months. diff --git a/bigquery/samples/tests/test_client_list_jobs.py b/bigquery/samples/tests/test_client_list_jobs.py index 825c984971ac..011e081fdee4 100644 --- a/bigquery/samples/tests/test_client_list_jobs.py +++ b/bigquery/samples/tests/test_client_list_jobs.py @@ -25,7 +25,7 @@ def test_client_list_jobs(capsys, client): client_list_jobs.client_list_jobs(client) out, err = capsys.readouterr() assert "Started job: {}".format(job.job_id) in out - assert "Last 10 jobs:\n{}".format(job.job_id) in out - assert "Jobs from the last ten minutes:\n{}".format(job.job_id) in out - assert "Last 10 jobs run by all users:\n{}".format(job.job_id) in out - assert "Last 10 jobs done:\n{}".format(job.job_id) in out + assert "Last 10 jobs:" in out + assert "Jobs from the last ten minutes:" in out + assert "Last 10 jobs run by all users:" in out + assert "Last 10 jobs done:" in out From 150bcf2d10f1809c0eb56076ce7b34aa6e3bb284 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 11 Sep 2019 13:14:00 +0300 Subject: [PATCH 60/68] cosmetic update --- bigquery/samples/client_query.py | 7 ++----- bigquery/samples/copy_table.py | 11 +++++------ bigquery/samples/table_exists.py | 2 +- bigquery/samples/table_insert_rows.py | 4 ++-- bigquery/samples/tests/test_table_insert_rows.py | 1 + 5 files changed, 11 insertions(+), 14 deletions(-) diff --git a/bigquery/samples/client_query.py b/bigquery/samples/client_query.py index 8bcaf4d517e3..024d7159e27d 100644 --- a/bigquery/samples/client_query.py +++ b/bigquery/samples/client_query.py @@ -27,12 +27,9 @@ def client_query(client): 'WHERE state = "TX" ' "LIMIT 100" ) - query_job = client.query( - query, - # Location must match that of the dataset(s) referenced in the query. - location="US", - ) + query, location="US" # Must match the destination dataset(s) location. + ) # API request. print("The query data:") for row in query_job: diff --git a/bigquery/samples/copy_table.py b/bigquery/samples/copy_table.py index 97b0299e3214..4d95775f32f9 100644 --- a/bigquery/samples/copy_table.py +++ b/bigquery/samples/copy_table.py @@ -28,19 +28,18 @@ def copy_table(client, dataset_id, table_id): # TODO(developer): Set table_id to the ID of the original table. # table_id = "your-project.your_dataset.your_table_name" - orig_table = client.get_table(table_id) - dataset = client.get_dataset(dataset_id) + orig_table = client.get_table(table_id) # API request. + dataset = client.get_dataset(dataset_id) # API request. dest_table = dataset.table("destination_table") job = client.copy_table( orig_table, dest_table, - # Location must match that of the source and destination tables. - location="US", + location="US", # Must match the source and destination tables location. ) - job.result() + job.result() # Waits for job to complete. - dest_table = client.get_table(dest_table) + dest_table = client.get_table(table_id) # API request. if dest_table.num_rows == orig_table.num_rows: print("A copy of the table created.") # [END bigquery_copy_table] diff --git a/bigquery/samples/table_exists.py b/bigquery/samples/table_exists.py index e8aa1c433da4..9339b08767a9 100644 --- a/bigquery/samples/table_exists.py +++ b/bigquery/samples/table_exists.py @@ -22,7 +22,7 @@ def table_exists(client, table_id): # table_id = "your-project.your_dataset.your_table" try: - client.get_table(table_id) + client.get_table(table_id) # API request. print("Table {} already exists.".format(table_id)) except NotFound: print("Table {} is not found.".format(table_id)) diff --git a/bigquery/samples/table_insert_rows.py b/bigquery/samples/table_insert_rows.py index d203ab651139..e8f945225f91 100644 --- a/bigquery/samples/table_insert_rows.py +++ b/bigquery/samples/table_insert_rows.py @@ -25,10 +25,10 @@ def table_insert_rows(client, table_id): # TODO(developer): Set table_id to the ID of the model to fetch. # table_id = "your-project.your_dataset.your_table" - table = client.get_table(table_id) + table = client.get_table(table_id) # API request. rows_to_insert = [(u"Phred Phlyntstone", 32), (u"Wylma Phlyntstone", 29)] - errors = client.insert_rows(table, rows_to_insert) + errors = client.insert_rows(table, rows_to_insert) # API request. if errors == []: print("New rows have been added.") # [END bigquery_table_insert_rows] diff --git a/bigquery/samples/tests/test_table_insert_rows.py b/bigquery/samples/tests/test_table_insert_rows.py index 913d2a670fdd..95d119dbdc93 100644 --- a/bigquery/samples/tests/test_table_insert_rows.py +++ b/bigquery/samples/tests/test_table_insert_rows.py @@ -27,6 +27,7 @@ def test_table_insert_rows(capsys, client, random_table_id): table = bigquery.Table(random_table_id, schema=schema) table = client.create_table(table) + table_insert_rows.table_insert_rows(client, random_table_id) out, err = capsys.readouterr() assert "New rows have been added." in out From ab504bf45fec749d6329149cd84c6cf9c9cf74a5 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 25 Sep 2019 10:47:40 +0300 Subject: [PATCH 61/68] comments rephrasing --- bigquery/samples/client_query.py | 2 +- bigquery/samples/copy_table.py | 6 +++--- bigquery/samples/table_exists.py | 2 +- bigquery/samples/table_insert_rows.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bigquery/samples/client_query.py b/bigquery/samples/client_query.py index 024d7159e27d..5daad9e24719 100644 --- a/bigquery/samples/client_query.py +++ b/bigquery/samples/client_query.py @@ -29,7 +29,7 @@ def client_query(client): ) query_job = client.query( query, location="US" # Must match the destination dataset(s) location. - ) # API request. + ) # Make an API request. print("The query data:") for row in query_job: diff --git a/bigquery/samples/copy_table.py b/bigquery/samples/copy_table.py index 4d95775f32f9..829881b44096 100644 --- a/bigquery/samples/copy_table.py +++ b/bigquery/samples/copy_table.py @@ -28,8 +28,8 @@ def copy_table(client, dataset_id, table_id): # TODO(developer): Set table_id to the ID of the original table. # table_id = "your-project.your_dataset.your_table_name" - orig_table = client.get_table(table_id) # API request. - dataset = client.get_dataset(dataset_id) # API request. + orig_table = client.get_table(table_id) # Make an API request. + dataset = client.get_dataset(dataset_id) # Make an API request. dest_table = dataset.table("destination_table") job = client.copy_table( @@ -39,7 +39,7 @@ def copy_table(client, dataset_id, table_id): ) job.result() # Waits for job to complete. - dest_table = client.get_table(table_id) # API request. + dest_table = client.get_table(table_id) # Make an API request. if dest_table.num_rows == orig_table.num_rows: print("A copy of the table created.") # [END bigquery_copy_table] diff --git a/bigquery/samples/table_exists.py b/bigquery/samples/table_exists.py index 9339b08767a9..a011e6e2915d 100644 --- a/bigquery/samples/table_exists.py +++ b/bigquery/samples/table_exists.py @@ -22,7 +22,7 @@ def table_exists(client, table_id): # table_id = "your-project.your_dataset.your_table" try: - client.get_table(table_id) # API request. + client.get_table(table_id) # Make an API request. print("Table {} already exists.".format(table_id)) except NotFound: print("Table {} is not found.".format(table_id)) diff --git a/bigquery/samples/table_insert_rows.py b/bigquery/samples/table_insert_rows.py index e8f945225f91..e2f949b635a6 100644 --- a/bigquery/samples/table_insert_rows.py +++ b/bigquery/samples/table_insert_rows.py @@ -25,10 +25,10 @@ def table_insert_rows(client, table_id): # TODO(developer): Set table_id to the ID of the model to fetch. # table_id = "your-project.your_dataset.your_table" - table = client.get_table(table_id) # API request. + table = client.get_table(table_id) # Make an API request. rows_to_insert = [(u"Phred Phlyntstone", 32), (u"Wylma Phlyntstone", 29)] - errors = client.insert_rows(table, rows_to_insert) # API request. + errors = client.insert_rows(table, rows_to_insert) # Make an API request. if errors == []: print("New rows have been added.") # [END bigquery_table_insert_rows] From e67d1720bdc165470bfb080e72b00455ceecfbad Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 27 Sep 2019 14:33:02 +0300 Subject: [PATCH 62/68] updated as requested - copy_table - client_query - test_copy_table - test_client_query --- bigquery/samples/client_query.py | 17 +++++++++------- bigquery/samples/copy_table.py | 22 ++++++++------------- bigquery/samples/tests/test_client_query.py | 2 +- bigquery/samples/tests/test_copy_table.py | 8 ++++++-- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/bigquery/samples/client_query.py b/bigquery/samples/client_query.py index 5daad9e24719..9dccfd38cbcf 100644 --- a/bigquery/samples/client_query.py +++ b/bigquery/samples/client_query.py @@ -22,17 +22,20 @@ def client_query(client): # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() - query = ( - "SELECT name FROM `bigquery-public-data.usa_names.usa_1910_2013` " - 'WHERE state = "TX" ' - "LIMIT 100" - ) + query = """ + SELECT name, SUM(number) as total_people + FROM `bigquery-public-data.usa_names.usa_1910_2013` + WHERE state = 'TX' + GROUP BY name, state + ORDER BY total_people DESC + LIMIT 20 + """ query_job = client.query( query, location="US" # Must match the destination dataset(s) location. ) # Make an API request. print("The query data:") for row in query_job: - # Row values can be accessed by field name or index - print(row) + # Row values can be accessed by field name or index. + print("name={}, count={}".format(row[0], row["total_people"])) # [END bigquery_query] diff --git a/bigquery/samples/copy_table.py b/bigquery/samples/copy_table.py index 829881b44096..f6ebd91470eb 100644 --- a/bigquery/samples/copy_table.py +++ b/bigquery/samples/copy_table.py @@ -13,7 +13,7 @@ # limitations under the License. -def copy_table(client, dataset_id, table_id): +def copy_table(client, source_table_id, destination_table_id): # [START bigquery_copy_table] # TODO(developer): Import the client library. @@ -22,24 +22,18 @@ def copy_table(client, dataset_id, table_id): # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() - # TODO(developer): Set dataset_id to the ID of the dataset where to copy a table. - # dataset_id = "your-project.your_dataset" + # TODO(developer): Set source_table_id to the ID of the original table. + # source_table_id = "your-project.source_dataset.source_table" - # TODO(developer): Set table_id to the ID of the original table. - # table_id = "your-project.your_dataset.your_table_name" - - orig_table = client.get_table(table_id) # Make an API request. - dataset = client.get_dataset(dataset_id) # Make an API request. - dest_table = dataset.table("destination_table") + # TODO(developer): Set destination_table_id to the ID of the destination table. + # destination_table_id = "your-project.destination_dataset.destination_table" job = client.copy_table( - orig_table, - dest_table, + source_table_id, + destination_table_id, location="US", # Must match the source and destination tables location. ) job.result() # Waits for job to complete. - dest_table = client.get_table(table_id) # Make an API request. - if dest_table.num_rows == orig_table.num_rows: - print("A copy of the table created.") + print("A copy of the table created.") # [END bigquery_copy_table] diff --git a/bigquery/samples/tests/test_client_query.py b/bigquery/samples/tests/test_client_query.py index 47969b880180..fd5b8e7edd97 100644 --- a/bigquery/samples/tests/test_client_query.py +++ b/bigquery/samples/tests/test_client_query.py @@ -21,4 +21,4 @@ def test_client_query(capsys, client): client_query.client_query(client) out, err = capsys.readouterr() assert "The query data:" in out - assert "Row(" in out + assert "name=James, count=272793" in out diff --git a/bigquery/samples/tests/test_copy_table.py b/bigquery/samples/tests/test_copy_table.py index 7826e8d8cfac..6d7de2d9132c 100644 --- a/bigquery/samples/tests/test_copy_table.py +++ b/bigquery/samples/tests/test_copy_table.py @@ -16,8 +16,12 @@ from .. import copy_table -def test_copy_table(capsys, client, dataset_id, table_with_data_id): +def test_copy_table(capsys, client, table_with_data_id, random_table_id): - copy_table.copy_table(client, dataset_id, table_with_data_id) + copy_table.copy_table(client, table_with_data_id, random_table_id) out, err = capsys.readouterr() assert "A copy of the table created." in out + assert ( + client.get_table(random_table_id).num_rows + == client.get_table(table_with_data_id).num_rows + ) From da4a2bcc13c06409cb27fbcae474b48fcc2662ec Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 27 Sep 2019 14:35:14 +0300 Subject: [PATCH 63/68] comment lines update --- bigquery/samples/add_empty_column.py | 2 +- bigquery/samples/browse_table_data.py | 2 +- bigquery/samples/create_routine_ddl.py | 4 ---- bigquery/samples/create_table.py | 2 +- bigquery/samples/delete_dataset.py | 2 +- bigquery/samples/delete_dataset_labels.py | 2 +- bigquery/samples/delete_table.py | 2 +- bigquery/samples/get_dataset.py | 6 +++--- bigquery/samples/get_dataset_labels.py | 2 +- bigquery/samples/update_dataset_access.py | 4 +++- bigquery/samples/update_dataset_default_table_expiration.py | 2 +- 11 files changed, 14 insertions(+), 16 deletions(-) diff --git a/bigquery/samples/add_empty_column.py b/bigquery/samples/add_empty_column.py index 4f0b971e577a..bd531898eb29 100644 --- a/bigquery/samples/add_empty_column.py +++ b/bigquery/samples/add_empty_column.py @@ -28,7 +28,7 @@ def add_empty_column(client, table_id): table = client.get_table(table_id) # Make an API request. original_schema = table.schema - new_schema = original_schema[:] # creates a copy of the schema + new_schema = original_schema[:] # Creates a copy of the schema. new_schema.append(bigquery.SchemaField("phone", "STRING")) table.schema = new_schema diff --git a/bigquery/samples/browse_table_data.py b/bigquery/samples/browse_table_data.py index bba8dc434dd9..78d1d351a7a7 100644 --- a/bigquery/samples/browse_table_data.py +++ b/bigquery/samples/browse_table_data.py @@ -39,7 +39,7 @@ def browse_table_data(client, table_id): # Specify selected fields to limit the results to certain columns. table = client.get_table(table_id) # Make an API request. - fields = table.schema[:2] # first two columns + fields = table.schema[:2] # First two columns. rows_iter = client.list_rows(table_id, selected_fields=fields, max_results=10) rows = list(rows_iter) print("Selected {} columns from table {}.".format(len(rows_iter.schema), table_id)) diff --git a/bigquery/samples/create_routine_ddl.py b/bigquery/samples/create_routine_ddl.py index 836e0cdde34a..17cdd695986c 100644 --- a/bigquery/samples/create_routine_ddl.py +++ b/bigquery/samples/create_routine_ddl.py @@ -34,11 +34,7 @@ def create_routine_ddl(client, routine_id): """.format( routine_id ) - - # Initiate the query to create the routine. query_job = client.query(sql) # Make an API request. - - # Wait for the query to complete. query_job.result() # Waits for the job to complete. print("Created routine {}".format(query_job.ddl_target_routine)) diff --git a/bigquery/samples/create_table.py b/bigquery/samples/create_table.py index b77812f7e0ce..ae26c57fed00 100644 --- a/bigquery/samples/create_table.py +++ b/bigquery/samples/create_table.py @@ -21,7 +21,7 @@ def create_table(client, table_id): # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() - # TODO(developer): Set table_id to the ID of the table to create + # TODO(developer): Set table_id to the ID of the table to create. # table_id = "your-project.your_dataset.your_table_name" schema = [ diff --git a/bigquery/samples/delete_dataset.py b/bigquery/samples/delete_dataset.py index 29302f099998..8ce95d953392 100644 --- a/bigquery/samples/delete_dataset.py +++ b/bigquery/samples/delete_dataset.py @@ -25,7 +25,7 @@ def delete_dataset(client, dataset_id): # TODO(developer): Set model_id to the ID of the model to fetch. # dataset_id = 'your-project.your_dataset' - # Use the delete_contents parameter to delete a dataset and its contents + # Use the delete_contents parameter to delete a dataset and its contents. # Use the not_found_ok parameter to not receive an error if the dataset has already been deleted. client.delete_dataset( dataset_id, delete_contents=True, not_found_ok=True diff --git a/bigquery/samples/delete_dataset_labels.py b/bigquery/samples/delete_dataset_labels.py index 425bc98dd96e..9e6493694ddc 100644 --- a/bigquery/samples/delete_dataset_labels.py +++ b/bigquery/samples/delete_dataset_labels.py @@ -27,7 +27,7 @@ def delete_dataset_labels(client, dataset_id): dataset = client.get_dataset(dataset_id) # Make an API request. - # To delete a label from a dataset, set its value to None + # To delete a label from a dataset, set its value to None. dataset.labels["color"] = None dataset = client.update_dataset(dataset, ["labels"]) # Make an API request. diff --git a/bigquery/samples/delete_table.py b/bigquery/samples/delete_table.py index 4c4377418556..b83a92890b09 100644 --- a/bigquery/samples/delete_table.py +++ b/bigquery/samples/delete_table.py @@ -26,7 +26,7 @@ def delete_table(client, table_id): # table_id = 'your-project.your_dataset.your_table' # If the table does not exist, delete_table raises - # google.api_core.exceptions.NotFound unless not_found_ok is True + # google.api_core.exceptions.NotFound unless not_found_ok is True. client.delete_table(table_id, not_found_ok=True) # Make an API request. print("Deleted table '{}'.".format(table_id)) # [END bigquery_delete_table] diff --git a/bigquery/samples/get_dataset.py b/bigquery/samples/get_dataset.py index cd35745c0dc5..bb3d4a0d4c40 100644 --- a/bigquery/samples/get_dataset.py +++ b/bigquery/samples/get_dataset.py @@ -35,7 +35,7 @@ def get_dataset(client, dataset_id): ) ) - # View dataset properties + # View dataset properties. print("Description: {}".format(dataset.description)) print("Labels:") labels = dataset.labels @@ -45,9 +45,9 @@ def get_dataset(client, dataset_id): else: print("\tDataset has no labels defined.") - # View tables in dataset + # View tables in dataset. print("Tables:") - tables = list(client.list_tables(dataset)) # API request(s) + tables = list(client.list_tables(dataset)) # Make an API request(s). if tables: for table in tables: print("\t{}".format(table.table_id)) diff --git a/bigquery/samples/get_dataset_labels.py b/bigquery/samples/get_dataset_labels.py index 46e38a3a9a56..411607f84664 100644 --- a/bigquery/samples/get_dataset_labels.py +++ b/bigquery/samples/get_dataset_labels.py @@ -27,7 +27,7 @@ def get_dataset_labels(client, dataset_id): dataset = client.get_dataset(dataset_id) # Make an API request. - # View dataset labels + # View dataset labels. print("Dataset ID: {}".format(dataset_id)) print("Labels:") if dataset.labels: diff --git a/bigquery/samples/update_dataset_access.py b/bigquery/samples/update_dataset_access.py index 134cf1b940cf..0383bd8a2e8d 100644 --- a/bigquery/samples/update_dataset_access.py +++ b/bigquery/samples/update_dataset_access.py @@ -36,7 +36,9 @@ def update_dataset_access(client, dataset_id): entries.append(entry) dataset.access_entries = entries - dataset = client.update_dataset(dataset, ["access_entries"]) # Make an API request. + dataset = client.update_dataset( + dataset, ["access_entries"] + ) # Make an API request. full_dataset_id = "{}.{}".format(dataset.project, dataset.dataset_id) print( diff --git a/bigquery/samples/update_dataset_default_table_expiration.py b/bigquery/samples/update_dataset_default_table_expiration.py index 7b68ede8d2be..8de354b1f21b 100644 --- a/bigquery/samples/update_dataset_default_table_expiration.py +++ b/bigquery/samples/update_dataset_default_table_expiration.py @@ -26,7 +26,7 @@ def update_dataset_default_table_expiration(client, dataset_id): # dataset_id = 'your-project.your_dataset' dataset = client.get_dataset(dataset_id) # Make an API request. - dataset.default_table_expiration_ms = 24 * 60 * 60 * 1000 # in milliseconds. + dataset.default_table_expiration_ms = 24 * 60 * 60 * 1000 # In milliseconds. dataset = client.update_dataset( dataset, ["default_table_expiration_ms"] From 88529514dc6544e443a969b5dafd53b94908fc0b Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 8 Oct 2019 10:47:57 +0300 Subject: [PATCH 64/68] lint fix --- bigquery/samples/update_dataset_access.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bigquery/samples/update_dataset_access.py b/bigquery/samples/update_dataset_access.py index 0383bd8a2e8d..134cf1b940cf 100644 --- a/bigquery/samples/update_dataset_access.py +++ b/bigquery/samples/update_dataset_access.py @@ -36,9 +36,7 @@ def update_dataset_access(client, dataset_id): entries.append(entry) dataset.access_entries = entries - dataset = client.update_dataset( - dataset, ["access_entries"] - ) # Make an API request. + dataset = client.update_dataset(dataset, ["access_entries"]) # Make an API request. full_dataset_id = "{}.{}".format(dataset.project, dataset.dataset_id) print( From 6d533b30b9800ef85411a78b8e6f16c8a69b5435 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 8 Oct 2019 10:56:37 +0300 Subject: [PATCH 65/68] comments update --- bigquery/samples/add_empty_column.py | 4 ++-- bigquery/samples/browse_table_data.py | 4 ++-- bigquery/samples/client_query.py | 2 +- bigquery/samples/create_dataset.py | 2 +- bigquery/samples/create_job.py | 2 +- bigquery/samples/create_routine.py | 2 +- bigquery/samples/create_routine_ddl.py | 2 +- bigquery/samples/create_table.py | 2 +- bigquery/samples/dataset_exists.py | 2 +- bigquery/samples/delete_dataset.py | 2 +- bigquery/samples/delete_dataset_labels.py | 4 ++-- bigquery/samples/delete_model.py | 2 +- bigquery/samples/delete_routine.py | 2 +- bigquery/samples/delete_table.py | 2 +- bigquery/samples/get_dataset.py | 2 +- bigquery/samples/get_dataset_labels.py | 2 +- bigquery/samples/get_model.py | 2 +- bigquery/samples/get_routine.py | 2 +- bigquery/samples/get_table.py | 2 +- bigquery/samples/label_dataset.py | 4 ++-- bigquery/samples/list_datasets.py | 2 +- bigquery/samples/list_datasets_by_label.py | 2 +- bigquery/samples/list_models.py | 2 +- bigquery/samples/list_routines.py | 2 +- bigquery/samples/list_tables.py | 2 +- bigquery/samples/load_table_dataframe.py | 4 ++-- bigquery/samples/query_to_arrow.py | 2 +- bigquery/samples/table_exists.py | 2 +- bigquery/samples/table_insert_rows.py | 4 ++-- bigquery/samples/update_dataset_access.py | 4 ++-- bigquery/samples/update_dataset_default_table_expiration.py | 4 ++-- bigquery/samples/update_dataset_description.py | 4 ++-- bigquery/samples/update_model.py | 4 ++-- 33 files changed, 43 insertions(+), 43 deletions(-) diff --git a/bigquery/samples/add_empty_column.py b/bigquery/samples/add_empty_column.py index bd531898eb29..608300c14d2d 100644 --- a/bigquery/samples/add_empty_column.py +++ b/bigquery/samples/add_empty_column.py @@ -25,14 +25,14 @@ def add_empty_column(client, table_id): # to add an empty column. # table_id = "your-project.your_dataset.your_table_name" - table = client.get_table(table_id) # Make an API request. + table = client.get_table(table_id) # Makes an API request. original_schema = table.schema new_schema = original_schema[:] # Creates a copy of the schema. new_schema.append(bigquery.SchemaField("phone", "STRING")) table.schema = new_schema - table = client.update_table(table, ["schema"]) # Make an API request. + table = client.update_table(table, ["schema"]) # Makes an API request. if len(table.schema) == len(original_schema) + 1 == len(new_schema): print("A new column has been added.") diff --git a/bigquery/samples/browse_table_data.py b/bigquery/samples/browse_table_data.py index 78d1d351a7a7..a0b090a0a835 100644 --- a/bigquery/samples/browse_table_data.py +++ b/bigquery/samples/browse_table_data.py @@ -26,7 +26,7 @@ def browse_table_data(client, table_id): # table_id = "your-project.your_dataset.your_table_name" # Download all rows from a table. - rows_iter = client.list_rows(table_id) # Make an API request. + rows_iter = client.list_rows(table_id) # Makes an API request. # Iterate over rows to make the API requests to fetch row data. rows = list(rows_iter) @@ -38,7 +38,7 @@ def browse_table_data(client, table_id): print("Downloaded {} rows from table {}".format(len(rows), table_id)) # Specify selected fields to limit the results to certain columns. - table = client.get_table(table_id) # Make an API request. + table = client.get_table(table_id) # Makes an API request. fields = table.schema[:2] # First two columns. rows_iter = client.list_rows(table_id, selected_fields=fields, max_results=10) rows = list(rows_iter) diff --git a/bigquery/samples/client_query.py b/bigquery/samples/client_query.py index 9dccfd38cbcf..76a38dabe377 100644 --- a/bigquery/samples/client_query.py +++ b/bigquery/samples/client_query.py @@ -32,7 +32,7 @@ def client_query(client): """ query_job = client.query( query, location="US" # Must match the destination dataset(s) location. - ) # Make an API request. + ) # Makes an API request. print("The query data:") for row in query_job: diff --git a/bigquery/samples/create_dataset.py b/bigquery/samples/create_dataset.py index 3d64473a2321..b769fde2924d 100644 --- a/bigquery/samples/create_dataset.py +++ b/bigquery/samples/create_dataset.py @@ -33,6 +33,6 @@ def create_dataset(client, dataset_id): # Send the dataset to the API for creation. # Raises google.api_core.exceptions.Conflict if the Dataset already # exists within the project. - dataset = client.create_dataset(dataset) # Make an API request. + dataset = client.create_dataset(dataset) # Makes an API request. print("Created dataset {}.{}".format(client.project, dataset.dataset_id)) # [END bigquery_create_dataset] diff --git a/bigquery/samples/create_job.py b/bigquery/samples/create_job.py index 4f7f27a8e668..845399da5f62 100644 --- a/bigquery/samples/create_job.py +++ b/bigquery/samples/create_job.py @@ -33,7 +33,7 @@ def create_job(client): # The client libraries automatically generate a job ID. Override the # generated ID with either the job_id_prefix or job_id parameters. job_id_prefix="code_sample_", - ) # Make an API request. + ) # Makes an API request. print("Started job: {}".format(query_job.job_id)) # [END bigquery_create_job] diff --git a/bigquery/samples/create_routine.py b/bigquery/samples/create_routine.py index 424ee4ef5553..520953985450 100644 --- a/bigquery/samples/create_routine.py +++ b/bigquery/samples/create_routine.py @@ -40,7 +40,7 @@ def create_routine(client, routine_id): ], ) - routine = client.create_routine(routine) # Make an API request. + routine = client.create_routine(routine) # Makes an API request. print("Created routine {}".format(routine.reference)) # [END bigquery_create_routine] diff --git a/bigquery/samples/create_routine_ddl.py b/bigquery/samples/create_routine_ddl.py index 17cdd695986c..9c5025ebc07d 100644 --- a/bigquery/samples/create_routine_ddl.py +++ b/bigquery/samples/create_routine_ddl.py @@ -34,7 +34,7 @@ def create_routine_ddl(client, routine_id): """.format( routine_id ) - query_job = client.query(sql) # Make an API request. + query_job = client.query(sql) # Makes an API request. query_job.result() # Waits for the job to complete. print("Created routine {}".format(query_job.ddl_target_routine)) diff --git a/bigquery/samples/create_table.py b/bigquery/samples/create_table.py index ae26c57fed00..d9583dda6fae 100644 --- a/bigquery/samples/create_table.py +++ b/bigquery/samples/create_table.py @@ -30,7 +30,7 @@ def create_table(client, table_id): ] table = bigquery.Table(table_id, schema=schema) - table = client.create_table(table) # Make an API request. + table = client.create_table(table) # Makes an API request. print( "Created table {}.{}.{}".format(table.project, table.dataset_id, table.table_id) ) diff --git a/bigquery/samples/dataset_exists.py b/bigquery/samples/dataset_exists.py index b8b53b8a4580..035a78c2a534 100644 --- a/bigquery/samples/dataset_exists.py +++ b/bigquery/samples/dataset_exists.py @@ -22,7 +22,7 @@ def dataset_exists(client, dataset_id): # dataset_id = "your-project.your_dataset" try: - client.get_dataset(dataset_id) # Make an API request. + client.get_dataset(dataset_id) # Makes an API request. print("Dataset {} already exists".format(dataset_id)) except NotFound: print("Dataset {} is not found".format(dataset_id)) diff --git a/bigquery/samples/delete_dataset.py b/bigquery/samples/delete_dataset.py index 8ce95d953392..c8cce5a440a3 100644 --- a/bigquery/samples/delete_dataset.py +++ b/bigquery/samples/delete_dataset.py @@ -29,7 +29,7 @@ def delete_dataset(client, dataset_id): # Use the not_found_ok parameter to not receive an error if the dataset has already been deleted. client.delete_dataset( dataset_id, delete_contents=True, not_found_ok=True - ) # Make an API request. + ) # Makes an API request. print("Deleted dataset '{}'.".format(dataset_id)) # [END bigquery_delete_dataset] diff --git a/bigquery/samples/delete_dataset_labels.py b/bigquery/samples/delete_dataset_labels.py index 9e6493694ddc..b008e44d6009 100644 --- a/bigquery/samples/delete_dataset_labels.py +++ b/bigquery/samples/delete_dataset_labels.py @@ -25,12 +25,12 @@ def delete_dataset_labels(client, dataset_id): # TODO(developer): Set dataset_id to the ID of the dataset to fetch. # dataset_id = "your-project.your_dataset" - dataset = client.get_dataset(dataset_id) # Make an API request. + dataset = client.get_dataset(dataset_id) # Makes an API request. # To delete a label from a dataset, set its value to None. dataset.labels["color"] = None - dataset = client.update_dataset(dataset, ["labels"]) # Make an API request. + dataset = client.update_dataset(dataset, ["labels"]) # Makes an API request. print("Labels deleted from {}".format(dataset_id)) # [END bigquery_delete_label_dataset] return dataset diff --git a/bigquery/samples/delete_model.py b/bigquery/samples/delete_model.py index b6f32a59ebd9..0a65a0a4d8a7 100644 --- a/bigquery/samples/delete_model.py +++ b/bigquery/samples/delete_model.py @@ -26,7 +26,7 @@ def delete_model(client, model_id): # TODO(developer): Set model_id to the ID of the model to fetch. # model_id = 'your-project.your_dataset.your_model' - client.delete_model(model_id) # Make an API request. + client.delete_model(model_id) # Makes an API request. print("Deleted model '{}'.".format(model_id)) # [END bigquery_delete_model] diff --git a/bigquery/samples/delete_routine.py b/bigquery/samples/delete_routine.py index c20b49837b75..6d0457c93114 100644 --- a/bigquery/samples/delete_routine.py +++ b/bigquery/samples/delete_routine.py @@ -25,7 +25,7 @@ def delete_routine(client, routine_id): # TODO(developer): Set the fully-qualified ID for the routine. # routine_id = "my-project.my_dataset.my_routine" - client.delete_routine(routine_id) # Make an API request. + client.delete_routine(routine_id) # Makes an API request. print("Deleted routine {}.".format(routine_id)) # [END bigquery_delete_routine] diff --git a/bigquery/samples/delete_table.py b/bigquery/samples/delete_table.py index b83a92890b09..76780dfec938 100644 --- a/bigquery/samples/delete_table.py +++ b/bigquery/samples/delete_table.py @@ -27,6 +27,6 @@ def delete_table(client, table_id): # If the table does not exist, delete_table raises # google.api_core.exceptions.NotFound unless not_found_ok is True. - client.delete_table(table_id, not_found_ok=True) # Make an API request. + client.delete_table(table_id, not_found_ok=True) # Makes an API request. print("Deleted table '{}'.".format(table_id)) # [END bigquery_delete_table] diff --git a/bigquery/samples/get_dataset.py b/bigquery/samples/get_dataset.py index bb3d4a0d4c40..620e9f4008c3 100644 --- a/bigquery/samples/get_dataset.py +++ b/bigquery/samples/get_dataset.py @@ -25,7 +25,7 @@ def get_dataset(client, dataset_id): # TODO(developer): Set dataset_id to the ID of the dataset to fetch. # dataset_id = 'your-project.your_dataset' - dataset = client.get_dataset(dataset_id) # Make an API request. + dataset = client.get_dataset(dataset_id) # Makes an API request. full_dataset_id = "{}.{}".format(dataset.project, dataset.dataset_id) friendly_name = dataset.friendly_name diff --git a/bigquery/samples/get_dataset_labels.py b/bigquery/samples/get_dataset_labels.py index 411607f84664..8c808ff172b7 100644 --- a/bigquery/samples/get_dataset_labels.py +++ b/bigquery/samples/get_dataset_labels.py @@ -25,7 +25,7 @@ def get_dataset_labels(client, dataset_id): # TODO(developer): Set dataset_id to the ID of the dataset to fetch. # dataset_id = "your-project.your_dataset" - dataset = client.get_dataset(dataset_id) # Make an API request. + dataset = client.get_dataset(dataset_id) # Makes an API request. # View dataset labels. print("Dataset ID: {}".format(dataset_id)) diff --git a/bigquery/samples/get_model.py b/bigquery/samples/get_model.py index 0ebd59c9d067..98e881920e0d 100644 --- a/bigquery/samples/get_model.py +++ b/bigquery/samples/get_model.py @@ -26,7 +26,7 @@ def get_model(client, model_id): # TODO(developer): Set model_id to the ID of the model to fetch. # model_id = 'your-project.your_dataset.your_model' - model = client.get_model(model_id) # Make an API request. + model = client.get_model(model_id) # Makes an API request. full_model_id = "{}.{}.{}".format(model.project, model.dataset_id, model.model_id) friendly_name = model.friendly_name diff --git a/bigquery/samples/get_routine.py b/bigquery/samples/get_routine.py index da4e89f57f19..e5eff0afc643 100644 --- a/bigquery/samples/get_routine.py +++ b/bigquery/samples/get_routine.py @@ -25,7 +25,7 @@ def get_routine(client, routine_id): # TODO(developer): Set the fully-qualified ID for the routine. # routine_id = "my-project.my_dataset.my_routine" - routine = client.get_routine(routine_id) # Make an API request. + routine = client.get_routine(routine_id) # Makes an API request. print("Routine '{}':".format(routine.reference)) print("\tType: '{}'".format(routine.type_)) diff --git a/bigquery/samples/get_table.py b/bigquery/samples/get_table.py index 201b8808a846..aa11f0d68d48 100644 --- a/bigquery/samples/get_table.py +++ b/bigquery/samples/get_table.py @@ -25,7 +25,7 @@ def get_table(client, table_id): # TODO(developer): Set table_id to the ID of the model to fetch. # table_id = 'your-project.your_dataset.your_table' - table = client.get_table(table_id) # Make an API request. + table = client.get_table(table_id) # Makes an API request. # View table properties print( diff --git a/bigquery/samples/label_dataset.py b/bigquery/samples/label_dataset.py index 019b2aa374a0..87ceed267e9e 100644 --- a/bigquery/samples/label_dataset.py +++ b/bigquery/samples/label_dataset.py @@ -25,9 +25,9 @@ def label_dataset(client, dataset_id): # TODO(developer): Set dataset_id to the ID of the dataset to fetch. # dataset_id = "your-project.your_dataset" - dataset = client.get_dataset(dataset_id) # Make an API request. + dataset = client.get_dataset(dataset_id) # Makes an API request. dataset.labels = {"color": "green"} - dataset = client.update_dataset(dataset, ["labels"]) # Make an API request. + dataset = client.update_dataset(dataset, ["labels"]) # Makes an API request. print("Labels added to {}".format(dataset_id)) # [END bigquery_label_dataset] diff --git a/bigquery/samples/list_datasets.py b/bigquery/samples/list_datasets.py index 77ae8c785d22..d10dbc602926 100644 --- a/bigquery/samples/list_datasets.py +++ b/bigquery/samples/list_datasets.py @@ -22,7 +22,7 @@ def list_datasets(client): # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() - datasets = list(client.list_datasets()) # Make an API request. + datasets = list(client.list_datasets()) # Makes an API request. project = client.project if datasets: diff --git a/bigquery/samples/list_datasets_by_label.py b/bigquery/samples/list_datasets_by_label.py index 9fa939ad0c19..102df2ca7698 100644 --- a/bigquery/samples/list_datasets_by_label.py +++ b/bigquery/samples/list_datasets_by_label.py @@ -23,7 +23,7 @@ def list_datasets_by_label(client): # client = bigquery.Client() label_filter = "labels.color:green" - datasets = list(client.list_datasets(filter=label_filter)) # Make an API request. + datasets = list(client.list_datasets(filter=label_filter)) # Makes an API request. if datasets: print("Datasets filtered by {}:".format(label_filter)) diff --git a/bigquery/samples/list_models.py b/bigquery/samples/list_models.py index a2477ffc795b..316710dae11b 100644 --- a/bigquery/samples/list_models.py +++ b/bigquery/samples/list_models.py @@ -27,7 +27,7 @@ def list_models(client, dataset_id): # the models you are listing. # dataset_id = 'your-project.your_dataset' - models = client.list_models(dataset_id) # Make an API request. + models = client.list_models(dataset_id) # Makes an API request. print("Models contained in '{}':".format(dataset_id)) for model in models: diff --git a/bigquery/samples/list_routines.py b/bigquery/samples/list_routines.py index 5eaad0cec8f4..38651b22917a 100644 --- a/bigquery/samples/list_routines.py +++ b/bigquery/samples/list_routines.py @@ -26,7 +26,7 @@ def list_routines(client, dataset_id): # the routines you are listing. # dataset_id = 'your-project.your_dataset' - routines = client.list_routines(dataset_id) # Make an API request. + routines = client.list_routines(dataset_id) # Makes an API request. print("Routines contained in dataset {}:".format(dataset_id)) for routine in routines: diff --git a/bigquery/samples/list_tables.py b/bigquery/samples/list_tables.py index d7576616e191..a765ee1f69ae 100644 --- a/bigquery/samples/list_tables.py +++ b/bigquery/samples/list_tables.py @@ -26,7 +26,7 @@ def list_tables(client, dataset_id): # the tables you are listing. # dataset_id = 'your-project.your_dataset' - tables = client.list_tables(dataset_id) # Make an API request. + tables = client.list_tables(dataset_id) # Makes an API request. print("Tables contained in '{}':".format(dataset_id)) for table in tables: diff --git a/bigquery/samples/load_table_dataframe.py b/bigquery/samples/load_table_dataframe.py index f08712d4dc32..3dd1c6f67e8c 100644 --- a/bigquery/samples/load_table_dataframe.py +++ b/bigquery/samples/load_table_dataframe.py @@ -65,10 +65,10 @@ def load_table_dataframe(client, table_id): table_id, job_config=job_config, location="US", # Must match the destination dataset location. - ) # Make an API request. + ) # Makes an API request. job.result() # Waits for the job to complete. - table = client.get_table(table_id) # Make an API request. + table = client.get_table(table_id) # Makes an API request. print( "Loaded {} rows and {} columns to {}".format( table.num_rows, len(table.schema), table_id diff --git a/bigquery/samples/query_to_arrow.py b/bigquery/samples/query_to_arrow.py index 4cc69d4e902a..6468339f3056 100644 --- a/bigquery/samples/query_to_arrow.py +++ b/bigquery/samples/query_to_arrow.py @@ -41,7 +41,7 @@ def query_to_arrow(client): CROSS JOIN UNNEST(r.participants) as participant; """ query_job = client.query(sql) - arrow_table = query_job.to_arrow() # Make an API request. + arrow_table = query_job.to_arrow() # Makes an API request. print( "Downloaded {} rows, {} columns.".format( diff --git a/bigquery/samples/table_exists.py b/bigquery/samples/table_exists.py index a011e6e2915d..d63878191540 100644 --- a/bigquery/samples/table_exists.py +++ b/bigquery/samples/table_exists.py @@ -22,7 +22,7 @@ def table_exists(client, table_id): # table_id = "your-project.your_dataset.your_table" try: - client.get_table(table_id) # Make an API request. + client.get_table(table_id) # Makes an API request. print("Table {} already exists.".format(table_id)) except NotFound: print("Table {} is not found.".format(table_id)) diff --git a/bigquery/samples/table_insert_rows.py b/bigquery/samples/table_insert_rows.py index e2f949b635a6..d255847892d5 100644 --- a/bigquery/samples/table_insert_rows.py +++ b/bigquery/samples/table_insert_rows.py @@ -25,10 +25,10 @@ def table_insert_rows(client, table_id): # TODO(developer): Set table_id to the ID of the model to fetch. # table_id = "your-project.your_dataset.your_table" - table = client.get_table(table_id) # Make an API request. + table = client.get_table(table_id) # Makes an API request. rows_to_insert = [(u"Phred Phlyntstone", 32), (u"Wylma Phlyntstone", 29)] - errors = client.insert_rows(table, rows_to_insert) # Make an API request. + errors = client.insert_rows(table, rows_to_insert) # Makes an API request. if errors == []: print("New rows have been added.") # [END bigquery_table_insert_rows] diff --git a/bigquery/samples/update_dataset_access.py b/bigquery/samples/update_dataset_access.py index 134cf1b940cf..a31c8e0f6d6d 100644 --- a/bigquery/samples/update_dataset_access.py +++ b/bigquery/samples/update_dataset_access.py @@ -24,7 +24,7 @@ def update_dataset_access(client, dataset_id): # TODO(developer): Set dataset_id to the ID of the dataset to fetch. # dataset_id = 'your-project.your_dataset' - dataset = client.get_dataset(dataset_id) # Make an API request. + dataset = client.get_dataset(dataset_id) # Makes an API request. entry = bigquery.AccessEntry( role="READER", @@ -36,7 +36,7 @@ def update_dataset_access(client, dataset_id): entries.append(entry) dataset.access_entries = entries - dataset = client.update_dataset(dataset, ["access_entries"]) # Make an API request. + dataset = client.update_dataset(dataset, ["access_entries"]) # Makes an API request. full_dataset_id = "{}.{}".format(dataset.project, dataset.dataset_id) print( diff --git a/bigquery/samples/update_dataset_default_table_expiration.py b/bigquery/samples/update_dataset_default_table_expiration.py index 8de354b1f21b..79881f7cc2df 100644 --- a/bigquery/samples/update_dataset_default_table_expiration.py +++ b/bigquery/samples/update_dataset_default_table_expiration.py @@ -25,12 +25,12 @@ def update_dataset_default_table_expiration(client, dataset_id): # TODO(developer): Set dataset_id to the ID of the dataset to fetch. # dataset_id = 'your-project.your_dataset' - dataset = client.get_dataset(dataset_id) # Make an API request. + dataset = client.get_dataset(dataset_id) # Makes an API request. dataset.default_table_expiration_ms = 24 * 60 * 60 * 1000 # In milliseconds. dataset = client.update_dataset( dataset, ["default_table_expiration_ms"] - ) # Make an API request. + ) # Makes an API request. full_dataset_id = "{}.{}".format(dataset.project, dataset.dataset_id) print( diff --git a/bigquery/samples/update_dataset_description.py b/bigquery/samples/update_dataset_description.py index 08eed8da2b64..260757cc4bbd 100644 --- a/bigquery/samples/update_dataset_description.py +++ b/bigquery/samples/update_dataset_description.py @@ -25,9 +25,9 @@ def update_dataset_description(client, dataset_id): # TODO(developer): Set dataset_id to the ID of the dataset to fetch. # dataset_id = 'your-project.your_dataset' - dataset = client.get_dataset(dataset_id) # Make an API request. + dataset = client.get_dataset(dataset_id) # Makes an API request. dataset.description = "Updated description." - dataset = client.update_dataset(dataset, ["description"]) # Make an API request. + dataset = client.update_dataset(dataset, ["description"]) # Makes an API request. full_dataset_id = "{}.{}".format(dataset.project, dataset.dataset_id) print( diff --git a/bigquery/samples/update_model.py b/bigquery/samples/update_model.py index 7583c410e1ef..8a71e4218ed9 100644 --- a/bigquery/samples/update_model.py +++ b/bigquery/samples/update_model.py @@ -26,9 +26,9 @@ def update_model(client, model_id): # TODO(developer): Set model_id to the ID of the model to fetch. # model_id = 'your-project.your_dataset.your_model' - model = client.get_model(model_id) # Make an API request. + model = client.get_model(model_id) # Makes an API request. model.description = "This model was modified from a Python program." - model = client.update_model(model, ["description"]) # Make an API request. + model = client.update_model(model, ["description"]) # Makes an API request. full_model_id = "{}.{}.{}".format(model.project, model.dataset_id, model.model_id) print( From e55f698eb1378a6ee4859b539abc96dbd90f08fe Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 8 Oct 2019 13:47:00 +0300 Subject: [PATCH 66/68] Update update_dataset_access.py --- bigquery/samples/update_dataset_access.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bigquery/samples/update_dataset_access.py b/bigquery/samples/update_dataset_access.py index a31c8e0f6d6d..1315a16ecb34 100644 --- a/bigquery/samples/update_dataset_access.py +++ b/bigquery/samples/update_dataset_access.py @@ -36,7 +36,9 @@ def update_dataset_access(client, dataset_id): entries.append(entry) dataset.access_entries = entries - dataset = client.update_dataset(dataset, ["access_entries"]) # Makes an API request. + dataset = client.update_dataset( + dataset, ["access_entries"] + ) # Makes an API request. full_dataset_id = "{}.{}".format(dataset.project, dataset.dataset_id) print( From ef884b8e5a42869d6e600a9c0bd1b0991d70a1ac Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 9 Oct 2019 10:36:45 +0300 Subject: [PATCH 67/68] revert commit with -s --- bigquery/samples/add_empty_column.py | 4 ++-- bigquery/samples/browse_table_data.py | 4 ++-- bigquery/samples/client_query.py | 2 +- bigquery/samples/create_dataset.py | 2 +- bigquery/samples/create_job.py | 2 +- bigquery/samples/create_routine.py | 2 +- bigquery/samples/create_routine_ddl.py | 2 +- bigquery/samples/create_table.py | 2 +- bigquery/samples/dataset_exists.py | 2 +- bigquery/samples/delete_dataset.py | 2 +- bigquery/samples/delete_dataset_labels.py | 4 ++-- bigquery/samples/delete_model.py | 2 +- bigquery/samples/delete_routine.py | 2 +- bigquery/samples/delete_table.py | 2 +- bigquery/samples/get_dataset.py | 2 +- bigquery/samples/get_dataset_labels.py | 2 +- bigquery/samples/get_model.py | 2 +- bigquery/samples/get_routine.py | 2 +- bigquery/samples/get_table.py | 2 +- bigquery/samples/label_dataset.py | 4 ++-- bigquery/samples/list_datasets.py | 2 +- bigquery/samples/list_datasets_by_label.py | 2 +- bigquery/samples/list_models.py | 2 +- bigquery/samples/list_routines.py | 2 +- bigquery/samples/list_tables.py | 2 +- bigquery/samples/load_table_dataframe.py | 4 ++-- bigquery/samples/query_to_arrow.py | 2 +- bigquery/samples/table_exists.py | 2 +- bigquery/samples/table_insert_rows.py | 4 ++-- bigquery/samples/update_dataset_access.py | 6 ++---- bigquery/samples/update_dataset_default_table_expiration.py | 4 ++-- bigquery/samples/update_dataset_description.py | 4 ++-- bigquery/samples/update_model.py | 4 ++-- 33 files changed, 43 insertions(+), 45 deletions(-) diff --git a/bigquery/samples/add_empty_column.py b/bigquery/samples/add_empty_column.py index 608300c14d2d..bd531898eb29 100644 --- a/bigquery/samples/add_empty_column.py +++ b/bigquery/samples/add_empty_column.py @@ -25,14 +25,14 @@ def add_empty_column(client, table_id): # to add an empty column. # table_id = "your-project.your_dataset.your_table_name" - table = client.get_table(table_id) # Makes an API request. + table = client.get_table(table_id) # Make an API request. original_schema = table.schema new_schema = original_schema[:] # Creates a copy of the schema. new_schema.append(bigquery.SchemaField("phone", "STRING")) table.schema = new_schema - table = client.update_table(table, ["schema"]) # Makes an API request. + table = client.update_table(table, ["schema"]) # Make an API request. if len(table.schema) == len(original_schema) + 1 == len(new_schema): print("A new column has been added.") diff --git a/bigquery/samples/browse_table_data.py b/bigquery/samples/browse_table_data.py index a0b090a0a835..78d1d351a7a7 100644 --- a/bigquery/samples/browse_table_data.py +++ b/bigquery/samples/browse_table_data.py @@ -26,7 +26,7 @@ def browse_table_data(client, table_id): # table_id = "your-project.your_dataset.your_table_name" # Download all rows from a table. - rows_iter = client.list_rows(table_id) # Makes an API request. + rows_iter = client.list_rows(table_id) # Make an API request. # Iterate over rows to make the API requests to fetch row data. rows = list(rows_iter) @@ -38,7 +38,7 @@ def browse_table_data(client, table_id): print("Downloaded {} rows from table {}".format(len(rows), table_id)) # Specify selected fields to limit the results to certain columns. - table = client.get_table(table_id) # Makes an API request. + table = client.get_table(table_id) # Make an API request. fields = table.schema[:2] # First two columns. rows_iter = client.list_rows(table_id, selected_fields=fields, max_results=10) rows = list(rows_iter) diff --git a/bigquery/samples/client_query.py b/bigquery/samples/client_query.py index 76a38dabe377..9dccfd38cbcf 100644 --- a/bigquery/samples/client_query.py +++ b/bigquery/samples/client_query.py @@ -32,7 +32,7 @@ def client_query(client): """ query_job = client.query( query, location="US" # Must match the destination dataset(s) location. - ) # Makes an API request. + ) # Make an API request. print("The query data:") for row in query_job: diff --git a/bigquery/samples/create_dataset.py b/bigquery/samples/create_dataset.py index b769fde2924d..3d64473a2321 100644 --- a/bigquery/samples/create_dataset.py +++ b/bigquery/samples/create_dataset.py @@ -33,6 +33,6 @@ def create_dataset(client, dataset_id): # Send the dataset to the API for creation. # Raises google.api_core.exceptions.Conflict if the Dataset already # exists within the project. - dataset = client.create_dataset(dataset) # Makes an API request. + dataset = client.create_dataset(dataset) # Make an API request. print("Created dataset {}.{}".format(client.project, dataset.dataset_id)) # [END bigquery_create_dataset] diff --git a/bigquery/samples/create_job.py b/bigquery/samples/create_job.py index 845399da5f62..4f7f27a8e668 100644 --- a/bigquery/samples/create_job.py +++ b/bigquery/samples/create_job.py @@ -33,7 +33,7 @@ def create_job(client): # The client libraries automatically generate a job ID. Override the # generated ID with either the job_id_prefix or job_id parameters. job_id_prefix="code_sample_", - ) # Makes an API request. + ) # Make an API request. print("Started job: {}".format(query_job.job_id)) # [END bigquery_create_job] diff --git a/bigquery/samples/create_routine.py b/bigquery/samples/create_routine.py index 520953985450..424ee4ef5553 100644 --- a/bigquery/samples/create_routine.py +++ b/bigquery/samples/create_routine.py @@ -40,7 +40,7 @@ def create_routine(client, routine_id): ], ) - routine = client.create_routine(routine) # Makes an API request. + routine = client.create_routine(routine) # Make an API request. print("Created routine {}".format(routine.reference)) # [END bigquery_create_routine] diff --git a/bigquery/samples/create_routine_ddl.py b/bigquery/samples/create_routine_ddl.py index 9c5025ebc07d..17cdd695986c 100644 --- a/bigquery/samples/create_routine_ddl.py +++ b/bigquery/samples/create_routine_ddl.py @@ -34,7 +34,7 @@ def create_routine_ddl(client, routine_id): """.format( routine_id ) - query_job = client.query(sql) # Makes an API request. + query_job = client.query(sql) # Make an API request. query_job.result() # Waits for the job to complete. print("Created routine {}".format(query_job.ddl_target_routine)) diff --git a/bigquery/samples/create_table.py b/bigquery/samples/create_table.py index d9583dda6fae..ae26c57fed00 100644 --- a/bigquery/samples/create_table.py +++ b/bigquery/samples/create_table.py @@ -30,7 +30,7 @@ def create_table(client, table_id): ] table = bigquery.Table(table_id, schema=schema) - table = client.create_table(table) # Makes an API request. + table = client.create_table(table) # Make an API request. print( "Created table {}.{}.{}".format(table.project, table.dataset_id, table.table_id) ) diff --git a/bigquery/samples/dataset_exists.py b/bigquery/samples/dataset_exists.py index 035a78c2a534..b8b53b8a4580 100644 --- a/bigquery/samples/dataset_exists.py +++ b/bigquery/samples/dataset_exists.py @@ -22,7 +22,7 @@ def dataset_exists(client, dataset_id): # dataset_id = "your-project.your_dataset" try: - client.get_dataset(dataset_id) # Makes an API request. + client.get_dataset(dataset_id) # Make an API request. print("Dataset {} already exists".format(dataset_id)) except NotFound: print("Dataset {} is not found".format(dataset_id)) diff --git a/bigquery/samples/delete_dataset.py b/bigquery/samples/delete_dataset.py index c8cce5a440a3..8ce95d953392 100644 --- a/bigquery/samples/delete_dataset.py +++ b/bigquery/samples/delete_dataset.py @@ -29,7 +29,7 @@ def delete_dataset(client, dataset_id): # Use the not_found_ok parameter to not receive an error if the dataset has already been deleted. client.delete_dataset( dataset_id, delete_contents=True, not_found_ok=True - ) # Makes an API request. + ) # Make an API request. print("Deleted dataset '{}'.".format(dataset_id)) # [END bigquery_delete_dataset] diff --git a/bigquery/samples/delete_dataset_labels.py b/bigquery/samples/delete_dataset_labels.py index b008e44d6009..9e6493694ddc 100644 --- a/bigquery/samples/delete_dataset_labels.py +++ b/bigquery/samples/delete_dataset_labels.py @@ -25,12 +25,12 @@ def delete_dataset_labels(client, dataset_id): # TODO(developer): Set dataset_id to the ID of the dataset to fetch. # dataset_id = "your-project.your_dataset" - dataset = client.get_dataset(dataset_id) # Makes an API request. + dataset = client.get_dataset(dataset_id) # Make an API request. # To delete a label from a dataset, set its value to None. dataset.labels["color"] = None - dataset = client.update_dataset(dataset, ["labels"]) # Makes an API request. + dataset = client.update_dataset(dataset, ["labels"]) # Make an API request. print("Labels deleted from {}".format(dataset_id)) # [END bigquery_delete_label_dataset] return dataset diff --git a/bigquery/samples/delete_model.py b/bigquery/samples/delete_model.py index 0a65a0a4d8a7..b6f32a59ebd9 100644 --- a/bigquery/samples/delete_model.py +++ b/bigquery/samples/delete_model.py @@ -26,7 +26,7 @@ def delete_model(client, model_id): # TODO(developer): Set model_id to the ID of the model to fetch. # model_id = 'your-project.your_dataset.your_model' - client.delete_model(model_id) # Makes an API request. + client.delete_model(model_id) # Make an API request. print("Deleted model '{}'.".format(model_id)) # [END bigquery_delete_model] diff --git a/bigquery/samples/delete_routine.py b/bigquery/samples/delete_routine.py index 6d0457c93114..c20b49837b75 100644 --- a/bigquery/samples/delete_routine.py +++ b/bigquery/samples/delete_routine.py @@ -25,7 +25,7 @@ def delete_routine(client, routine_id): # TODO(developer): Set the fully-qualified ID for the routine. # routine_id = "my-project.my_dataset.my_routine" - client.delete_routine(routine_id) # Makes an API request. + client.delete_routine(routine_id) # Make an API request. print("Deleted routine {}.".format(routine_id)) # [END bigquery_delete_routine] diff --git a/bigquery/samples/delete_table.py b/bigquery/samples/delete_table.py index 76780dfec938..b83a92890b09 100644 --- a/bigquery/samples/delete_table.py +++ b/bigquery/samples/delete_table.py @@ -27,6 +27,6 @@ def delete_table(client, table_id): # If the table does not exist, delete_table raises # google.api_core.exceptions.NotFound unless not_found_ok is True. - client.delete_table(table_id, not_found_ok=True) # Makes an API request. + client.delete_table(table_id, not_found_ok=True) # Make an API request. print("Deleted table '{}'.".format(table_id)) # [END bigquery_delete_table] diff --git a/bigquery/samples/get_dataset.py b/bigquery/samples/get_dataset.py index 620e9f4008c3..bb3d4a0d4c40 100644 --- a/bigquery/samples/get_dataset.py +++ b/bigquery/samples/get_dataset.py @@ -25,7 +25,7 @@ def get_dataset(client, dataset_id): # TODO(developer): Set dataset_id to the ID of the dataset to fetch. # dataset_id = 'your-project.your_dataset' - dataset = client.get_dataset(dataset_id) # Makes an API request. + dataset = client.get_dataset(dataset_id) # Make an API request. full_dataset_id = "{}.{}".format(dataset.project, dataset.dataset_id) friendly_name = dataset.friendly_name diff --git a/bigquery/samples/get_dataset_labels.py b/bigquery/samples/get_dataset_labels.py index 8c808ff172b7..411607f84664 100644 --- a/bigquery/samples/get_dataset_labels.py +++ b/bigquery/samples/get_dataset_labels.py @@ -25,7 +25,7 @@ def get_dataset_labels(client, dataset_id): # TODO(developer): Set dataset_id to the ID of the dataset to fetch. # dataset_id = "your-project.your_dataset" - dataset = client.get_dataset(dataset_id) # Makes an API request. + dataset = client.get_dataset(dataset_id) # Make an API request. # View dataset labels. print("Dataset ID: {}".format(dataset_id)) diff --git a/bigquery/samples/get_model.py b/bigquery/samples/get_model.py index 98e881920e0d..0ebd59c9d067 100644 --- a/bigquery/samples/get_model.py +++ b/bigquery/samples/get_model.py @@ -26,7 +26,7 @@ def get_model(client, model_id): # TODO(developer): Set model_id to the ID of the model to fetch. # model_id = 'your-project.your_dataset.your_model' - model = client.get_model(model_id) # Makes an API request. + model = client.get_model(model_id) # Make an API request. full_model_id = "{}.{}.{}".format(model.project, model.dataset_id, model.model_id) friendly_name = model.friendly_name diff --git a/bigquery/samples/get_routine.py b/bigquery/samples/get_routine.py index e5eff0afc643..da4e89f57f19 100644 --- a/bigquery/samples/get_routine.py +++ b/bigquery/samples/get_routine.py @@ -25,7 +25,7 @@ def get_routine(client, routine_id): # TODO(developer): Set the fully-qualified ID for the routine. # routine_id = "my-project.my_dataset.my_routine" - routine = client.get_routine(routine_id) # Makes an API request. + routine = client.get_routine(routine_id) # Make an API request. print("Routine '{}':".format(routine.reference)) print("\tType: '{}'".format(routine.type_)) diff --git a/bigquery/samples/get_table.py b/bigquery/samples/get_table.py index aa11f0d68d48..201b8808a846 100644 --- a/bigquery/samples/get_table.py +++ b/bigquery/samples/get_table.py @@ -25,7 +25,7 @@ def get_table(client, table_id): # TODO(developer): Set table_id to the ID of the model to fetch. # table_id = 'your-project.your_dataset.your_table' - table = client.get_table(table_id) # Makes an API request. + table = client.get_table(table_id) # Make an API request. # View table properties print( diff --git a/bigquery/samples/label_dataset.py b/bigquery/samples/label_dataset.py index 87ceed267e9e..019b2aa374a0 100644 --- a/bigquery/samples/label_dataset.py +++ b/bigquery/samples/label_dataset.py @@ -25,9 +25,9 @@ def label_dataset(client, dataset_id): # TODO(developer): Set dataset_id to the ID of the dataset to fetch. # dataset_id = "your-project.your_dataset" - dataset = client.get_dataset(dataset_id) # Makes an API request. + dataset = client.get_dataset(dataset_id) # Make an API request. dataset.labels = {"color": "green"} - dataset = client.update_dataset(dataset, ["labels"]) # Makes an API request. + dataset = client.update_dataset(dataset, ["labels"]) # Make an API request. print("Labels added to {}".format(dataset_id)) # [END bigquery_label_dataset] diff --git a/bigquery/samples/list_datasets.py b/bigquery/samples/list_datasets.py index d10dbc602926..77ae8c785d22 100644 --- a/bigquery/samples/list_datasets.py +++ b/bigquery/samples/list_datasets.py @@ -22,7 +22,7 @@ def list_datasets(client): # TODO(developer): Construct a BigQuery client object. # client = bigquery.Client() - datasets = list(client.list_datasets()) # Makes an API request. + datasets = list(client.list_datasets()) # Make an API request. project = client.project if datasets: diff --git a/bigquery/samples/list_datasets_by_label.py b/bigquery/samples/list_datasets_by_label.py index 102df2ca7698..9fa939ad0c19 100644 --- a/bigquery/samples/list_datasets_by_label.py +++ b/bigquery/samples/list_datasets_by_label.py @@ -23,7 +23,7 @@ def list_datasets_by_label(client): # client = bigquery.Client() label_filter = "labels.color:green" - datasets = list(client.list_datasets(filter=label_filter)) # Makes an API request. + datasets = list(client.list_datasets(filter=label_filter)) # Make an API request. if datasets: print("Datasets filtered by {}:".format(label_filter)) diff --git a/bigquery/samples/list_models.py b/bigquery/samples/list_models.py index 316710dae11b..a2477ffc795b 100644 --- a/bigquery/samples/list_models.py +++ b/bigquery/samples/list_models.py @@ -27,7 +27,7 @@ def list_models(client, dataset_id): # the models you are listing. # dataset_id = 'your-project.your_dataset' - models = client.list_models(dataset_id) # Makes an API request. + models = client.list_models(dataset_id) # Make an API request. print("Models contained in '{}':".format(dataset_id)) for model in models: diff --git a/bigquery/samples/list_routines.py b/bigquery/samples/list_routines.py index 38651b22917a..5eaad0cec8f4 100644 --- a/bigquery/samples/list_routines.py +++ b/bigquery/samples/list_routines.py @@ -26,7 +26,7 @@ def list_routines(client, dataset_id): # the routines you are listing. # dataset_id = 'your-project.your_dataset' - routines = client.list_routines(dataset_id) # Makes an API request. + routines = client.list_routines(dataset_id) # Make an API request. print("Routines contained in dataset {}:".format(dataset_id)) for routine in routines: diff --git a/bigquery/samples/list_tables.py b/bigquery/samples/list_tables.py index a765ee1f69ae..d7576616e191 100644 --- a/bigquery/samples/list_tables.py +++ b/bigquery/samples/list_tables.py @@ -26,7 +26,7 @@ def list_tables(client, dataset_id): # the tables you are listing. # dataset_id = 'your-project.your_dataset' - tables = client.list_tables(dataset_id) # Makes an API request. + tables = client.list_tables(dataset_id) # Make an API request. print("Tables contained in '{}':".format(dataset_id)) for table in tables: diff --git a/bigquery/samples/load_table_dataframe.py b/bigquery/samples/load_table_dataframe.py index 3dd1c6f67e8c..f08712d4dc32 100644 --- a/bigquery/samples/load_table_dataframe.py +++ b/bigquery/samples/load_table_dataframe.py @@ -65,10 +65,10 @@ def load_table_dataframe(client, table_id): table_id, job_config=job_config, location="US", # Must match the destination dataset location. - ) # Makes an API request. + ) # Make an API request. job.result() # Waits for the job to complete. - table = client.get_table(table_id) # Makes an API request. + table = client.get_table(table_id) # Make an API request. print( "Loaded {} rows and {} columns to {}".format( table.num_rows, len(table.schema), table_id diff --git a/bigquery/samples/query_to_arrow.py b/bigquery/samples/query_to_arrow.py index 6468339f3056..4cc69d4e902a 100644 --- a/bigquery/samples/query_to_arrow.py +++ b/bigquery/samples/query_to_arrow.py @@ -41,7 +41,7 @@ def query_to_arrow(client): CROSS JOIN UNNEST(r.participants) as participant; """ query_job = client.query(sql) - arrow_table = query_job.to_arrow() # Makes an API request. + arrow_table = query_job.to_arrow() # Make an API request. print( "Downloaded {} rows, {} columns.".format( diff --git a/bigquery/samples/table_exists.py b/bigquery/samples/table_exists.py index d63878191540..a011e6e2915d 100644 --- a/bigquery/samples/table_exists.py +++ b/bigquery/samples/table_exists.py @@ -22,7 +22,7 @@ def table_exists(client, table_id): # table_id = "your-project.your_dataset.your_table" try: - client.get_table(table_id) # Makes an API request. + client.get_table(table_id) # Make an API request. print("Table {} already exists.".format(table_id)) except NotFound: print("Table {} is not found.".format(table_id)) diff --git a/bigquery/samples/table_insert_rows.py b/bigquery/samples/table_insert_rows.py index d255847892d5..e2f949b635a6 100644 --- a/bigquery/samples/table_insert_rows.py +++ b/bigquery/samples/table_insert_rows.py @@ -25,10 +25,10 @@ def table_insert_rows(client, table_id): # TODO(developer): Set table_id to the ID of the model to fetch. # table_id = "your-project.your_dataset.your_table" - table = client.get_table(table_id) # Makes an API request. + table = client.get_table(table_id) # Make an API request. rows_to_insert = [(u"Phred Phlyntstone", 32), (u"Wylma Phlyntstone", 29)] - errors = client.insert_rows(table, rows_to_insert) # Makes an API request. + errors = client.insert_rows(table, rows_to_insert) # Make an API request. if errors == []: print("New rows have been added.") # [END bigquery_table_insert_rows] diff --git a/bigquery/samples/update_dataset_access.py b/bigquery/samples/update_dataset_access.py index 1315a16ecb34..134cf1b940cf 100644 --- a/bigquery/samples/update_dataset_access.py +++ b/bigquery/samples/update_dataset_access.py @@ -24,7 +24,7 @@ def update_dataset_access(client, dataset_id): # TODO(developer): Set dataset_id to the ID of the dataset to fetch. # dataset_id = 'your-project.your_dataset' - dataset = client.get_dataset(dataset_id) # Makes an API request. + dataset = client.get_dataset(dataset_id) # Make an API request. entry = bigquery.AccessEntry( role="READER", @@ -36,9 +36,7 @@ def update_dataset_access(client, dataset_id): entries.append(entry) dataset.access_entries = entries - dataset = client.update_dataset( - dataset, ["access_entries"] - ) # Makes an API request. + dataset = client.update_dataset(dataset, ["access_entries"]) # Make an API request. full_dataset_id = "{}.{}".format(dataset.project, dataset.dataset_id) print( diff --git a/bigquery/samples/update_dataset_default_table_expiration.py b/bigquery/samples/update_dataset_default_table_expiration.py index 79881f7cc2df..8de354b1f21b 100644 --- a/bigquery/samples/update_dataset_default_table_expiration.py +++ b/bigquery/samples/update_dataset_default_table_expiration.py @@ -25,12 +25,12 @@ def update_dataset_default_table_expiration(client, dataset_id): # TODO(developer): Set dataset_id to the ID of the dataset to fetch. # dataset_id = 'your-project.your_dataset' - dataset = client.get_dataset(dataset_id) # Makes an API request. + dataset = client.get_dataset(dataset_id) # Make an API request. dataset.default_table_expiration_ms = 24 * 60 * 60 * 1000 # In milliseconds. dataset = client.update_dataset( dataset, ["default_table_expiration_ms"] - ) # Makes an API request. + ) # Make an API request. full_dataset_id = "{}.{}".format(dataset.project, dataset.dataset_id) print( diff --git a/bigquery/samples/update_dataset_description.py b/bigquery/samples/update_dataset_description.py index 260757cc4bbd..08eed8da2b64 100644 --- a/bigquery/samples/update_dataset_description.py +++ b/bigquery/samples/update_dataset_description.py @@ -25,9 +25,9 @@ def update_dataset_description(client, dataset_id): # TODO(developer): Set dataset_id to the ID of the dataset to fetch. # dataset_id = 'your-project.your_dataset' - dataset = client.get_dataset(dataset_id) # Makes an API request. + dataset = client.get_dataset(dataset_id) # Make an API request. dataset.description = "Updated description." - dataset = client.update_dataset(dataset, ["description"]) # Makes an API request. + dataset = client.update_dataset(dataset, ["description"]) # Make an API request. full_dataset_id = "{}.{}".format(dataset.project, dataset.dataset_id) print( diff --git a/bigquery/samples/update_model.py b/bigquery/samples/update_model.py index 8a71e4218ed9..7583c410e1ef 100644 --- a/bigquery/samples/update_model.py +++ b/bigquery/samples/update_model.py @@ -26,9 +26,9 @@ def update_model(client, model_id): # TODO(developer): Set model_id to the ID of the model to fetch. # model_id = 'your-project.your_dataset.your_model' - model = client.get_model(model_id) # Makes an API request. + model = client.get_model(model_id) # Make an API request. model.description = "This model was modified from a Python program." - model = client.update_model(model, ["description"]) # Makes an API request. + model = client.update_model(model, ["description"]) # Make an API request. full_model_id = "{}.{}.{}".format(model.project, model.dataset_id, model.model_id) print( From 50394cdb225312a8a933b040ba2c31e6a3b239a3 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 11 Oct 2019 12:01:39 +0300 Subject: [PATCH 68/68] delete -s from comment lines --- bigquery/samples/create_routine_ddl.py | 2 +- bigquery/samples/load_table_dataframe.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bigquery/samples/create_routine_ddl.py b/bigquery/samples/create_routine_ddl.py index 17cdd695986c..eb5af0388503 100644 --- a/bigquery/samples/create_routine_ddl.py +++ b/bigquery/samples/create_routine_ddl.py @@ -35,7 +35,7 @@ def create_routine_ddl(client, routine_id): routine_id ) query_job = client.query(sql) # Make an API request. - query_job.result() # Waits for the job to complete. + query_job.result() # Wait for the job to complete. print("Created routine {}".format(query_job.ddl_target_routine)) # [END bigquery_create_routine_ddl] diff --git a/bigquery/samples/load_table_dataframe.py b/bigquery/samples/load_table_dataframe.py index f08712d4dc32..ea6fe5d02384 100644 --- a/bigquery/samples/load_table_dataframe.py +++ b/bigquery/samples/load_table_dataframe.py @@ -66,7 +66,7 @@ def load_table_dataframe(client, table_id): job_config=job_config, location="US", # Must match the destination dataset location. ) # Make an API request. - job.result() # Waits for the job to complete. + job.result() # Wait for the job to complete. table = client.get_table(table_id) # Make an API request. print(