Skip to content

Commit

Permalink
Merge pull request #7721 from tvalentyn/fix_bq_client
Browse files Browse the repository at this point in the history 
[BEAM-6532] Regenerate BQ client to match apitools==0.5.26 and fix BQ io to work on Python 3
  • Loading branch information
@charlesccychen
charlesccychen authored Feb 4, 2019
2 parents 32ca551 + cd0a379 commit 5f6d31b
Show file tree
Hide file tree
Showing 3 changed files with 168 additions and 146 deletions.
4 changes: 3 additions & 1 deletion sdks/python/apache_beam/io/gcp/bigquery_tools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import json
import logging
import re
import sys
import time
import uuid
from builtins import object
Expand Down Expand Up @@ -177,7 +178,8 @@ class BigQueryWrapper(object):
def __init__(self, client=None):
self.client = client or bigquery.BigqueryV2(
http=get_new_http(),
credentials=auth.get_service_credentials())
credentials=auth.get_service_credentials(),
response_encoding=None if sys.version_info[0] < 3 else 'utf8')
self._unique_row_id = 0
# For testing scenarios where we pass in a client we do not want a
# randomized prefix for row IDs.
Expand Down
49 changes: 25 additions & 24 deletions sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def __init__(self, url='', credentials=None,
get_credentials=True, http=None, model=None,
log_request=False, log_response=False,
credentials_args=None, default_global_params=None,
additional_http_headers=None):
additional_http_headers=None, response_encoding=None):
"""Create a new bigquery handle."""
url = url or self.BASE_URL
super(BigqueryV2, self).__init__(
Expand All @@ -54,7 +54,8 @@ def __init__(self, url='', credentials=None,
log_request=log_request, log_response=log_response,
credentials_args=credentials_args,
default_global_params=default_global_params,
additional_http_headers=additional_http_headers)
additional_http_headers=additional_http_headers,
response_encoding=response_encoding)
self.datasets = self.DatasetsService(self)
self.jobs = self.JobsService(self)
self.projects = self.ProjectsService(self)
Expand All @@ -72,7 +73,7 @@ def __init__(self, client):
}

def Delete(self, request, global_params=None):
"""Deletes the dataset specified by the datasetId value. Before you can delete a dataset, you must delete all its tables, either manually or by specifying deleteContents. Immediately after deletion, you can create another dataset with the same name.
r"""Deletes the dataset specified by the datasetId value. Before you can delete a dataset, you must delete all its tables, either manually or by specifying deleteContents. Immediately after deletion, you can create another dataset with the same name.
Args:
request: (BigqueryDatasetsDeleteRequest) input message
Expand All @@ -98,7 +99,7 @@ def Delete(self, request, global_params=None):
)

def Get(self, request, global_params=None):
"""Returns the dataset specified by datasetID.
r"""Returns the dataset specified by datasetID.
Args:
request: (BigqueryDatasetsGetRequest) input message
Expand All @@ -124,7 +125,7 @@ def Get(self, request, global_params=None):
)

def Insert(self, request, global_params=None):
"""Creates a new empty dataset.
r"""Creates a new empty dataset.
Args:
request: (BigqueryDatasetsInsertRequest) input message
Expand All @@ -150,7 +151,7 @@ def Insert(self, request, global_params=None):
)

def List(self, request, global_params=None):
"""Lists all datasets in the specified project to which you have been granted the READER dataset role.
r"""Lists all datasets in the specified project to which you have been granted the READER dataset role.
Args:
request: (BigqueryDatasetsListRequest) input message
Expand All @@ -176,7 +177,7 @@ def List(self, request, global_params=None):
)

def Patch(self, request, global_params=None):
"""Updates information in an existing dataset. The update method replaces the entire dataset resource, whereas the patch method only replaces fields that are provided in the submitted dataset resource. This method supports patch semantics.
r"""Updates information in an existing dataset. The update method replaces the entire dataset resource, whereas the patch method only replaces fields that are provided in the submitted dataset resource. This method supports patch semantics.
Args:
request: (BigqueryDatasetsPatchRequest) input message
Expand All @@ -202,7 +203,7 @@ def Patch(self, request, global_params=None):
)

def Update(self, request, global_params=None):
"""Updates information in an existing dataset. The update method replaces the entire dataset resource, whereas the patch method only replaces fields that are provided in the submitted dataset resource.
r"""Updates information in an existing dataset. The update method replaces the entire dataset resource, whereas the patch method only replaces fields that are provided in the submitted dataset resource.
Args:
request: (BigqueryDatasetsUpdateRequest) input message
Expand Down Expand Up @@ -246,7 +247,7 @@ def __init__(self, client):
}

def Cancel(self, request, global_params=None):
"""Requests that a job be cancelled. This call will return immediately, and the client will need to poll for the job status to see if the cancel completed successfully. Cancelled jobs may still incur costs.
r"""Requests that a job be cancelled. This call will return immediately, and the client will need to poll for the job status to see if the cancel completed successfully. Cancelled jobs may still incur costs.
Args:
request: (BigqueryJobsCancelRequest) input message
Expand All @@ -272,7 +273,7 @@ def Cancel(self, request, global_params=None):
)

def Get(self, request, global_params=None):
"""Returns information about a specific job. Job information is available for a six month period after creation. Requires that you're the person who ran the job, or have the Is Owner project role.
r"""Returns information about a specific job. Job information is available for a six month period after creation. Requires that you're the person who ran the job, or have the Is Owner project role.
Args:
request: (BigqueryJobsGetRequest) input message
Expand All @@ -298,7 +299,7 @@ def Get(self, request, global_params=None):
)

def GetQueryResults(self, request, global_params=None):
"""Retrieves the results of a query job.
r"""Retrieves the results of a query job.
Args:
request: (BigqueryJobsGetQueryResultsRequest) input message
Expand All @@ -324,7 +325,7 @@ def GetQueryResults(self, request, global_params=None):
)

def Insert(self, request, global_params=None, upload=None):
"""Starts a new asynchronous job. Requires the Can View project role.
r"""Starts a new asynchronous job. Requires the Can View project role.
Args:
request: (BigqueryJobsInsertRequest) input message
Expand Down Expand Up @@ -354,7 +355,7 @@ def Insert(self, request, global_params=None, upload=None):
)

def List(self, request, global_params=None):
"""Lists all jobs that you started in the specified project. Job information is available for a six month period after creation. The job list is sorted in reverse chronological order, by job creation time. Requires the Can View project role, or the Is Owner project role if you set the allUsers property.
r"""Lists all jobs that you started in the specified project. Job information is available for a six month period after creation. The job list is sorted in reverse chronological order, by job creation time. Requires the Can View project role, or the Is Owner project role if you set the allUsers property.
Args:
request: (BigqueryJobsListRequest) input message
Expand All @@ -380,7 +381,7 @@ def List(self, request, global_params=None):
)

def Query(self, request, global_params=None):
"""Runs a BigQuery SQL query synchronously and returns query results if the query completes within a specified timeout.
r"""Runs a BigQuery SQL query synchronously and returns query results if the query completes within a specified timeout.
Args:
request: (BigqueryJobsQueryRequest) input message
Expand Down Expand Up @@ -416,7 +417,7 @@ def __init__(self, client):
}

def GetServiceAccount(self, request, global_params=None):
"""Returns the email address of the service account for your project used for interactions with Google Cloud KMS.
r"""Returns the email address of the service account for your project used for interactions with Google Cloud KMS.
Args:
request: (BigqueryProjectsGetServiceAccountRequest) input message
Expand All @@ -442,7 +443,7 @@ def GetServiceAccount(self, request, global_params=None):
)

def List(self, request, global_params=None):
"""Lists all projects to which you have been granted any project role.
r"""Lists all projects to which you have been granted any project role.
Args:
request: (BigqueryProjectsListRequest) input message
Expand Down Expand Up @@ -478,7 +479,7 @@ def __init__(self, client):
}

def InsertAll(self, request, global_params=None):
"""Streams data into BigQuery one record at a time without needing to run a load job. Requires the WRITER dataset role.
r"""Streams data into BigQuery one record at a time without needing to run a load job. Requires the WRITER dataset role.
Args:
request: (BigqueryTabledataInsertAllRequest) input message
Expand All @@ -504,7 +505,7 @@ def InsertAll(self, request, global_params=None):
)

def List(self, request, global_params=None):
"""Retrieves table data from a specified set of rows. Requires the READER dataset role.
r"""Retrieves table data from a specified set of rows. Requires the READER dataset role.
Args:
request: (BigqueryTabledataListRequest) input message
Expand Down Expand Up @@ -540,7 +541,7 @@ def __init__(self, client):
}

def Delete(self, request, global_params=None):
"""Deletes the table specified by tableId from the dataset. If the table contains data, all the data will be deleted.
r"""Deletes the table specified by tableId from the dataset. If the table contains data, all the data will be deleted.
Args:
request: (BigqueryTablesDeleteRequest) input message
Expand All @@ -566,7 +567,7 @@ def Delete(self, request, global_params=None):
)

def Get(self, request, global_params=None):
"""Gets the specified table resource by table ID. This method does not return the data in the table, it only returns the table resource, which describes the structure of this table.
r"""Gets the specified table resource by table ID. This method does not return the data in the table, it only returns the table resource, which describes the structure of this table.
Args:
request: (BigqueryTablesGetRequest) input message
Expand All @@ -592,7 +593,7 @@ def Get(self, request, global_params=None):
)

def Insert(self, request, global_params=None):
"""Creates a new, empty table in the dataset.
r"""Creates a new, empty table in the dataset.
Args:
request: (BigqueryTablesInsertRequest) input message
Expand All @@ -618,7 +619,7 @@ def Insert(self, request, global_params=None):
)

def List(self, request, global_params=None):
"""Lists all tables in the specified dataset. Requires the READER dataset role.
r"""Lists all tables in the specified dataset. Requires the READER dataset role.
Args:
request: (BigqueryTablesListRequest) input message
Expand All @@ -644,7 +645,7 @@ def List(self, request, global_params=None):
)

def Patch(self, request, global_params=None):
"""Updates information in an existing table. The update method replaces the entire table resource, whereas the patch method only replaces fields that are provided in the submitted table resource. This method supports patch semantics.
r"""Updates information in an existing table. The update method replaces the entire table resource, whereas the patch method only replaces fields that are provided in the submitted table resource. This method supports patch semantics.
Args:
request: (BigqueryTablesPatchRequest) input message
Expand All @@ -670,7 +671,7 @@ def Patch(self, request, global_params=None):
)

def Update(self, request, global_params=None):
"""Updates information in an existing table. The update method replaces the entire table resource, whereas the patch method only replaces fields that are provided in the submitted table resource.
r"""Updates information in an existing table. The update method replaces the entire table resource, whereas the patch method only replaces fields that are provided in the submitted table resource.
Args:
request: (BigqueryTablesUpdateRequest) input message
Expand Down
Loading

0 comments on commit 5f6d31b

Please sign in to comment.