Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(bigquery): add TypeError if wrong job_config type is passed to client job methods #9506

Merged
merged 3 commits into from
Oct 23, 2019
Merged

feat(bigquery): add TypeError if wrong job_config type is passed to client job methods #9506

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1c2cc40
feat(bigquery): add ValueError if wrong job_config type is passed to …
HemangChothani Oct 18, 2019
8475568
feat(bigquery): cosmetic changes
HemangChothani Oct 21, 2019
b644878
feat(bigquery): code refactor
HemangChothani Oct 21, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions bigquery/google/cloud/bigquery/_helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -658,3 +658,18 @@ def _build_resource_from_properties(obj, filter_fields):
partial[filter_field] = obj._properties[filter_field]

return partial


def _verify_job_config_type(job_config, expected_type, param_name="job_config"):
if not isinstance(job_config, expected_type):
msg = (
"Expected an instance of {expected_type} class for the {param_name} parameter, "
"but received {param_name} = {job_config}"
)
raise TypeError(
msg.format(
expected_type=expected_type.__name__,
param_name=param_name,
job_config=job_config,
)
)
69 changes: 63 additions & 6 deletions bigquery/google/cloud/bigquery/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@

from google.cloud.bigquery._helpers import _record_field_to_json
from google.cloud.bigquery._helpers import _str_or_none
from google.cloud.bigquery._helpers import _verify_job_config_type
from google.cloud.bigquery._http import Connection
from google.cloud.bigquery import _pandas_helpers
from google.cloud.bigquery.dataset import Dataset
Expand Down Expand Up @@ -1355,6 +1356,11 @@ def load_table_from_uri(

Returns:
google.cloud.bigquery.job.LoadJob: A new load job.

Raises:
TypeError:
If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.LoadJobConfig`
class.
"""
job_id = _make_job_id(job_id, job_id_prefix)

Expand All @@ -1370,6 +1376,10 @@ def load_table_from_uri(
source_uris = [source_uris]

destination = _table_arg_to_table_ref(destination, default_project=self.project)

if job_config:
_verify_job_config_type(job_config, google.cloud.bigquery.job.LoadJobConfig)

load_job = job.LoadJob(job_ref, source_uris, destination, self, job_config)
load_job._begin(retry=retry)

Expand Down Expand Up @@ -1436,6 +1446,10 @@ def load_table_from_file(
If ``size`` is not passed in and can not be determined, or if
the ``file_obj`` can be detected to be a file opened in text
mode.

TypeError:
If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.LoadJobConfig`
class.
"""
job_id = _make_job_id(job_id, job_id_prefix)

Expand All @@ -1447,6 +1461,8 @@ def load_table_from_file(

destination = _table_arg_to_table_ref(destination, default_project=self.project)
job_ref = job._JobReference(job_id, project=project, location=location)
if job_config:
_verify_job_config_type(job_config, google.cloud.bigquery.job.LoadJobConfig)
load_job = job.LoadJob(job_ref, None, destination, self, job_config)
job_resource = load_job.to_api_repr()

Expand Down Expand Up @@ -1545,16 +1561,22 @@ def load_table_from_dataframe(
If a usable parquet engine cannot be found. This method
requires :mod:`pyarrow` or :mod:`fastparquet` to be
installed.
TypeError:
If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.LoadJobConfig`
class.
"""
job_id = _make_job_id(job_id, job_id_prefix)

if job_config is None:
job_config = job.LoadJobConfig()
else:
if job_config:
_verify_job_config_type(job_config, google.cloud.bigquery.job.LoadJobConfig)
# Make a copy so that the job config isn't modified in-place.
job_config_properties = copy.deepcopy(job_config._properties)
job_config = job.LoadJobConfig()
job_config._properties = job_config_properties

else:
job_config = job.LoadJobConfig()

job_config.source_format = job.SourceFormat.PARQUET

if location is None:
Expand Down Expand Up @@ -1700,14 +1722,21 @@ def load_table_from_json(

Returns:
google.cloud.bigquery.job.LoadJob: A new load job.

Raises:
TypeError:
If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.LoadJobConfig`
class.
"""
job_id = _make_job_id(job_id, job_id_prefix)

if job_config is None:
job_config = job.LoadJobConfig()
else:
if job_config:
_verify_job_config_type(job_config, google.cloud.bigquery.job.LoadJobConfig)
# Make a copy so that the job config isn't modified in-place.
job_config = copy.deepcopy(job_config)
else:
job_config = job.LoadJobConfig()

job_config.source_format = job.SourceFormat.NEWLINE_DELIMITED_JSON

if job_config.schema is None:
Expand Down Expand Up @@ -1900,6 +1929,11 @@ def copy_table(

Returns:
google.cloud.bigquery.job.CopyJob: A new copy job instance.

Raises:
TypeError:
If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.CopyJobConfig`
class.
"""
job_id = _make_job_id(job_id, job_id_prefix)

Expand Down Expand Up @@ -1928,6 +1962,8 @@ def copy_table(

destination = _table_arg_to_table_ref(destination, default_project=self.project)

if job_config:
_verify_job_config_type(job_config, google.cloud.bigquery.job.CopyJobConfig)
copy_job = job.CopyJob(
job_ref, sources, destination, client=self, job_config=job_config
)
Expand Down Expand Up @@ -1985,6 +2021,11 @@ def extract_table(

Returns:
google.cloud.bigquery.job.ExtractJob: A new extract job instance.

Raises:
TypeError:
If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.ExtractJobConfig`
class.
"""
job_id = _make_job_id(job_id, job_id_prefix)

Expand All @@ -2000,6 +2041,10 @@ def extract_table(
if isinstance(destination_uris, six.string_types):
destination_uris = [destination_uris]

if job_config:
_verify_job_config_type(
job_config, google.cloud.bigquery.job.ExtractJobConfig
)
extract_job = job.ExtractJob(
job_ref, source, destination_uris, client=self, job_config=job_config
)
Expand Down Expand Up @@ -2049,6 +2094,11 @@ def query(

Returns:
google.cloud.bigquery.job.QueryJob: A new query job instance.

Raises:
TypeError:
If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.QueryJobConfig`
class.
"""
job_id = _make_job_id(job_id, job_id_prefix)

Expand All @@ -2060,6 +2110,9 @@ def query(

if self._default_query_job_config:
if job_config:
_verify_job_config_type(
job_config, google.cloud.bigquery.job.QueryJobConfig
)
# anything that's not defined on the incoming
# that is in the default,
# should be filled in with the default
Expand All @@ -2068,6 +2121,10 @@ def query(
self._default_query_job_config
)
else:
_verify_job_config_type(
self._default_query_job_config,
google.cloud.bigquery.job.QueryJobConfig,
)
job_config = self._default_query_job_config

job_ref = job._JobReference(job_id, project=project, location=location)
Expand Down
Loading