Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

BigQuery: Add properties to job config constructors #6397

Merged
merged 4 commits into from
Nov 6, 2018
Merged

BigQuery: Add properties to job config constructors #6397

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2a58eae
add properties to job config constructors
alixhami Nov 5, 2018
2fb5317
update properties arg
alixhami Nov 6, 2018
4a5fd0e
update **properties to **kwargs
alixhami Nov 6, 2018
e6221ce
fix docstrings
alixhami Nov 6, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 38 additions & 30 deletions bigquery/google/cloud/bigquery/job.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -719,9 +719,11 @@ class _JobConfig(object):
job_type (str): The key to use for the job configuration.
"""

def __init__(self, job_type):
def __init__(self, job_type, **kwargs):
self._job_type = job_type
self._properties = {job_type: {}}
for prop, val in kwargs.items():
setattr(self, prop, val)

@property
def labels(self):
Expand Down Expand Up @@ -762,7 +764,7 @@ def _get_sub_prop(self, key, default=None):
``self._properties[self._job_type]`` dictionary.
default (object):
(Optional) Default value to return if the key is not found.
Defaults to ``None``.
Defaults to :data:`None`.

Returns:
object: The value if present or the default.
Expand Down Expand Up @@ -793,7 +795,7 @@ def _set_sub_prop(self, key, value):
_helpers._set_sub_prop(self._properties, [self._job_type, key], value)

def _del_sub_prop(self, key):
"""Reove ``key`` from the ``self._properties[self._job_type]`` dict.
"""Remove ``key`` from the ``self._properties[self._job_type]`` dict.

Most job properties are inside the dictionary related to the job type
(e.g. 'copy', 'extract', 'load', 'query'). Use this method to clear
Expand Down Expand Up @@ -874,12 +876,13 @@ def from_api_repr(cls, resource):
class LoadJobConfig(_JobConfig):
"""Configuration options for load jobs.

All properties in this class are optional. Values which are ``None`` ->
server defaults.
All properties in this class are optional. Values which are :data:`None` ->
server defaults. Set properties on the constructed configuration by using
the property name as the name of a keyword argument.
"""

def __init__(self):
super(LoadJobConfig, self).__init__('load')
def __init__(self, **kwargs):
super(LoadJobConfig, self).__init__('load', **kwargs)

@property
def allow_jagged_rows(self):
Expand Down Expand Up @@ -967,7 +970,7 @@ def destination_encryption_configuration(self):
"""google.cloud.bigquery.table.EncryptionConfiguration: Custom
encryption configuration for the destination table.

Custom encryption configuration (e.g., Cloud KMS keys) or ``None``
Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None`
if using default encryption.

See
Expand Down Expand Up @@ -1338,7 +1341,7 @@ def destination_encryption_configuration(self):
encryption configuration for the destination table.

Custom encryption configuration (e.g., Cloud KMS keys)
or ``None`` if using default encryption.
or :data:`None` if using default encryption.

See
:attr:`google.cloud.bigquery.job.LoadJobConfig.destination_encryption_configuration`.
Expand Down Expand Up @@ -1469,12 +1472,13 @@ def from_api_repr(cls, resource, client):
class CopyJobConfig(_JobConfig):
"""Configuration options for copy jobs.

All properties in this class are optional. Values which are ``None`` ->
server defaults.
All properties in this class are optional. Values which are :data:`None` ->
server defaults. Set properties on the constructed configuration by using
the property name as the name of a keyword argument.
"""

def __init__(self):
super(CopyJobConfig, self).__init__('copy')
def __init__(self, **kwargs):
super(CopyJobConfig, self).__init__('copy', **kwargs)

@property
def create_disposition(self):
Expand Down Expand Up @@ -1509,7 +1513,7 @@ def destination_encryption_configuration(self):
"""google.cloud.bigquery.table.EncryptionConfiguration: Custom
encryption configuration for the destination table.

Custom encryption configuration (e.g., Cloud KMS keys) or ``None``
Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None`
if using default encryption.

See
Expand Down Expand Up @@ -1579,7 +1583,7 @@ def destination_encryption_configuration(self):
"""google.cloud.bigquery.table.EncryptionConfiguration: Custom
encryption configuration for the destination table.

Custom encryption configuration (e.g., Cloud KMS keys) or ``None``
Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None`
if using default encryption.

See
Expand Down Expand Up @@ -1662,12 +1666,13 @@ def from_api_repr(cls, resource, client):
class ExtractJobConfig(_JobConfig):
"""Configuration options for extract jobs.

All properties in this class are optional. Values which are ``None`` ->
server defaults.
All properties in this class are optional. Values which are :data:`None` ->
server defaults. Set properties on the constructed configuration by using
the property name as the name of a keyword argument.
"""

def __init__(self):
super(ExtractJobConfig, self).__init__('extract')
def __init__(self, **kwargs):
super(ExtractJobConfig, self).__init__('extract', **kwargs)

@property
def compression(self):
Expand Down Expand Up @@ -1906,19 +1911,20 @@ def _to_api_repr_table_defs(value):
class QueryJobConfig(_JobConfig):
"""Configuration options for query jobs.

All properties in this class are optional. Values which are ``None`` ->
server defaults.
All properties in this class are optional. Values which are :data:`None` ->
server defaults. Set properties on the constructed configuration by using
the property name as the name of a keyword argument.
"""

def __init__(self):
super(QueryJobConfig, self).__init__('query')
def __init__(self, **kwargs):
super(QueryJobConfig, self).__init__('query', **kwargs)

@property
def destination_encryption_configuration(self):
"""google.cloud.bigquery.table.EncryptionConfiguration: Custom
encryption configuration for the destination table.

Custom encryption configuration (e.g., Cloud KMS keys) or ``None``
Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None`
if using default encryption.

See
Expand Down Expand Up @@ -1966,7 +1972,8 @@ def create_disposition(self, value):
@property
def default_dataset(self):
"""google.cloud.bigquery.dataset.DatasetReference: the default dataset
to use for unqualified table names in the query or ``None`` if not set.
to use for unqualified table names in the query or :data:`None` if not
set.

See
https://g.co/cloud/bigquery/docs/reference/v2/jobs#configuration.query.defaultDataset
Expand All @@ -1986,7 +1993,7 @@ def default_dataset(self, value):
@property
def destination(self):
"""google.cloud.bigquery.table.TableReference: table where results are
written or ``None`` if not set.
written or :data:`None` if not set.

See
https://g.co/cloud/bigquery/docs/reference/rest/v2/jobs#configuration.query.destinationTable
Expand All @@ -2005,7 +2012,8 @@ def destination(self, value):

@property
def dry_run(self):
"""bool: ``True`` if this query should be a dry run to estimate costs.
"""bool: :data:`True` if this query should be a dry run to estimate
costs.

See
https://g.co/cloud/bigquery/docs/reference/v2/jobs#configuration.dryRun
Expand Down Expand Up @@ -2045,7 +2053,7 @@ def maximum_billing_tier(self, value):

@property
def maximum_bytes_billed(self):
"""int: Maximum bytes to be billed for this job or ``None`` if not set.
"""int: Maximum bytes to be billed for this job or :data:`None` if not set.

See
https://g.co/cloud/bigquery/docs/reference/rest/v2/jobs#configuration.query.maximumBytesBilled
Expand Down Expand Up @@ -2147,7 +2155,7 @@ def write_disposition(self, value):
@property
def table_definitions(self):
"""Dict[str, google.cloud.bigquery.external_config.ExternalConfig]:
Definitions for external tables or ``None`` if not set.
Definitions for external tables or :data:`None` if not set.

See
https://g.co/cloud/bigquery/docs/reference/rest/v2/jobs#configuration.query.tableDefinitions
Expand Down Expand Up @@ -2305,7 +2313,7 @@ def destination_encryption_configuration(self):
"""google.cloud.bigquery.table.EncryptionConfiguration: Custom
encryption configuration for the destination table.

Custom encryption configuration (e.g., Cloud KMS keys) or ``None``
Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None`
if using default encryption.

See
Expand Down
38 changes: 36 additions & 2 deletions bigquery/tests/unit/test_job.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1152,6 +1152,13 @@ def _get_target_class():
from google.cloud.bigquery.job import LoadJobConfig
return LoadJobConfig

def test_ctor_w_properties(self):
config = self._get_target_class()(
allow_jagged_rows=True, allow_quoted_newlines=True)

self.assertTrue(config.allow_jagged_rows)
self.assertTrue(config.allow_quoted_newlines)

def test_allow_jagged_rows_missing(self):
config = self._get_target_class()()
self.assertIsNone(config.allow_jagged_rows)
Expand Down Expand Up @@ -2482,6 +2489,20 @@ def _get_target_class():
from google.cloud.bigquery.job import CopyJobConfig
return CopyJobConfig

def test_ctor_w_properties(self):
from google.cloud.bigquery.job import CreateDisposition
from google.cloud.bigquery.job import WriteDisposition

create_disposition = CreateDisposition.CREATE_NEVER
write_disposition = WriteDisposition.WRITE_TRUNCATE
config = self._get_target_class()(
create_disposition=create_disposition,
write_disposition=write_disposition
)

self.assertEqual(config.create_disposition, create_disposition)
self.assertEqual(config.write_disposition, write_disposition)

def test_to_api_repr_with_encryption(self):
from google.cloud.bigquery.table import EncryptionConfiguration

Expand Down Expand Up @@ -2916,6 +2937,13 @@ def _get_target_class():
from google.cloud.bigquery.job import ExtractJobConfig
return ExtractJobConfig

def test_ctor_w_properties(self):
config = self._get_target_class()(
field_delimiter='\t', print_header=True)

self.assertEqual(config.field_delimiter, '\t')
self.assertTrue(config.print_header)

def test_to_api_repr(self):
from google.cloud.bigquery import job
config = self._make_one()
Expand Down Expand Up @@ -3299,6 +3327,13 @@ def test_ctor_w_none(self):
self.assertIsNone(config.default_dataset)
self.assertIsNone(config.destination)

def test_ctor_w_properties(self):
config = self._get_target_class()(
use_query_cache=False, use_legacy_sql=True)

self.assertFalse(config.use_query_cache)
self.assertTrue(config.use_legacy_sql)

def test_time_partitioning(self):
from google.cloud.bigquery import table

Expand Down Expand Up @@ -3637,8 +3672,7 @@ def test_ctor_w_query_parameters(self):

query_parameters = [ScalarQueryParameter("foo", 'INT64', 123)]
client = _make_client(project=self.PROJECT)
config = QueryJobConfig()
config.query_parameters = query_parameters
config = QueryJobConfig(query_parameters=query_parameters)
job = self._make_one(
self.JOB_ID, self.QUERY, client, job_config=config)
self.assertEqual(job.query_parameters, query_parameters)
Expand Down