From 2a58eae79f2dc37031d02a8eb1eca11c2d3e8bee Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Mon, 5 Nov 2018 14:10:11 -0800 Subject: [PATCH 1/4] add properties to job config constructors --- bigquery/google/cloud/bigquery/job.py | 22 +++++++++------- bigquery/tests/unit/test_job.py | 38 +++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/bigquery/google/cloud/bigquery/job.py b/bigquery/google/cloud/bigquery/job.py index 053de18fb827..dd8614539a22 100644 --- a/bigquery/google/cloud/bigquery/job.py +++ b/bigquery/google/cloud/bigquery/job.py @@ -719,9 +719,11 @@ class _JobConfig(object): job_type (str): The key to use for the job configuration. """ - def __init__(self, job_type): + def __init__(self, job_type, properties={}): self._job_type = job_type self._properties = {job_type: {}} + for prop, val in properties.items(): + setattr(self, prop, val) @property def labels(self): @@ -793,7 +795,7 @@ def _set_sub_prop(self, key, value): _helpers._set_sub_prop(self._properties, [self._job_type, key], value) def _del_sub_prop(self, key): - """Reove ``key`` from the ``self._properties[self._job_type]`` dict. + """Remove ``key`` from the ``self._properties[self._job_type]`` dict. Most job properties are inside the dictionary related to the job type (e.g. 'copy', 'extract', 'load', 'query'). Use this method to clear @@ -878,8 +880,8 @@ class LoadJobConfig(_JobConfig): server defaults. """ - def __init__(self): - super(LoadJobConfig, self).__init__('load') + def __init__(self, **properties): + super(LoadJobConfig, self).__init__('load', properties) @property def allow_jagged_rows(self): @@ -1473,8 +1475,8 @@ class CopyJobConfig(_JobConfig): server defaults. """ - def __init__(self): - super(CopyJobConfig, self).__init__('copy') + def __init__(self, **properties): + super(CopyJobConfig, self).__init__('copy', properties) @property def create_disposition(self): @@ -1666,8 +1668,8 @@ class ExtractJobConfig(_JobConfig): server defaults. """ - def __init__(self): - super(ExtractJobConfig, self).__init__('extract') + def __init__(self, **properties): + super(ExtractJobConfig, self).__init__('extract', properties) @property def compression(self): @@ -1910,8 +1912,8 @@ class QueryJobConfig(_JobConfig): server defaults. """ - def __init__(self): - super(QueryJobConfig, self).__init__('query') + def __init__(self, **properties): + super(QueryJobConfig, self).__init__('query', properties) @property def destination_encryption_configuration(self): diff --git a/bigquery/tests/unit/test_job.py b/bigquery/tests/unit/test_job.py index 88deb068f871..edddebc05e31 100644 --- a/bigquery/tests/unit/test_job.py +++ b/bigquery/tests/unit/test_job.py @@ -1152,6 +1152,13 @@ def _get_target_class(): from google.cloud.bigquery.job import LoadJobConfig return LoadJobConfig + def test_ctor_w_properties(self): + config = self._get_target_class()( + allow_jagged_rows=True, allow_quoted_newlines=True) + + self.assertTrue(config.allow_jagged_rows) + self.assertTrue(config.allow_quoted_newlines) + def test_allow_jagged_rows_missing(self): config = self._get_target_class()() self.assertIsNone(config.allow_jagged_rows) @@ -2482,6 +2489,20 @@ def _get_target_class(): from google.cloud.bigquery.job import CopyJobConfig return CopyJobConfig + def test_ctor_w_properties(self): + from google.cloud.bigquery.job import CreateDisposition + from google.cloud.bigquery.job import WriteDisposition + + create_disposition = CreateDisposition.CREATE_NEVER + write_disposition = WriteDisposition.WRITE_TRUNCATE + config = self._get_target_class()( + create_disposition=create_disposition, + write_disposition=write_disposition + ) + + self.assertEqual(config.create_disposition, create_disposition) + self.assertEqual(config.write_disposition, write_disposition) + def test_to_api_repr_with_encryption(self): from google.cloud.bigquery.table import EncryptionConfiguration @@ -2916,6 +2937,13 @@ def _get_target_class(): from google.cloud.bigquery.job import ExtractJobConfig return ExtractJobConfig + def test_ctor_w_properties(self): + config = self._get_target_class()( + field_delimiter='\t', print_header=True) + + self.assertEqual(config.field_delimiter, '\t') + self.assertTrue(config.print_header) + def test_to_api_repr(self): from google.cloud.bigquery import job config = self._make_one() @@ -3299,6 +3327,13 @@ def test_ctor_w_none(self): self.assertIsNone(config.default_dataset) self.assertIsNone(config.destination) + def test_ctor_w_properties(self): + config = self._get_target_class()( + use_query_cache=False, use_legacy_sql=True) + + self.assertFalse(config.use_query_cache) + self.assertTrue(config.use_legacy_sql) + def test_time_partitioning(self): from google.cloud.bigquery import table @@ -3637,8 +3672,7 @@ def test_ctor_w_query_parameters(self): query_parameters = [ScalarQueryParameter("foo", 'INT64', 123)] client = _make_client(project=self.PROJECT) - config = QueryJobConfig() - config.query_parameters = query_parameters + config = QueryJobConfig(query_parameters=query_parameters) job = self._make_one( self.JOB_ID, self.QUERY, client, job_config=config) self.assertEqual(job.query_parameters, query_parameters) From 2fb531763a13fc5e9e3d7a687cb23776f45b342f Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Mon, 5 Nov 2018 16:27:06 -0800 Subject: [PATCH 2/4] update properties arg --- bigquery/google/cloud/bigquery/job.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bigquery/google/cloud/bigquery/job.py b/bigquery/google/cloud/bigquery/job.py index dd8614539a22..8abf4b752050 100644 --- a/bigquery/google/cloud/bigquery/job.py +++ b/bigquery/google/cloud/bigquery/job.py @@ -719,7 +719,7 @@ class _JobConfig(object): job_type (str): The key to use for the job configuration. """ - def __init__(self, job_type, properties={}): + def __init__(self, job_type, **properties): self._job_type = job_type self._properties = {job_type: {}} for prop, val in properties.items(): @@ -881,7 +881,7 @@ class LoadJobConfig(_JobConfig): """ def __init__(self, **properties): - super(LoadJobConfig, self).__init__('load', properties) + super(LoadJobConfig, self).__init__('load', **properties) @property def allow_jagged_rows(self): @@ -1476,7 +1476,7 @@ class CopyJobConfig(_JobConfig): """ def __init__(self, **properties): - super(CopyJobConfig, self).__init__('copy', properties) + super(CopyJobConfig, self).__init__('copy', **properties) @property def create_disposition(self): @@ -1669,7 +1669,7 @@ class ExtractJobConfig(_JobConfig): """ def __init__(self, **properties): - super(ExtractJobConfig, self).__init__('extract', properties) + super(ExtractJobConfig, self).__init__('extract', **properties) @property def compression(self): @@ -1913,7 +1913,7 @@ class QueryJobConfig(_JobConfig): """ def __init__(self, **properties): - super(QueryJobConfig, self).__init__('query', properties) + super(QueryJobConfig, self).__init__('query', **properties) @property def destination_encryption_configuration(self): From 4a5fd0ea6cab269d2fbbe99d8240e1e0a76f78b2 Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Mon, 5 Nov 2018 17:24:48 -0800 Subject: [PATCH 3/4] update **properties to **kwargs --- bigquery/google/cloud/bigquery/job.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/bigquery/google/cloud/bigquery/job.py b/bigquery/google/cloud/bigquery/job.py index 8abf4b752050..7664f323a828 100644 --- a/bigquery/google/cloud/bigquery/job.py +++ b/bigquery/google/cloud/bigquery/job.py @@ -719,10 +719,10 @@ class _JobConfig(object): job_type (str): The key to use for the job configuration. """ - def __init__(self, job_type, **properties): + def __init__(self, job_type, **kwargs): self._job_type = job_type self._properties = {job_type: {}} - for prop, val in properties.items(): + for prop, val in kwargs.items(): setattr(self, prop, val) @property @@ -880,8 +880,8 @@ class LoadJobConfig(_JobConfig): server defaults. """ - def __init__(self, **properties): - super(LoadJobConfig, self).__init__('load', **properties) + def __init__(self, **kwargs): + super(LoadJobConfig, self).__init__('load', **kwargs) @property def allow_jagged_rows(self): @@ -1475,8 +1475,8 @@ class CopyJobConfig(_JobConfig): server defaults. """ - def __init__(self, **properties): - super(CopyJobConfig, self).__init__('copy', **properties) + def __init__(self, **kwargs): + super(CopyJobConfig, self).__init__('copy', **kwargs) @property def create_disposition(self): @@ -1668,8 +1668,8 @@ class ExtractJobConfig(_JobConfig): server defaults. """ - def __init__(self, **properties): - super(ExtractJobConfig, self).__init__('extract', **properties) + def __init__(self, **kwargs): + super(ExtractJobConfig, self).__init__('extract', **kwargs) @property def compression(self): @@ -1912,8 +1912,8 @@ class QueryJobConfig(_JobConfig): server defaults. """ - def __init__(self, **properties): - super(QueryJobConfig, self).__init__('query', **properties) + def __init__(self, **kwargs): + super(QueryJobConfig, self).__init__('query', **kwargs) @property def destination_encryption_configuration(self): From e6221ceacfea52d1cee826e81c3592dbca2c152b Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Mon, 5 Nov 2018 17:27:44 -0800 Subject: [PATCH 4/4] fix docstrings --- bigquery/google/cloud/bigquery/job.py | 46 +++++++++++++++------------ 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/bigquery/google/cloud/bigquery/job.py b/bigquery/google/cloud/bigquery/job.py index 7664f323a828..5eec77de7e3a 100644 --- a/bigquery/google/cloud/bigquery/job.py +++ b/bigquery/google/cloud/bigquery/job.py @@ -764,7 +764,7 @@ def _get_sub_prop(self, key, default=None): ``self._properties[self._job_type]`` dictionary. default (object): (Optional) Default value to return if the key is not found. - Defaults to ``None``. + Defaults to :data:`None`. Returns: object: The value if present or the default. @@ -876,8 +876,9 @@ def from_api_repr(cls, resource): class LoadJobConfig(_JobConfig): """Configuration options for load jobs. - All properties in this class are optional. Values which are ``None`` -> - server defaults. + All properties in this class are optional. Values which are :data:`None` -> + server defaults. Set properties on the constructed configuration by using + the property name as the name of a keyword argument. """ def __init__(self, **kwargs): @@ -969,7 +970,7 @@ def destination_encryption_configuration(self): """google.cloud.bigquery.table.EncryptionConfiguration: Custom encryption configuration for the destination table. - Custom encryption configuration (e.g., Cloud KMS keys) or ``None`` + Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None` if using default encryption. See @@ -1340,7 +1341,7 @@ def destination_encryption_configuration(self): encryption configuration for the destination table. Custom encryption configuration (e.g., Cloud KMS keys) - or ``None`` if using default encryption. + or :data:`None` if using default encryption. See :attr:`google.cloud.bigquery.job.LoadJobConfig.destination_encryption_configuration`. @@ -1471,8 +1472,9 @@ def from_api_repr(cls, resource, client): class CopyJobConfig(_JobConfig): """Configuration options for copy jobs. - All properties in this class are optional. Values which are ``None`` -> - server defaults. + All properties in this class are optional. Values which are :data:`None` -> + server defaults. Set properties on the constructed configuration by using + the property name as the name of a keyword argument. """ def __init__(self, **kwargs): @@ -1511,7 +1513,7 @@ def destination_encryption_configuration(self): """google.cloud.bigquery.table.EncryptionConfiguration: Custom encryption configuration for the destination table. - Custom encryption configuration (e.g., Cloud KMS keys) or ``None`` + Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None` if using default encryption. See @@ -1581,7 +1583,7 @@ def destination_encryption_configuration(self): """google.cloud.bigquery.table.EncryptionConfiguration: Custom encryption configuration for the destination table. - Custom encryption configuration (e.g., Cloud KMS keys) or ``None`` + Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None` if using default encryption. See @@ -1664,8 +1666,9 @@ def from_api_repr(cls, resource, client): class ExtractJobConfig(_JobConfig): """Configuration options for extract jobs. - All properties in this class are optional. Values which are ``None`` -> - server defaults. + All properties in this class are optional. Values which are :data:`None` -> + server defaults. Set properties on the constructed configuration by using + the property name as the name of a keyword argument. """ def __init__(self, **kwargs): @@ -1908,8 +1911,9 @@ def _to_api_repr_table_defs(value): class QueryJobConfig(_JobConfig): """Configuration options for query jobs. - All properties in this class are optional. Values which are ``None`` -> - server defaults. + All properties in this class are optional. Values which are :data:`None` -> + server defaults. Set properties on the constructed configuration by using + the property name as the name of a keyword argument. """ def __init__(self, **kwargs): @@ -1920,7 +1924,7 @@ def destination_encryption_configuration(self): """google.cloud.bigquery.table.EncryptionConfiguration: Custom encryption configuration for the destination table. - Custom encryption configuration (e.g., Cloud KMS keys) or ``None`` + Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None` if using default encryption. See @@ -1968,7 +1972,8 @@ def create_disposition(self, value): @property def default_dataset(self): """google.cloud.bigquery.dataset.DatasetReference: the default dataset - to use for unqualified table names in the query or ``None`` if not set. + to use for unqualified table names in the query or :data:`None` if not + set. See https://g.co/cloud/bigquery/docs/reference/v2/jobs#configuration.query.defaultDataset @@ -1988,7 +1993,7 @@ def default_dataset(self, value): @property def destination(self): """google.cloud.bigquery.table.TableReference: table where results are - written or ``None`` if not set. + written or :data:`None` if not set. See https://g.co/cloud/bigquery/docs/reference/rest/v2/jobs#configuration.query.destinationTable @@ -2007,7 +2012,8 @@ def destination(self, value): @property def dry_run(self): - """bool: ``True`` if this query should be a dry run to estimate costs. + """bool: :data:`True` if this query should be a dry run to estimate + costs. See https://g.co/cloud/bigquery/docs/reference/v2/jobs#configuration.dryRun @@ -2047,7 +2053,7 @@ def maximum_billing_tier(self, value): @property def maximum_bytes_billed(self): - """int: Maximum bytes to be billed for this job or ``None`` if not set. + """int: Maximum bytes to be billed for this job or :data:`None` if not set. See https://g.co/cloud/bigquery/docs/reference/rest/v2/jobs#configuration.query.maximumBytesBilled @@ -2149,7 +2155,7 @@ def write_disposition(self, value): @property def table_definitions(self): """Dict[str, google.cloud.bigquery.external_config.ExternalConfig]: - Definitions for external tables or ``None`` if not set. + Definitions for external tables or :data:`None` if not set. See https://g.co/cloud/bigquery/docs/reference/rest/v2/jobs#configuration.query.tableDefinitions @@ -2307,7 +2313,7 @@ def destination_encryption_configuration(self): """google.cloud.bigquery.table.EncryptionConfiguration: Custom encryption configuration for the destination table. - Custom encryption configuration (e.g., Cloud KMS keys) or ``None`` + Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None` if using default encryption. See