From bd260822f9452d982000cd88d79749aa4616eadf Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 2 Jul 2021 13:01:12 -0500 Subject: [PATCH 1/3] feat: add LoadJobConfig.projection_fields to select DATASTORE_BACKUP fields --- google/cloud/bigquery/job/load.py | 21 +++++++++++++++++++++ tests/unit/job/test_load_config.py | 11 +++++++++++ 2 files changed, 32 insertions(+) diff --git a/google/cloud/bigquery/job/load.py b/google/cloud/bigquery/job/load.py index 41d38dd74..5e8e3dab2 100644 --- a/google/cloud/bigquery/job/load.py +++ b/google/cloud/bigquery/job/load.py @@ -277,6 +277,27 @@ def null_marker(self): def null_marker(self, value): self._set_sub_prop("nullMarker", value) + @property + def projection_fields(self): + """Optional[List[str]]: If + :attr:`google.cloud.bigquery.job.LoadJobConfig.source_format` is set to + "DATASTORE_BACKUP", indicates which entity properties to load into + BigQuery from a Cloud Datastore backup. + + Property names are case sensitive and must be top-level properties. If + no properties are specified, BigQuery loads all properties. If any + named property isn't found in the Cloud Datastore backup, an invalid + error is returned in the job result. + + See: + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.projection_fields + """ + return self._get_sub_prop("projectionFields") + + @projection_fields.setter + def projection_fields(self, value): + self._set_sub_prop("projectionFields", value) + @property def quote_character(self): """Optional[str]: Character used to quote data sections (CSV only). diff --git a/tests/unit/job/test_load_config.py b/tests/unit/job/test_load_config.py index eafe7e046..2e7ea75c1 100644 --- a/tests/unit/job/test_load_config.py +++ b/tests/unit/job/test_load_config.py @@ -385,6 +385,17 @@ def test_null_marker_setter(self): config.null_marker = null_marker self.assertEqual(config._properties["load"]["nullMarker"], null_marker) + def test_projection_fields_miss(self): + config = self._get_target_class()() + self.assertIsNone(config.projection_fields) + + def test_projection_fields_hit(self): + config = self._get_target_class()() + fields = ["email", "postal_code"] + config.projection_fields = fields + self.assertEqual(config._properties["load"]["projectionFields"], fields) + self.assertEqual(config.projection_fields, fields) + def test_quote_character_missing(self): config = self._get_target_class()() self.assertIsNone(config.quote_character) From 354c42cd4d70b4fe6934eef27b7420e8af8e61c0 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 9 Jul 2021 13:52:46 -0500 Subject: [PATCH 2/3] add type annotations --- google/cloud/bigquery/job/load.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/google/cloud/bigquery/job/load.py b/google/cloud/bigquery/job/load.py index 5e8e3dab2..bedc33f32 100644 --- a/google/cloud/bigquery/job/load.py +++ b/google/cloud/bigquery/job/load.py @@ -14,6 +14,8 @@ """Classes for load jobs.""" +from typing import Optional, List + from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration from google.cloud.bigquery.external_config import HivePartitioningOptions from google.cloud.bigquery.format_options import ParquetOptions @@ -23,7 +25,6 @@ from google.cloud.bigquery.table import RangePartitioning from google.cloud.bigquery.table import TableReference from google.cloud.bigquery.table import TimePartitioning - from google.cloud.bigquery.job.base import _AsyncJob from google.cloud.bigquery.job.base import _JobConfig from google.cloud.bigquery.job.base import _JobReference @@ -278,7 +279,7 @@ def null_marker(self, value): self._set_sub_prop("nullMarker", value) @property - def projection_fields(self): + def projection_fields(self) -> Optional[List[str]]: """Optional[List[str]]: If :attr:`google.cloud.bigquery.job.LoadJobConfig.source_format` is set to "DATASTORE_BACKUP", indicates which entity properties to load into From 7c107a8a81e8312ff3389bc0efce94588ce9df98 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 14 Jul 2021 15:24:20 -0500 Subject: [PATCH 3/3] annotate setter too --- google/cloud/bigquery/job/load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/bigquery/job/load.py b/google/cloud/bigquery/job/load.py index 64b5ca231..f1b045412 100644 --- a/google/cloud/bigquery/job/load.py +++ b/google/cloud/bigquery/job/load.py @@ -317,7 +317,7 @@ def projection_fields(self) -> Optional[List[str]]: return self._get_sub_prop("projectionFields") @projection_fields.setter - def projection_fields(self, value): + def projection_fields(self, value: Optional[List[str]]): self._set_sub_prop("projectionFields", value) @property