From 1c2cc400e24d89aac9763929400b5b15a9e98c7b Mon Sep 17 00:00:00 2001 From: Hemang Date: Fri, 18 Oct 2019 18:17:01 +0530 Subject: [PATCH 1/3] feat(bigquery): add ValueError if wrong job_config type is passed to client job methods --- bigquery/google/cloud/bigquery/client.py | 90 ++++++++++- bigquery/tests/unit/test_client.py | 190 ++++++++++++++++++++++- 2 files changed, 272 insertions(+), 8 deletions(-) diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index 1ad107ba8151..1d54d64c2cf7 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -1355,6 +1355,11 @@ def load_table_from_uri( Returns: google.cloud.bigquery.job.LoadJob: A new load job. + + Raises: + ValueError: + If ``job_config`` is not an instance of ``google.cloud.bigquery.job.LoadJobConfig`` + class. """ job_id = _make_job_id(job_id, job_id_prefix) @@ -1370,6 +1375,14 @@ def load_table_from_uri( source_uris = [source_uris] destination = _table_arg_to_table_ref(destination, default_project=self.project) + + if job_config: + if not isinstance(job_config, google.cloud.bigquery.job.LoadJobConfig): + raise ValueError( + "Expected an instance of LoadJobConfig class for the job_config parameter, but received job_config = %s" + % (job_config,) + ) + load_job = job.LoadJob(job_ref, source_uris, destination, self, job_config) load_job._begin(retry=retry) @@ -1436,6 +1449,9 @@ def load_table_from_file( If ``size`` is not passed in and can not be determined, or if the ``file_obj`` can be detected to be a file opened in text mode. + + ValueError: + If ``job_config`` is not an instance of ``google.cloud.bigquery.job.LoadJobConfig`` class. """ job_id = _make_job_id(job_id, job_id_prefix) @@ -1447,6 +1463,12 @@ def load_table_from_file( destination = _table_arg_to_table_ref(destination, default_project=self.project) job_ref = job._JobReference(job_id, project=project, location=location) + if job_config: + if not isinstance(job_config, google.cloud.bigquery.job.LoadJobConfig): + raise ValueError( + "Expected an instance of LoadJobConfig class for the job_config parameter, but received job_config = %s" + % (job_config,) + ) load_job = job.LoadJob(job_ref, None, destination, self, job_config) job_resource = load_job.to_api_repr() @@ -1545,16 +1567,25 @@ def load_table_from_dataframe( If a usable parquet engine cannot be found. This method requires :mod:`pyarrow` or :mod:`fastparquet` to be installed. + ValueError: + If ``job_config`` is not an instance of ``google.cloud.bigquery.job.LoadJobConfig`` class. """ job_id = _make_job_id(job_id, job_id_prefix) - if job_config is None: - job_config = job.LoadJobConfig() - else: + if job_config: + if not isinstance(job_config, google.cloud.bigquery.job.LoadJobConfig): + raise ValueError( + "Expected an instance of LoadJobConfig class for the job_config parameter, but received job_config = %s" + % (job_config,) + ) # Make a copy so that the job config isn't modified in-place. job_config_properties = copy.deepcopy(job_config._properties) job_config = job.LoadJobConfig() job_config._properties = job_config_properties + + else: + job_config = job.LoadJobConfig() + job_config.source_format = job.SourceFormat.PARQUET if location is None: @@ -1700,14 +1731,24 @@ def load_table_from_json( Returns: google.cloud.bigquery.job.LoadJob: A new load job. + + Raises: + ValueError: + If ``job_config`` is not an instance of ``google.cloud.bigquery.job.LoadJobConfig`` class. """ job_id = _make_job_id(job_id, job_id_prefix) - if job_config is None: - job_config = job.LoadJobConfig() - else: + if job_config: + if not isinstance(job_config, google.cloud.bigquery.job.LoadJobConfig): + raise ValueError( + "Expected an instance of LoadJobConfig class for the job_config parameter, but received job_config = %s" + % (job_config,) + ) # Make a copy so that the job config isn't modified in-place. job_config = copy.deepcopy(job_config) + else: + job_config = job.LoadJobConfig() + job_config.source_format = job.SourceFormat.NEWLINE_DELIMITED_JSON if job_config.schema is None: @@ -1900,6 +1941,10 @@ def copy_table( Returns: google.cloud.bigquery.job.CopyJob: A new copy job instance. + + Raises: + ValueError: + If ``job_config`` is not an instance of ``google.cloud.bigquery.job.CopyJobConfig`` class. """ job_id = _make_job_id(job_id, job_id_prefix) @@ -1928,6 +1973,12 @@ def copy_table( destination = _table_arg_to_table_ref(destination, default_project=self.project) + if job_config: + if not isinstance(job_config, google.cloud.bigquery.job.CopyJobConfig): + raise ValueError( + "Expected an instance of CopyJobConfig class for the job_config parameter, but received job_config = %s" + % (job_config,) + ) copy_job = job.CopyJob( job_ref, sources, destination, client=self, job_config=job_config ) @@ -1985,6 +2036,10 @@ def extract_table( Returns: google.cloud.bigquery.job.ExtractJob: A new extract job instance. + + Raises: + ValueError: + If ``job_config`` is not an instance of ``google.cloud.bigquery.job.ExtractJobConfig`` class. """ job_id = _make_job_id(job_id, job_id_prefix) @@ -2000,6 +2055,12 @@ def extract_table( if isinstance(destination_uris, six.string_types): destination_uris = [destination_uris] + if job_config: + if not isinstance(job_config, google.cloud.bigquery.job.ExtractJobConfig): + raise ValueError( + "Expected an instance of ExtractJobConfig class for the job_config parameter, but received job_config = %s" + % (job_config,) + ) extract_job = job.ExtractJob( job_ref, source, destination_uris, client=self, job_config=job_config ) @@ -2049,6 +2110,10 @@ def query( Returns: google.cloud.bigquery.job.QueryJob: A new query job instance. + + Raises: + ValueError: + If ``job_config`` is not an instance of ``google.cloud.bigquery.job.QueryJobConfig`` class. """ job_id = _make_job_id(job_id, job_id_prefix) @@ -2060,6 +2125,11 @@ def query( if self._default_query_job_config: if job_config: + if not isinstance(job_config, google.cloud.bigquery.job.QueryJobConfig): + raise ValueError( + "Expected an instance of QueryJobConfig class for the job_config parameter, but received job_config = %s" + % (job_config,) + ) # anything that's not defined on the incoming # that is in the default, # should be filled in with the default @@ -2068,6 +2138,14 @@ def query( self._default_query_job_config ) else: + if not isinstance( + self._default_query_job_config, + google.cloud.bigquery.job.QueryJobConfig, + ): + raise ValueError( + "Expected an instance of QueryJobConfig class for the job_config parameter, but received job_config = %s" + % (job_config,) + ) job_config = self._default_query_job_config job_ref = job._JobReference(job_id, project=project, location=location) diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index b8a367e17cb9..ac6fe19ad6c6 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -2973,7 +2973,7 @@ def test_list_jobs_w_parent_job_filter(self): conn.api_request.reset_mock() def test_load_table_from_uri(self): - from google.cloud.bigquery.job import LoadJob + from google.cloud.bigquery.job import LoadJob, LoadJobConfig JOB = "job_name" DESTINATION = "destination_table" @@ -2993,11 +2993,14 @@ def test_load_table_from_uri(self): } creds = _make_credentials() http = object() + job_config = LoadJobConfig() client = self._make_one(project=self.PROJECT, credentials=creds, _http=http) conn = client._connection = make_connection(RESOURCE) destination = client.dataset(self.DS_ID).table(DESTINATION) - job = client.load_table_from_uri(SOURCE_URI, destination, job_id=JOB) + job = client.load_table_from_uri( + SOURCE_URI, destination, job_id=JOB, job_config=job_config + ) # Check that load_table_from_uri actually starts the job. conn.api_request.assert_called_once_with( @@ -3005,6 +3008,7 @@ def test_load_table_from_uri(self): ) self.assertIsInstance(job, LoadJob) + self.assertIsInstance(job._configuration, LoadJobConfig) self.assertIs(job._client, client) self.assertEqual(job.job_id, JOB) self.assertEqual(list(job.source_uris), [SOURCE_URI]) @@ -3100,6 +3104,24 @@ def test_load_table_from_uri_w_client_location(self): method="POST", path="/projects/other-project/jobs", data=resource ) + def test_load_table_from_uri_w_invalid_job_config(self): + JOB = "job_name" + DESTINATION = "destination_table" + SOURCE_URI = "http://example.com/source.csv" + + creds = _make_credentials() + http = object() + job_config = object() + client = self._make_one(project=self.PROJECT, credentials=creds, _http=http) + destination = client.dataset(self.DS_ID).table(DESTINATION) + + with self.assertRaises(ValueError) as exc: + client.load_table_from_uri( + SOURCE_URI, destination, job_id=JOB, job_config=job_config + ) + + self.assertIn("Expected an instance of LoadJobConfig", exc.exception.args[0]) + @staticmethod def _mock_requests_response(status_code, headers, content=b""): return mock.Mock( @@ -3422,6 +3444,64 @@ def test_copy_table_w_source_strings(self): ).table("destination_table") self.assertEqual(job.destination, expected_destination) + def test_copy_table_w_invalid_job_config(self): + JOB = "job_name" + SOURCE = "source_table" + DESTINATION = "destination_table" + + creds = _make_credentials() + http = object() + client = self._make_one(project=self.PROJECT, credentials=creds, _http=http) + job_config = object() + dataset = client.dataset(self.DS_ID) + source = dataset.table(SOURCE) + destination = dataset.table(DESTINATION) + with self.assertRaises(ValueError) as exc: + client.copy_table(source, destination, job_id=JOB, job_config=job_config) + + self.assertIn("Expected an instance of CopyJobConfig", exc.exception.args[0]) + + def test_copy_table_w_valid_job_config(self): + from google.cloud.bigquery.job import CopyJobConfig + + JOB = "job_name" + SOURCE = "source_table" + DESTINATION = "destination_table" + RESOURCE = { + "jobReference": {"projectId": self.PROJECT, "jobId": JOB}, + "configuration": { + "copy": { + "sourceTables": [ + { + "projectId": self.PROJECT, + "datasetId": self.DS_ID, + "tableId": SOURCE, + } + ], + "destinationTable": { + "projectId": self.PROJECT, + "datasetId": self.DS_ID, + "tableId": DESTINATION, + }, + } + }, + } + creds = _make_credentials() + http = object() + client = self._make_one(project=self.PROJECT, credentials=creds, _http=http) + job_config = CopyJobConfig() + conn = client._connection = make_connection(RESOURCE) + dataset = client.dataset(self.DS_ID) + source = dataset.table(SOURCE) + destination = dataset.table(DESTINATION) + + job = client.copy_table(source, destination, job_id=JOB, job_config=job_config) + # Check that copy_table actually starts the job. + conn.api_request.assert_called_once_with( + method="POST", path="/projects/%s/jobs" % self.PROJECT, data=RESOURCE + ) + self.assertIsInstance(job._configuration, CopyJobConfig) + def test_extract_table(self): from google.cloud.bigquery.job import ExtractJob @@ -3462,6 +3542,22 @@ def test_extract_table(self): self.assertEqual(job.source, source) self.assertEqual(list(job.destination_uris), [DESTINATION]) + def test_extract_table_w_invalid_job_config(self): + JOB = "job_id" + SOURCE = "source_table" + DESTINATION = "gs://bucket_name/object_name" + + creds = _make_credentials() + http = object() + client = self._make_one(project=self.PROJECT, credentials=creds, _http=http) + dataset = client.dataset(self.DS_ID) + source = dataset.table(SOURCE) + job_config = object() + with self.assertRaises(ValueError) as exc: + client.extract_table(source, DESTINATION, job_id=JOB, job_config=job_config) + + self.assertIn("Expected an instance of ExtractJobConfig", exc.exception.args[0]) + def test_extract_table_w_explicit_project(self): job_id = "job_id" source_id = "source_table" @@ -3745,6 +3841,34 @@ def test_query_w_explicit_job_config(self): method="POST", path="/projects/PROJECT/jobs", data=resource ) + def test_query_w_invalid_job_config(self): + from google.cloud.bigquery import QueryJobConfig, DatasetReference + + job_id = "some-job-id" + query = "select count(*) from persons" + creds = _make_credentials() + http = object() + default_job_config = QueryJobConfig() + default_job_config.default_dataset = DatasetReference( + self.PROJECT, "some-dataset" + ) + default_job_config.maximum_bytes_billed = 1000 + + client = self._make_one( + project=self.PROJECT, + credentials=creds, + _http=http, + default_query_job_config=default_job_config, + ) + + job_config = object() + + with self.assertRaises(ValueError) as exc: + client.query( + query, job_id=job_id, location=self.LOCATION, job_config=job_config + ) + self.assertIn("Expected an instance of QueryJobConfig", exc.exception.args[0]) + def test_query_w_explicit_job_config_override(self): job_id = "some-job-id" query = "select count(*) from persons" @@ -3839,6 +3963,23 @@ def test_query_w_client_default_config_no_incoming(self): method="POST", path="/projects/PROJECT/jobs", data=resource ) + def test_query_w_invalid_default_job_config(self): + job_id = "some-job-id" + query = "select count(*) from persons" + creds = _make_credentials() + http = object() + default_job_config = object() + client = self._make_one( + project=self.PROJECT, + credentials=creds, + _http=http, + default_query_job_config=default_job_config, + ) + + with self.assertRaises(ValueError) as exc: + client.query(query, job_id=job_id, location=self.LOCATION) + self.assertIn("Expected an instance of QueryJobConfig", exc.exception.args[0]) + def test_query_w_client_location(self): job_id = "some-job-id" query = "select count(*) from persons" @@ -5419,6 +5560,17 @@ def test_load_table_from_file_bad_mode(self): with pytest.raises(ValueError): client.load_table_from_file(file_obj, self.TABLE_REF) + def test_load_table_from_file_w_invalid_job_config(self): + client = self._make_client() + gzip_file = self._make_gzip_file_obj(writable=True) + + with pytest.raises(ValueError) as exc: + client.load_table_from_file( + gzip_file, self.TABLE_REF, job_id="job_id", job_config=object() + ) + err_msg = str(exc.value) + assert "Expected an instance of LoadJobConfig" in err_msg + @unittest.skipIf(pandas is None, "Requires `pandas`") @unittest.skipIf(pyarrow is None, "Requires `pyarrow`") def test_load_table_from_dataframe(self): @@ -6118,6 +6270,22 @@ def test_load_table_from_dataframe_w_nulls(self): assert sent_config.schema == schema assert sent_config.source_format == job.SourceFormat.PARQUET + @unittest.skipIf(pandas is None, "Requires `pandas`") + def test_load_table_from_dataframe_w_invaild_job_config(self): + client = self._make_client() + + records = [{"float_column": 3.14, "struct_column": [{"foo": 1}, {"bar": -1}]}] + dataframe = pandas.DataFrame(data=records) + job_config = object() + + with pytest.raises(ValueError) as exc: + client.load_table_from_dataframe( + dataframe, self.TABLE_REF, job_config=job_config, location=self.LOCATION + ) + + err_msg = str(exc.value) + assert "Expected an instance of LoadJobConfig" in err_msg + def test_load_table_from_json_basic_use(self): from google.cloud.bigquery.client import _DEFAULT_NUM_RETRIES from google.cloud.bigquery import job @@ -6206,6 +6374,24 @@ def test_load_table_from_json_non_default_args(self): # all properties should have been cloned and sent to the backend assert sent_config._properties.get("load", {}).get("unknown_field") == "foobar" + def test_load_table_from_json_w_invalid_job_config(self): + client = self._make_client() + json_rows = [ + {"name": "One", "age": 11, "birthday": "2008-09-10", "adult": False}, + {"name": "Two", "age": 22, "birthday": "1997-08-09", "adult": True}, + ] + job_config = object() + with pytest.raises(ValueError) as exc: + client.load_table_from_json( + json_rows, + self.TABLE_REF, + job_config=job_config, + project="project-x", + location="EU", + ) + err_msg = str(exc.value) + assert "Expected an instance of LoadJobConfig" in err_msg + # Low-level tests @classmethod From 847556888cbd222285d3063195d68cabcdf8312b Mon Sep 17 00:00:00 2001 From: Hemang Date: Mon, 21 Oct 2019 11:42:02 +0530 Subject: [PATCH 2/3] feat(bigquery): cosmetic changes --- bigquery/google/cloud/bigquery/client.py | 40 ++++++++++++++---------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index 1d54d64c2cf7..56d890886dac 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -1379,8 +1379,9 @@ def load_table_from_uri( if job_config: if not isinstance(job_config, google.cloud.bigquery.job.LoadJobConfig): raise ValueError( - "Expected an instance of LoadJobConfig class for the job_config parameter, but received job_config = %s" - % (job_config,) + "Expected an instance of LoadJobConfig class for the job_config parameter, but received job_config = {}".format( + job_config + ) ) load_job = job.LoadJob(job_ref, source_uris, destination, self, job_config) @@ -1466,8 +1467,9 @@ def load_table_from_file( if job_config: if not isinstance(job_config, google.cloud.bigquery.job.LoadJobConfig): raise ValueError( - "Expected an instance of LoadJobConfig class for the job_config parameter, but received job_config = %s" - % (job_config,) + "Expected an instance of LoadJobConfig class for the job_config parameter, but received job_config {}".format( + job_config + ) ) load_job = job.LoadJob(job_ref, None, destination, self, job_config) job_resource = load_job.to_api_repr() @@ -1575,8 +1577,9 @@ def load_table_from_dataframe( if job_config: if not isinstance(job_config, google.cloud.bigquery.job.LoadJobConfig): raise ValueError( - "Expected an instance of LoadJobConfig class for the job_config parameter, but received job_config = %s" - % (job_config,) + "Expected an instance of LoadJobConfig class for the job_config parameter, but received job_config = {}".format( + job_config + ) ) # Make a copy so that the job config isn't modified in-place. job_config_properties = copy.deepcopy(job_config._properties) @@ -1741,8 +1744,9 @@ def load_table_from_json( if job_config: if not isinstance(job_config, google.cloud.bigquery.job.LoadJobConfig): raise ValueError( - "Expected an instance of LoadJobConfig class for the job_config parameter, but received job_config = %s" - % (job_config,) + "Expected an instance of LoadJobConfig class for the job_config parameter, but received job_config = {}".format( + job_config + ) ) # Make a copy so that the job config isn't modified in-place. job_config = copy.deepcopy(job_config) @@ -1976,8 +1980,9 @@ def copy_table( if job_config: if not isinstance(job_config, google.cloud.bigquery.job.CopyJobConfig): raise ValueError( - "Expected an instance of CopyJobConfig class for the job_config parameter, but received job_config = %s" - % (job_config,) + "Expected an instance of CopyJobConfig class for the job_config parameter, but received job_config = {}".format( + job_config + ) ) copy_job = job.CopyJob( job_ref, sources, destination, client=self, job_config=job_config @@ -2058,8 +2063,9 @@ def extract_table( if job_config: if not isinstance(job_config, google.cloud.bigquery.job.ExtractJobConfig): raise ValueError( - "Expected an instance of ExtractJobConfig class for the job_config parameter, but received job_config = %s" - % (job_config,) + "Expected an instance of ExtractJobConfig class for the job_config parameter, but received job_config = {}".format( + job_config + ) ) extract_job = job.ExtractJob( job_ref, source, destination_uris, client=self, job_config=job_config @@ -2127,8 +2133,9 @@ def query( if job_config: if not isinstance(job_config, google.cloud.bigquery.job.QueryJobConfig): raise ValueError( - "Expected an instance of QueryJobConfig class for the job_config parameter, but received job_config = %s" - % (job_config,) + "Expected an instance of QueryJobConfig class for the job_config parameter, but received job_config = {}".format( + job_config + ) ) # anything that's not defined on the incoming # that is in the default, @@ -2143,8 +2150,9 @@ def query( google.cloud.bigquery.job.QueryJobConfig, ): raise ValueError( - "Expected an instance of QueryJobConfig class for the job_config parameter, but received job_config = %s" - % (job_config,) + "Expected an instance of QueryJobConfig class for the job_config parameter, but received job_config = {}".format( + job_config + ) ) job_config = self._default_query_job_config From b6448783af5b124d52f3b4dc55a3e3fef9c60b79 Mon Sep 17 00:00:00 2001 From: Hemang Date: Mon, 21 Oct 2019 19:08:11 +0530 Subject: [PATCH 3/3] feat(bigquery): code refactor --- bigquery/google/cloud/bigquery/_helpers.py | 15 ++++ bigquery/google/cloud/bigquery/client.py | 97 ++++++++-------------- bigquery/tests/unit/test_client.py | 45 ++++++---- 3 files changed, 78 insertions(+), 79 deletions(-) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index eb5161c9fe71..bcb9d0696bc3 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -658,3 +658,18 @@ def _build_resource_from_properties(obj, filter_fields): partial[filter_field] = obj._properties[filter_field] return partial + + +def _verify_job_config_type(job_config, expected_type, param_name="job_config"): + if not isinstance(job_config, expected_type): + msg = ( + "Expected an instance of {expected_type} class for the {param_name} parameter, " + "but received {param_name} = {job_config}" + ) + raise TypeError( + msg.format( + expected_type=expected_type.__name__, + param_name=param_name, + job_config=job_config, + ) + ) diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index 56d890886dac..e7810dbbd66a 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -53,6 +53,7 @@ from google.cloud.bigquery._helpers import _record_field_to_json from google.cloud.bigquery._helpers import _str_or_none +from google.cloud.bigquery._helpers import _verify_job_config_type from google.cloud.bigquery._http import Connection from google.cloud.bigquery import _pandas_helpers from google.cloud.bigquery.dataset import Dataset @@ -1357,8 +1358,8 @@ def load_table_from_uri( google.cloud.bigquery.job.LoadJob: A new load job. Raises: - ValueError: - If ``job_config`` is not an instance of ``google.cloud.bigquery.job.LoadJobConfig`` + TypeError: + If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.LoadJobConfig` class. """ job_id = _make_job_id(job_id, job_id_prefix) @@ -1377,12 +1378,7 @@ def load_table_from_uri( destination = _table_arg_to_table_ref(destination, default_project=self.project) if job_config: - if not isinstance(job_config, google.cloud.bigquery.job.LoadJobConfig): - raise ValueError( - "Expected an instance of LoadJobConfig class for the job_config parameter, but received job_config = {}".format( - job_config - ) - ) + _verify_job_config_type(job_config, google.cloud.bigquery.job.LoadJobConfig) load_job = job.LoadJob(job_ref, source_uris, destination, self, job_config) load_job._begin(retry=retry) @@ -1451,8 +1447,9 @@ def load_table_from_file( the ``file_obj`` can be detected to be a file opened in text mode. - ValueError: - If ``job_config`` is not an instance of ``google.cloud.bigquery.job.LoadJobConfig`` class. + TypeError: + If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.LoadJobConfig` + class. """ job_id = _make_job_id(job_id, job_id_prefix) @@ -1465,12 +1462,7 @@ def load_table_from_file( destination = _table_arg_to_table_ref(destination, default_project=self.project) job_ref = job._JobReference(job_id, project=project, location=location) if job_config: - if not isinstance(job_config, google.cloud.bigquery.job.LoadJobConfig): - raise ValueError( - "Expected an instance of LoadJobConfig class for the job_config parameter, but received job_config {}".format( - job_config - ) - ) + _verify_job_config_type(job_config, google.cloud.bigquery.job.LoadJobConfig) load_job = job.LoadJob(job_ref, None, destination, self, job_config) job_resource = load_job.to_api_repr() @@ -1569,18 +1561,14 @@ def load_table_from_dataframe( If a usable parquet engine cannot be found. This method requires :mod:`pyarrow` or :mod:`fastparquet` to be installed. - ValueError: - If ``job_config`` is not an instance of ``google.cloud.bigquery.job.LoadJobConfig`` class. + TypeError: + If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.LoadJobConfig` + class. """ job_id = _make_job_id(job_id, job_id_prefix) if job_config: - if not isinstance(job_config, google.cloud.bigquery.job.LoadJobConfig): - raise ValueError( - "Expected an instance of LoadJobConfig class for the job_config parameter, but received job_config = {}".format( - job_config - ) - ) + _verify_job_config_type(job_config, google.cloud.bigquery.job.LoadJobConfig) # Make a copy so that the job config isn't modified in-place. job_config_properties = copy.deepcopy(job_config._properties) job_config = job.LoadJobConfig() @@ -1736,18 +1724,14 @@ def load_table_from_json( google.cloud.bigquery.job.LoadJob: A new load job. Raises: - ValueError: - If ``job_config`` is not an instance of ``google.cloud.bigquery.job.LoadJobConfig`` class. + TypeError: + If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.LoadJobConfig` + class. """ job_id = _make_job_id(job_id, job_id_prefix) if job_config: - if not isinstance(job_config, google.cloud.bigquery.job.LoadJobConfig): - raise ValueError( - "Expected an instance of LoadJobConfig class for the job_config parameter, but received job_config = {}".format( - job_config - ) - ) + _verify_job_config_type(job_config, google.cloud.bigquery.job.LoadJobConfig) # Make a copy so that the job config isn't modified in-place. job_config = copy.deepcopy(job_config) else: @@ -1947,8 +1931,9 @@ def copy_table( google.cloud.bigquery.job.CopyJob: A new copy job instance. Raises: - ValueError: - If ``job_config`` is not an instance of ``google.cloud.bigquery.job.CopyJobConfig`` class. + TypeError: + If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.CopyJobConfig` + class. """ job_id = _make_job_id(job_id, job_id_prefix) @@ -1978,12 +1963,7 @@ def copy_table( destination = _table_arg_to_table_ref(destination, default_project=self.project) if job_config: - if not isinstance(job_config, google.cloud.bigquery.job.CopyJobConfig): - raise ValueError( - "Expected an instance of CopyJobConfig class for the job_config parameter, but received job_config = {}".format( - job_config - ) - ) + _verify_job_config_type(job_config, google.cloud.bigquery.job.CopyJobConfig) copy_job = job.CopyJob( job_ref, sources, destination, client=self, job_config=job_config ) @@ -2043,8 +2023,9 @@ def extract_table( google.cloud.bigquery.job.ExtractJob: A new extract job instance. Raises: - ValueError: - If ``job_config`` is not an instance of ``google.cloud.bigquery.job.ExtractJobConfig`` class. + TypeError: + If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.ExtractJobConfig` + class. """ job_id = _make_job_id(job_id, job_id_prefix) @@ -2061,12 +2042,9 @@ def extract_table( destination_uris = [destination_uris] if job_config: - if not isinstance(job_config, google.cloud.bigquery.job.ExtractJobConfig): - raise ValueError( - "Expected an instance of ExtractJobConfig class for the job_config parameter, but received job_config = {}".format( - job_config - ) - ) + _verify_job_config_type( + job_config, google.cloud.bigquery.job.ExtractJobConfig + ) extract_job = job.ExtractJob( job_ref, source, destination_uris, client=self, job_config=job_config ) @@ -2118,8 +2096,9 @@ def query( google.cloud.bigquery.job.QueryJob: A new query job instance. Raises: - ValueError: - If ``job_config`` is not an instance of ``google.cloud.bigquery.job.QueryJobConfig`` class. + TypeError: + If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.QueryJobConfig` + class. """ job_id = _make_job_id(job_id, job_id_prefix) @@ -2131,12 +2110,9 @@ def query( if self._default_query_job_config: if job_config: - if not isinstance(job_config, google.cloud.bigquery.job.QueryJobConfig): - raise ValueError( - "Expected an instance of QueryJobConfig class for the job_config parameter, but received job_config = {}".format( - job_config - ) - ) + _verify_job_config_type( + job_config, google.cloud.bigquery.job.QueryJobConfig + ) # anything that's not defined on the incoming # that is in the default, # should be filled in with the default @@ -2145,15 +2121,10 @@ def query( self._default_query_job_config ) else: - if not isinstance( + _verify_job_config_type( self._default_query_job_config, google.cloud.bigquery.job.QueryJobConfig, - ): - raise ValueError( - "Expected an instance of QueryJobConfig class for the job_config parameter, but received job_config = {}".format( - job_config - ) - ) + ) job_config = self._default_query_job_config job_ref = job._JobReference(job_id, project=project, location=location) diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index ac6fe19ad6c6..91b9bc642187 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -3105,17 +3105,19 @@ def test_load_table_from_uri_w_client_location(self): ) def test_load_table_from_uri_w_invalid_job_config(self): + from google.cloud.bigquery import job + JOB = "job_name" DESTINATION = "destination_table" SOURCE_URI = "http://example.com/source.csv" creds = _make_credentials() http = object() - job_config = object() + job_config = job.CopyJobConfig() client = self._make_one(project=self.PROJECT, credentials=creds, _http=http) destination = client.dataset(self.DS_ID).table(DESTINATION) - with self.assertRaises(ValueError) as exc: + with self.assertRaises(TypeError) as exc: client.load_table_from_uri( SOURCE_URI, destination, job_id=JOB, job_config=job_config ) @@ -3445,6 +3447,8 @@ def test_copy_table_w_source_strings(self): self.assertEqual(job.destination, expected_destination) def test_copy_table_w_invalid_job_config(self): + from google.cloud.bigquery import job + JOB = "job_name" SOURCE = "source_table" DESTINATION = "destination_table" @@ -3452,11 +3456,11 @@ def test_copy_table_w_invalid_job_config(self): creds = _make_credentials() http = object() client = self._make_one(project=self.PROJECT, credentials=creds, _http=http) - job_config = object() + job_config = job.ExtractJobConfig() dataset = client.dataset(self.DS_ID) source = dataset.table(SOURCE) destination = dataset.table(DESTINATION) - with self.assertRaises(ValueError) as exc: + with self.assertRaises(TypeError) as exc: client.copy_table(source, destination, job_id=JOB, job_config=job_config) self.assertIn("Expected an instance of CopyJobConfig", exc.exception.args[0]) @@ -3543,6 +3547,8 @@ def test_extract_table(self): self.assertEqual(list(job.destination_uris), [DESTINATION]) def test_extract_table_w_invalid_job_config(self): + from google.cloud.bigquery import job + JOB = "job_id" SOURCE = "source_table" DESTINATION = "gs://bucket_name/object_name" @@ -3552,8 +3558,8 @@ def test_extract_table_w_invalid_job_config(self): client = self._make_one(project=self.PROJECT, credentials=creds, _http=http) dataset = client.dataset(self.DS_ID) source = dataset.table(SOURCE) - job_config = object() - with self.assertRaises(ValueError) as exc: + job_config = job.LoadJobConfig() + with self.assertRaises(TypeError) as exc: client.extract_table(source, DESTINATION, job_id=JOB, job_config=job_config) self.assertIn("Expected an instance of ExtractJobConfig", exc.exception.args[0]) @@ -3843,6 +3849,7 @@ def test_query_w_explicit_job_config(self): def test_query_w_invalid_job_config(self): from google.cloud.bigquery import QueryJobConfig, DatasetReference + from google.cloud.bigquery import job job_id = "some-job-id" query = "select count(*) from persons" @@ -3861,9 +3868,9 @@ def test_query_w_invalid_job_config(self): default_query_job_config=default_job_config, ) - job_config = object() + job_config = job.LoadJobConfig() - with self.assertRaises(ValueError) as exc: + with self.assertRaises(TypeError) as exc: client.query( query, job_id=job_id, location=self.LOCATION, job_config=job_config ) @@ -3976,7 +3983,7 @@ def test_query_w_invalid_default_job_config(self): default_query_job_config=default_job_config, ) - with self.assertRaises(ValueError) as exc: + with self.assertRaises(TypeError) as exc: client.query(query, job_id=job_id, location=self.LOCATION) self.assertIn("Expected an instance of QueryJobConfig", exc.exception.args[0]) @@ -5561,12 +5568,14 @@ def test_load_table_from_file_bad_mode(self): client.load_table_from_file(file_obj, self.TABLE_REF) def test_load_table_from_file_w_invalid_job_config(self): + from google.cloud.bigquery import job + client = self._make_client() gzip_file = self._make_gzip_file_obj(writable=True) - - with pytest.raises(ValueError) as exc: + config = job.QueryJobConfig() + with pytest.raises(TypeError) as exc: client.load_table_from_file( - gzip_file, self.TABLE_REF, job_id="job_id", job_config=object() + gzip_file, self.TABLE_REF, job_id="job_id", job_config=config ) err_msg = str(exc.value) assert "Expected an instance of LoadJobConfig" in err_msg @@ -6272,13 +6281,15 @@ def test_load_table_from_dataframe_w_nulls(self): @unittest.skipIf(pandas is None, "Requires `pandas`") def test_load_table_from_dataframe_w_invaild_job_config(self): + from google.cloud.bigquery import job + client = self._make_client() records = [{"float_column": 3.14, "struct_column": [{"foo": 1}, {"bar": -1}]}] dataframe = pandas.DataFrame(data=records) - job_config = object() + job_config = job.CopyJobConfig() - with pytest.raises(ValueError) as exc: + with pytest.raises(TypeError) as exc: client.load_table_from_dataframe( dataframe, self.TABLE_REF, job_config=job_config, location=self.LOCATION ) @@ -6375,13 +6386,15 @@ def test_load_table_from_json_non_default_args(self): assert sent_config._properties.get("load", {}).get("unknown_field") == "foobar" def test_load_table_from_json_w_invalid_job_config(self): + from google.cloud.bigquery import job + client = self._make_client() json_rows = [ {"name": "One", "age": 11, "birthday": "2008-09-10", "adult": False}, {"name": "Two", "age": 22, "birthday": "1997-08-09", "adult": True}, ] - job_config = object() - with pytest.raises(ValueError) as exc: + job_config = job.CopyJobConfig() + with pytest.raises(TypeError) as exc: client.load_table_from_json( json_rows, self.TABLE_REF,