From 3a681e046819df18118aa0b2b5733416d004c9b3 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 16 Nov 2021 22:57:51 +0100 Subject: [PATCH 01/26] feat: allow cell magic body to be a $variable (#1053) * feat: allow cell magic body to be a $variable * Fix missing indefinitive article in error msg * Adjust test assertion to error message change * Refactor logic for extracting query variable * Explicitly warn about missing query variable name * Thest the query "variable" is not identifier case --- google/cloud/bigquery/magics/magics.py | 23 ++++ tests/unit/test_magics.py | 139 ++++++++++++++++++++++++- 2 files changed, 161 insertions(+), 1 deletion(-) diff --git a/google/cloud/bigquery/magics/magics.py b/google/cloud/bigquery/magics/magics.py index 1d8d8ed30..5af0a3b51 100644 --- a/google/cloud/bigquery/magics/magics.py +++ b/google/cloud/bigquery/magics/magics.py @@ -596,6 +596,29 @@ def _cell_magic(line, query): _handle_error(error, args.destination_var) return + # Check if query is given as a reference to a variable. + if query.startswith("$"): + query_var_name = query[1:] + + if not query_var_name: + missing_msg = 'Missing query variable name, empty "$" is not allowed.' + raise NameError(missing_msg) + + if query_var_name.isidentifier(): + ip = IPython.get_ipython() + query = ip.user_ns.get(query_var_name, ip) # ip serves as a sentinel + + if query is ip: + raise NameError( + f"Unknown query, variable {query_var_name} does not exist." + ) + else: + if not isinstance(query, (str, bytes)): + raise TypeError( + f"Query variable {query_var_name} must be a string " + "or a bytes-like value." + ) + # Any query that does not contain whitespace (aside from leading and trailing whitespace) # is assumed to be a table id if not re.search(r"\s", query): diff --git a/tests/unit/test_magics.py b/tests/unit/test_magics.py index 36cbf4993..e18d04d64 100644 --- a/tests/unit/test_magics.py +++ b/tests/unit/test_magics.py @@ -584,7 +584,7 @@ def test_bigquery_magic_does_not_clear_display_in_verbose_mode(): @pytest.mark.usefixtures("ipython_interactive") -def test_bigquery_magic_clears_display_in_verbose_mode(): +def test_bigquery_magic_clears_display_in_non_verbose_mode(): ip = IPython.get_ipython() ip.extension_manager.load_extension("google.cloud.bigquery") magics.context.credentials = mock.create_autospec( @@ -1710,6 +1710,143 @@ def test_bigquery_magic_with_improperly_formatted_params(): ip.run_cell_magic("bigquery", "--params {17}", sql) +@pytest.mark.parametrize( + "raw_sql", ("SELECT answer AS 42", " \t SELECT answer AS 42 \t ") +) +@pytest.mark.usefixtures("ipython_interactive") +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") +def test_bigquery_magic_valid_query_in_existing_variable(ipython_ns_cleanup, raw_sql): + ip = IPython.get_ipython() + ip.extension_manager.load_extension("google.cloud.bigquery") + magics.context.credentials = mock.create_autospec( + google.auth.credentials.Credentials, instance=True + ) + + ipython_ns_cleanup.append((ip, "custom_query")) + ipython_ns_cleanup.append((ip, "query_results_df")) + + run_query_patch = mock.patch( + "google.cloud.bigquery.magics.magics._run_query", autospec=True + ) + query_job_mock = mock.create_autospec( + google.cloud.bigquery.job.QueryJob, instance=True + ) + mock_result = pandas.DataFrame([42], columns=["answer"]) + query_job_mock.to_dataframe.return_value = mock_result + + ip.user_ns["custom_query"] = raw_sql + cell_body = "$custom_query" # Referring to an existing variable name (custom_query) + assert "query_results_df" not in ip.user_ns + + with run_query_patch as run_query_mock: + run_query_mock.return_value = query_job_mock + + ip.run_cell_magic("bigquery", "query_results_df", cell_body) + + run_query_mock.assert_called_once_with(mock.ANY, raw_sql, mock.ANY) + + assert "query_results_df" in ip.user_ns # verify that the variable exists + df = ip.user_ns["query_results_df"] + assert len(df) == len(mock_result) # verify row count + assert list(df) == list(mock_result) # verify column names + assert list(df["answer"]) == [42] + + +@pytest.mark.usefixtures("ipython_interactive") +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") +def test_bigquery_magic_nonexisting_query_variable(): + ip = IPython.get_ipython() + ip.extension_manager.load_extension("google.cloud.bigquery") + magics.context.credentials = mock.create_autospec( + google.auth.credentials.Credentials, instance=True + ) + + run_query_patch = mock.patch( + "google.cloud.bigquery.magics.magics._run_query", autospec=True + ) + + ip.user_ns.pop("custom_query", None) # Make sure the variable does NOT exist. + cell_body = "$custom_query" # Referring to a non-existing variable name. + + with pytest.raises( + NameError, match=r".*custom_query does not exist.*" + ), run_query_patch as run_query_mock: + ip.run_cell_magic("bigquery", "", cell_body) + + run_query_mock.assert_not_called() + + +@pytest.mark.usefixtures("ipython_interactive") +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") +def test_bigquery_magic_empty_query_variable_name(): + ip = IPython.get_ipython() + ip.extension_manager.load_extension("google.cloud.bigquery") + magics.context.credentials = mock.create_autospec( + google.auth.credentials.Credentials, instance=True + ) + + run_query_patch = mock.patch( + "google.cloud.bigquery.magics.magics._run_query", autospec=True + ) + cell_body = "$" # Not referring to any variable (name omitted). + + with pytest.raises( + NameError, match=r"(?i).*missing query variable name.*" + ), run_query_patch as run_query_mock: + ip.run_cell_magic("bigquery", "", cell_body) + + run_query_mock.assert_not_called() + + +@pytest.mark.usefixtures("ipython_interactive") +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") +def test_bigquery_magic_query_variable_non_string(ipython_ns_cleanup): + ip = IPython.get_ipython() + ip.extension_manager.load_extension("google.cloud.bigquery") + magics.context.credentials = mock.create_autospec( + google.auth.credentials.Credentials, instance=True + ) + + run_query_patch = mock.patch( + "google.cloud.bigquery.magics.magics._run_query", autospec=True + ) + + ipython_ns_cleanup.append((ip, "custom_query")) + + ip.user_ns["custom_query"] = object() + cell_body = "$custom_query" # Referring to a non-string variable. + + with pytest.raises( + TypeError, match=r".*must be a string or a bytes-like.*" + ), run_query_patch as run_query_mock: + ip.run_cell_magic("bigquery", "", cell_body) + + run_query_mock.assert_not_called() + + +@pytest.mark.usefixtures("ipython_interactive") +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") +def test_bigquery_magic_query_variable_not_identifier(): + ip = IPython.get_ipython() + ip.extension_manager.load_extension("google.cloud.bigquery") + magics.context.credentials = mock.create_autospec( + google.auth.credentials.Credentials, instance=True + ) + + cell_body = "$123foo" # 123foo is not valid Python identifier + + with io.capture_output() as captured_io: + ip.run_cell_magic("bigquery", "", cell_body) + + # If "$" prefixes a string that is not a Python identifier, we do not treat such + # cell_body as a variable reference and just treat is as any other cell body input. + # If at the same time the cell body does not contain any whitespace, it is + # considered a table name, thus we expect an error that the table ID is not valid. + output = captured_io.stderr + assert "ERROR:" in output + assert "must be a fully-qualified ID" in output + + @pytest.mark.usefixtures("ipython_interactive") @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_with_invalid_multiple_option_values(): From 1b5dc5ce1c6419233d9564aa32edbdee523f0f10 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 18 Nov 2021 08:22:57 +0100 Subject: [PATCH 02/26] cleanup: silence non-relevant system test warnings (#1068) --- tests/system/test_pandas.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/system/test_pandas.py b/tests/system/test_pandas.py index 1f43a369a..1541dd3b9 100644 --- a/tests/system/test_pandas.py +++ b/tests/system/test_pandas.py @@ -20,6 +20,7 @@ import json import io import operator +import warnings import google.api_core.retry import pkg_resources @@ -976,9 +977,17 @@ def test_to_geodataframe(bigquery_client, dataset_id): assert df["geog"][2] == wkt.loads("point(0 0)") assert isinstance(df, geopandas.GeoDataFrame) assert isinstance(df["geog"], geopandas.GeoSeries) - assert df.area[0] == 0.5 - assert pandas.isna(df.area[1]) - assert df.area[2] == 0.0 + + with warnings.catch_warnings(): + # Computing the area on a GeoDataFrame that uses a geographic Coordinate + # Reference System (CRS) produces a warning that we are not interested in. + # We do not mind if the computed area is incorrect with respect to the + # GeoDataFrame data, as long as it matches the expected "incorrect" value. + warnings.filterwarnings("ignore", category=UserWarning) + assert df.area[0] == 0.5 + assert pandas.isna(df.area[1]) + assert df.area[2] == 0.0 + assert df.crs.srs == "EPSG:4326" assert df.crs.name == "WGS 84" assert df.geog.crs.srs == "EPSG:4326" From 21cd71022d60c32104f8f90ee2ca445fbb43f7f3 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Fri, 19 Nov 2021 22:40:42 +0000 Subject: [PATCH 03/26] feat: promote `RowIterator.to_arrow_iterable` to public method (#1073) * feat: promote `to_arrow_iterable` to public method * use correct version number * Update google/cloud/bigquery/table.py Co-authored-by: Tim Swast --- google/cloud/bigquery/_pandas_helpers.py | 8 +- google/cloud/bigquery/table.py | 75 +++++++- tests/unit/test_table.py | 218 +++++++++++++++++++++++ 3 files changed, 297 insertions(+), 4 deletions(-) diff --git a/google/cloud/bigquery/_pandas_helpers.py b/google/cloud/bigquery/_pandas_helpers.py index de6356c2a..263a1a9cf 100644 --- a/google/cloud/bigquery/_pandas_helpers.py +++ b/google/cloud/bigquery/_pandas_helpers.py @@ -838,7 +838,12 @@ def _download_table_bqstorage( def download_arrow_bqstorage( - project_id, table, bqstorage_client, preserve_order=False, selected_fields=None, + project_id, + table, + bqstorage_client, + preserve_order=False, + selected_fields=None, + max_queue_size=_MAX_QUEUE_SIZE_DEFAULT, ): return _download_table_bqstorage( project_id, @@ -847,6 +852,7 @@ def download_arrow_bqstorage( preserve_order=preserve_order, selected_fields=selected_fields, page_to_item=_bqstorage_page_to_arrow, + max_queue_size=max_queue_size, ) diff --git a/google/cloud/bigquery/table.py b/google/cloud/bigquery/table.py index 60c8593c7..a0696f83f 100644 --- a/google/cloud/bigquery/table.py +++ b/google/cloud/bigquery/table.py @@ -1629,8 +1629,49 @@ def _to_page_iterable( ) yield from result_pages - def _to_arrow_iterable(self, bqstorage_client=None): - """Create an iterable of arrow RecordBatches, to process the table as a stream.""" + def to_arrow_iterable( + self, + bqstorage_client: "bigquery_storage.BigQueryReadClient" = None, + max_queue_size: int = _pandas_helpers._MAX_QUEUE_SIZE_DEFAULT, # type: ignore + ) -> Iterator["pyarrow.RecordBatch"]: + """[Beta] Create an iterable of class:`pyarrow.RecordBatch`, to process the table as a stream. + + Args: + bqstorage_client (Optional[google.cloud.bigquery_storage_v1.BigQueryReadClient]): + A BigQuery Storage API client. If supplied, use the faster + BigQuery Storage API to fetch rows from BigQuery. + + This method requires the ``pyarrow`` and + ``google-cloud-bigquery-storage`` libraries. + + This method only exposes a subset of the capabilities of the + BigQuery Storage API. For full access to all features + (projections, filters, snapshots) use the Storage API directly. + + max_queue_size (Optional[int]): + The maximum number of result pages to hold in the internal queue when + streaming query results over the BigQuery Storage API. Ignored if + Storage API is not used. + + By default, the max queue size is set to the number of BQ Storage streams + created by the server. If ``max_queue_size`` is :data:`None`, the queue + size is infinite. + + Returns: + pyarrow.RecordBatch: + A generator of :class:`~pyarrow.RecordBatch`. + + Raises: + ValueError: + If the :mod:`pyarrow` library cannot be imported. + + .. versionadded:: 2.31.0 + """ + if pyarrow is None: + raise ValueError(_NO_PYARROW_ERROR) + + self._maybe_warn_max_results(bqstorage_client) + bqstorage_download = functools.partial( _pandas_helpers.download_arrow_bqstorage, self._project, @@ -1638,6 +1679,7 @@ def _to_arrow_iterable(self, bqstorage_client=None): bqstorage_client, preserve_order=self._preserve_order, selected_fields=self._selected_fields, + max_queue_size=max_queue_size, ) tabledata_list_download = functools.partial( _pandas_helpers.download_arrow_row_iterator, iter(self.pages), self.schema @@ -1729,7 +1771,7 @@ def to_arrow( ) record_batches = [] - for record_batch in self._to_arrow_iterable( + for record_batch in self.to_arrow_iterable( bqstorage_client=bqstorage_client ): record_batches.append(record_batch) @@ -2225,6 +2267,33 @@ def to_dataframe_iterable( raise ValueError(_NO_PANDAS_ERROR) return iter((pandas.DataFrame(),)) + def to_arrow_iterable( + self, + bqstorage_client: Optional["bigquery_storage.BigQueryReadClient"] = None, + max_queue_size: Optional[int] = None, + ) -> Iterator["pyarrow.RecordBatch"]: + """Create an iterable of pandas DataFrames, to process the table as a stream. + + .. versionadded:: 2.31.0 + + Args: + bqstorage_client: + Ignored. Added for compatibility with RowIterator. + + max_queue_size: + Ignored. Added for compatibility with RowIterator. + + Returns: + An iterator yielding a single empty :class:`~pyarrow.RecordBatch`. + + Raises: + ValueError: + If the :mod:`pyarrow` library cannot be imported. + """ + if pyarrow is None: + raise ValueError(_NO_PYARROW_ERROR) + return iter((pyarrow.record_batch([]),)) + def __iter__(self): return iter(()) diff --git a/tests/unit/test_table.py b/tests/unit/test_table.py index 3c68e3c5e..4f45eac3d 100644 --- a/tests/unit/test_table.py +++ b/tests/unit/test_table.py @@ -1840,6 +1840,25 @@ def test_to_arrow(self): self.assertIsInstance(tbl, pyarrow.Table) self.assertEqual(tbl.num_rows, 0) + @mock.patch("google.cloud.bigquery.table.pyarrow", new=None) + def test_to_arrow_iterable_error_if_pyarrow_is_none(self): + row_iterator = self._make_one() + with self.assertRaises(ValueError): + row_iterator.to_arrow_iterable() + + @unittest.skipIf(pyarrow is None, "Requires `pyarrow`") + def test_to_arrow_iterable(self): + row_iterator = self._make_one() + arrow_iter = row_iterator.to_arrow_iterable() + + result = list(arrow_iter) + + self.assertEqual(len(result), 1) + record_batch = result[0] + self.assertIsInstance(record_batch, pyarrow.RecordBatch) + self.assertEqual(record_batch.num_rows, 0) + self.assertEqual(record_batch.num_columns, 0) + @mock.patch("google.cloud.bigquery.table.pandas", new=None) def test_to_dataframe_error_if_pandas_is_none(self): row_iterator = self._make_one() @@ -2151,6 +2170,205 @@ def test__validate_bqstorage_returns_false_w_warning_if_obsolete_version(self): ] assert matching_warnings, "Obsolete dependency warning not raised." + @unittest.skipIf(pyarrow is None, "Requires `pyarrow`") + def test_to_arrow_iterable(self): + from google.cloud.bigquery.schema import SchemaField + + schema = [ + SchemaField("name", "STRING", mode="REQUIRED"), + SchemaField("age", "INTEGER", mode="REQUIRED"), + SchemaField( + "child", + "RECORD", + mode="REPEATED", + fields=[ + SchemaField("name", "STRING", mode="REQUIRED"), + SchemaField("age", "INTEGER", mode="REQUIRED"), + ], + ), + ] + rows = [ + { + "f": [ + {"v": "Bharney Rhubble"}, + {"v": "33"}, + { + "v": [ + {"v": {"f": [{"v": "Whamm-Whamm Rhubble"}, {"v": "3"}]}}, + {"v": {"f": [{"v": "Hoppy"}, {"v": "1"}]}}, + ] + }, + ] + }, + { + "f": [ + {"v": "Wylma Phlyntstone"}, + {"v": "29"}, + { + "v": [ + {"v": {"f": [{"v": "Bepples Phlyntstone"}, {"v": "0"}]}}, + {"v": {"f": [{"v": "Dino"}, {"v": "4"}]}}, + ] + }, + ] + }, + ] + path = "/foo" + api_request = mock.Mock( + side_effect=[ + {"rows": [rows[0]], "pageToken": "NEXTPAGE"}, + {"rows": [rows[1]]}, + ] + ) + row_iterator = self._make_one( + _mock_client(), api_request, path, schema, page_size=1, max_results=5 + ) + + record_batches = row_iterator.to_arrow_iterable() + self.assertIsInstance(record_batches, types.GeneratorType) + record_batches = list(record_batches) + self.assertEqual(len(record_batches), 2) + + # Check the schema. + for record_batch in record_batches: + self.assertIsInstance(record_batch, pyarrow.RecordBatch) + self.assertEqual(record_batch.schema[0].name, "name") + self.assertTrue(pyarrow.types.is_string(record_batch.schema[0].type)) + self.assertEqual(record_batch.schema[1].name, "age") + self.assertTrue(pyarrow.types.is_int64(record_batch.schema[1].type)) + child_field = record_batch.schema[2] + self.assertEqual(child_field.name, "child") + self.assertTrue(pyarrow.types.is_list(child_field.type)) + self.assertTrue(pyarrow.types.is_struct(child_field.type.value_type)) + self.assertEqual(child_field.type.value_type[0].name, "name") + self.assertEqual(child_field.type.value_type[1].name, "age") + + # Check the data. + record_batch_1 = record_batches[0].to_pydict() + names = record_batch_1["name"] + ages = record_batch_1["age"] + children = record_batch_1["child"] + self.assertEqual(names, ["Bharney Rhubble"]) + self.assertEqual(ages, [33]) + self.assertEqual( + children, + [ + [ + {"name": "Whamm-Whamm Rhubble", "age": 3}, + {"name": "Hoppy", "age": 1}, + ], + ], + ) + + record_batch_2 = record_batches[1].to_pydict() + names = record_batch_2["name"] + ages = record_batch_2["age"] + children = record_batch_2["child"] + self.assertEqual(names, ["Wylma Phlyntstone"]) + self.assertEqual(ages, [29]) + self.assertEqual( + children, + [[{"name": "Bepples Phlyntstone", "age": 0}, {"name": "Dino", "age": 4}]], + ) + + @mock.patch("google.cloud.bigquery.table.pyarrow", new=None) + def test_to_arrow_iterable_error_if_pyarrow_is_none(self): + from google.cloud.bigquery.schema import SchemaField + + schema = [ + SchemaField("name", "STRING", mode="REQUIRED"), + SchemaField("age", "INTEGER", mode="REQUIRED"), + ] + rows = [ + {"f": [{"v": "Phred Phlyntstone"}, {"v": "32"}]}, + {"f": [{"v": "Bharney Rhubble"}, {"v": "33"}]}, + ] + path = "/foo" + api_request = mock.Mock(return_value={"rows": rows}) + row_iterator = self._make_one(_mock_client(), api_request, path, schema) + + with pytest.raises(ValueError, match="pyarrow"): + row_iterator.to_arrow_iterable() + + @unittest.skipIf(pyarrow is None, "Requires `pyarrow`") + @unittest.skipIf( + bigquery_storage is None, "Requires `google-cloud-bigquery-storage`" + ) + def test_to_arrow_iterable_w_bqstorage(self): + from google.cloud.bigquery import schema + from google.cloud.bigquery import table as mut + from google.cloud.bigquery_storage_v1 import reader + + bqstorage_client = mock.create_autospec(bigquery_storage.BigQueryReadClient) + bqstorage_client._transport = mock.create_autospec( + big_query_read_grpc_transport.BigQueryReadGrpcTransport + ) + streams = [ + # Use two streams we want to check frames are read from each stream. + {"name": "/projects/proj/dataset/dset/tables/tbl/streams/1234"}, + {"name": "/projects/proj/dataset/dset/tables/tbl/streams/5678"}, + ] + session = bigquery_storage.types.ReadSession(streams=streams) + arrow_schema = pyarrow.schema( + [ + pyarrow.field("colA", pyarrow.int64()), + # Not alphabetical to test column order. + pyarrow.field("colC", pyarrow.float64()), + pyarrow.field("colB", pyarrow.string()), + ] + ) + session.arrow_schema.serialized_schema = arrow_schema.serialize().to_pybytes() + bqstorage_client.create_read_session.return_value = session + + mock_rowstream = mock.create_autospec(reader.ReadRowsStream) + bqstorage_client.read_rows.return_value = mock_rowstream + + mock_rows = mock.create_autospec(reader.ReadRowsIterable) + mock_rowstream.rows.return_value = mock_rows + page_items = [ + pyarrow.array([1, -1]), + pyarrow.array([2.0, 4.0]), + pyarrow.array(["abc", "def"]), + ] + + expected_record_batch = pyarrow.RecordBatch.from_arrays( + page_items, schema=arrow_schema + ) + expected_num_record_batches = 3 + + mock_page = mock.create_autospec(reader.ReadRowsPage) + mock_page.to_arrow.return_value = expected_record_batch + mock_pages = (mock_page,) * expected_num_record_batches + type(mock_rows).pages = mock.PropertyMock(return_value=mock_pages) + + schema = [ + schema.SchemaField("colA", "INTEGER"), + schema.SchemaField("colC", "FLOAT"), + schema.SchemaField("colB", "STRING"), + ] + + row_iterator = mut.RowIterator( + _mock_client(), + None, # api_request: ignored + None, # path: ignored + schema, + table=mut.TableReference.from_string("proj.dset.tbl"), + selected_fields=schema, + ) + + record_batches = list( + row_iterator.to_arrow_iterable(bqstorage_client=bqstorage_client) + ) + total_record_batches = len(streams) * len(mock_pages) + self.assertEqual(len(record_batches), total_record_batches) + + for record_batch in record_batches: + # Are the record batches return as expected? + self.assertEqual(record_batch, expected_record_batch) + + # Don't close the client if it was passed in. + bqstorage_client._transport.grpc_channel.close.assert_not_called() + @unittest.skipIf(pyarrow is None, "Requires `pyarrow`") def test_to_arrow(self): from google.cloud.bigquery.schema import SchemaField From 3314dfbed62488503dc41b11e403a672fcf71048 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 24 Nov 2021 16:12:10 +0100 Subject: [PATCH 04/26] fix: apply timeout to all resumable upload requests (#1070) * fix: apply timeout to all resumable upload requests * Fix stub in test case * Improve timeout type and other type annotations * Annnotate return type of _do_resumable_upload() --- google/cloud/bigquery/client.py | 186 +++++++++++++++++++------------- tests/unit/test_client.py | 18 +++- 2 files changed, 128 insertions(+), 76 deletions(-) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 3e641e195..a5f3d5419 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -31,9 +31,10 @@ import typing from typing import ( Any, - BinaryIO, Dict, + IO, Iterable, + Mapping, List, Optional, Sequence, @@ -112,10 +113,15 @@ pyarrow = _helpers.PYARROW_VERSIONS.try_import() TimeoutType = Union[float, None] +ResumableTimeoutType = Union[ + None, float, Tuple[float, float] +] # for resumable media methods if typing.TYPE_CHECKING: # pragma: NO COVER # os.PathLike is only subscriptable in Python 3.9+, thus shielding with a condition. PathType = Union[str, bytes, os.PathLike[str], os.PathLike[bytes]] + import pandas # type: ignore + import requests # required by api-core _DEFAULT_CHUNKSIZE = 100 * 1024 * 1024 # 100 MB _MAX_MULTIPART_SIZE = 5 * 1024 * 1024 @@ -2348,7 +2354,7 @@ def load_table_from_uri( def load_table_from_file( self, - file_obj: BinaryIO, + file_obj: IO[bytes], destination: Union[Table, TableReference, TableListItem, str], rewind: bool = False, size: int = None, @@ -2358,7 +2364,7 @@ def load_table_from_file( location: str = None, project: str = None, job_config: LoadJobConfig = None, - timeout: TimeoutType = DEFAULT_TIMEOUT, + timeout: ResumableTimeoutType = DEFAULT_TIMEOUT, ) -> job.LoadJob: """Upload the contents of this table from a file-like object. @@ -2366,42 +2372,42 @@ def load_table_from_file( returns a :class:`~google.cloud.bigquery.job.LoadJob`. Args: - file_obj (file): A file handle opened in binary mode for reading. - destination (Union[ \ - google.cloud.bigquery.table.Table, \ - google.cloud.bigquery.table.TableReference, \ - google.cloud.bigquery.table.TableListItem, \ - str, \ - ]): + file_obj: + A file handle opened in binary mode for reading. + destination: Table into which data is to be loaded. If a string is passed in, this method attempts to create a table reference from a string using :func:`google.cloud.bigquery.table.TableReference.from_string`. Keyword Arguments: - rewind (Optional[bool]): + rewind: If True, seek to the beginning of the file handle before reading the file. - size (Optional[int]): + size: The number of bytes to read from the file handle. If size is ``None`` or large, resumable upload will be used. Otherwise, multipart upload will be used. - num_retries (Optional[int]): Number of upload retries. Defaults to 6. - job_id (Optional[str]): Name of the job. - job_id_prefix (Optional[str]): + num_retries: Number of upload retries. Defaults to 6. + job_id: Name of the job. + job_id_prefix: The user-provided prefix for a randomly generated job ID. This parameter will be ignored if a ``job_id`` is also given. - location (Optional[str]): + location: Location where to run the job. Must match the location of the destination table. - project (Optional[str]): + project: Project ID of the project of where to run the job. Defaults to the client's project. - job_config (Optional[google.cloud.bigquery.job.LoadJobConfig]): + job_config: Extra configuration options for the job. - timeout (Optional[float]): + timeout: The number of seconds to wait for the underlying HTTP transport - before using ``retry``. + before using ``retry``. Depending on the retry strategy, a request + may be repeated several times using the same timeout each time. + + Can also be passed as a tuple (connect_timeout, read_timeout). + See :meth:`requests.Session.request` documentation for details. Returns: google.cloud.bigquery.job.LoadJob: A new load job. @@ -2453,7 +2459,7 @@ def load_table_from_file( def load_table_from_dataframe( self, - dataframe, + dataframe: "pandas.DataFrame", destination: Union[Table, TableReference, str], num_retries: int = _DEFAULT_NUM_RETRIES, job_id: str = None, @@ -2462,7 +2468,7 @@ def load_table_from_dataframe( project: str = None, job_config: LoadJobConfig = None, parquet_compression: str = "snappy", - timeout: TimeoutType = DEFAULT_TIMEOUT, + timeout: ResumableTimeoutType = DEFAULT_TIMEOUT, ) -> job.LoadJob: """Upload the contents of a table from a pandas DataFrame. @@ -2481,9 +2487,9 @@ def load_table_from_dataframe( https://github.com/googleapis/python-bigquery/issues/19 Args: - dataframe (pandas.DataFrame): + dataframe: A :class:`~pandas.DataFrame` containing the data to load. - destination (google.cloud.bigquery.table.TableReference): + destination: The destination table to use for loading the data. If it is an existing table, the schema of the :class:`~pandas.DataFrame` must match the schema of the destination table. If the table @@ -2495,19 +2501,19 @@ def load_table_from_dataframe( :func:`google.cloud.bigquery.table.TableReference.from_string`. Keyword Arguments: - num_retries (Optional[int]): Number of upload retries. - job_id (Optional[str]): Name of the job. - job_id_prefix (Optional[str]): + num_retries: Number of upload retries. + job_id: Name of the job. + job_id_prefix: The user-provided prefix for a randomly generated job ID. This parameter will be ignored if a ``job_id`` is also given. - location (Optional[str]): + location: Location where to run the job. Must match the location of the destination table. - project (Optional[str]): + project: Project ID of the project of where to run the job. Defaults to the client's project. - job_config (Optional[google.cloud.bigquery.job.LoadJobConfig]): + job_config: Extra configuration options for the job. To override the default pandas data type conversions, supply @@ -2524,7 +2530,7 @@ def load_table_from_dataframe( :attr:`~google.cloud.bigquery.job.SourceFormat.CSV` and :attr:`~google.cloud.bigquery.job.SourceFormat.PARQUET` are supported. - parquet_compression (Optional[str]): + parquet_compression: [Beta] The compression method to use if intermittently serializing ``dataframe`` to a parquet file. @@ -2537,9 +2543,13 @@ def load_table_from_dataframe( passed as the ``compression`` argument to the underlying ``DataFrame.to_parquet()`` method. https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_parquet.html#pandas.DataFrame.to_parquet - timeout (Optional[float]): + timeout: The number of seconds to wait for the underlying HTTP transport - before using ``retry``. + before using ``retry``. Depending on the retry strategy, a request may + be repeated several times using the same timeout each time. + + Can also be passed as a tuple (connect_timeout, read_timeout). + See :meth:`requests.Session.request` documentation for details. Returns: google.cloud.bigquery.job.LoadJob: A new load job. @@ -2717,7 +2727,7 @@ def load_table_from_json( location: str = None, project: str = None, job_config: LoadJobConfig = None, - timeout: TimeoutType = DEFAULT_TIMEOUT, + timeout: ResumableTimeoutType = DEFAULT_TIMEOUT, ) -> job.LoadJob: """Upload the contents of a table from a JSON string or dict. @@ -2741,36 +2751,35 @@ def load_table_from_json( client = bigquery.Client() client.load_table_from_file(data_as_file, ...) - destination (Union[ \ - google.cloud.bigquery.table.Table, \ - google.cloud.bigquery.table.TableReference, \ - google.cloud.bigquery.table.TableListItem, \ - str, \ - ]): + destination: Table into which data is to be loaded. If a string is passed in, this method attempts to create a table reference from a string using :func:`google.cloud.bigquery.table.TableReference.from_string`. Keyword Arguments: - num_retries (Optional[int]): Number of upload retries. - job_id (Optional[str]): Name of the job. - job_id_prefix (Optional[str]): + num_retries: Number of upload retries. + job_id: Name of the job. + job_id_prefix: The user-provided prefix for a randomly generated job ID. This parameter will be ignored if a ``job_id`` is also given. - location (Optional[str]): + location: Location where to run the job. Must match the location of the destination table. - project (Optional[str]): + project: Project ID of the project of where to run the job. Defaults to the client's project. - job_config (Optional[google.cloud.bigquery.job.LoadJobConfig]): + job_config: Extra configuration options for the job. The ``source_format`` setting is always set to :attr:`~google.cloud.bigquery.job.SourceFormat.NEWLINE_DELIMITED_JSON`. - timeout (Optional[float]): + timeout: The number of seconds to wait for the underlying HTTP transport - before using ``retry``. + before using ``retry``. Depending on the retry strategy, a request may + be repeated several times using the same timeout each time. + + Can also be passed as a tuple (connect_timeout, read_timeout). + See :meth:`requests.Session.request` documentation for details. Returns: google.cloud.bigquery.job.LoadJob: A new load job. @@ -2819,60 +2828,77 @@ def load_table_from_json( ) def _do_resumable_upload( - self, stream, metadata, num_retries, timeout, project=None - ): + self, + stream: IO[bytes], + metadata: Mapping[str, str], + num_retries: int, + timeout: Optional[ResumableTimeoutType], + project: Optional[str] = None, + ) -> "requests.Response": """Perform a resumable upload. Args: - stream (IO[bytes]): A bytes IO object open for reading. + stream: A bytes IO object open for reading. - metadata (Dict): The metadata associated with the upload. + metadata: The metadata associated with the upload. - num_retries (int): + num_retries: Number of upload retries. (Deprecated: This argument will be removed in a future release.) - timeout (float): + timeout: The number of seconds to wait for the underlying HTTP transport - before using ``retry``. + before using ``retry``. Depending on the retry strategy, a request may + be repeated several times using the same timeout each time. - project (Optional[str]): + Can also be passed as a tuple (connect_timeout, read_timeout). + See :meth:`requests.Session.request` documentation for details. + + project: Project ID of the project of where to run the upload. Defaults to the client's project. Returns: - requests.Response: - The "200 OK" response object returned after the final chunk - is uploaded. + The "200 OK" response object returned after the final chunk + is uploaded. """ upload, transport = self._initiate_resumable_upload( stream, metadata, num_retries, timeout, project=project ) while not upload.finished: - response = upload.transmit_next_chunk(transport) + response = upload.transmit_next_chunk(transport, timeout=timeout) return response def _initiate_resumable_upload( - self, stream, metadata, num_retries, timeout, project=None + self, + stream: IO[bytes], + metadata: Mapping[str, str], + num_retries: int, + timeout: Optional[ResumableTimeoutType], + project: Optional[str] = None, ): """Initiate a resumable upload. Args: - stream (IO[bytes]): A bytes IO object open for reading. + stream: A bytes IO object open for reading. - metadata (Dict): The metadata associated with the upload. + metadata: The metadata associated with the upload. - num_retries (int): + num_retries: Number of upload retries. (Deprecated: This argument will be removed in a future release.) - timeout (float): + timeout: The number of seconds to wait for the underlying HTTP transport - before using ``retry``. + before using ``retry``. Depending on the retry strategy, a request may + be repeated several times using the same timeout each time. - project (Optional[str]): + Can also be passed as a tuple (connect_timeout, read_timeout). + See :meth:`requests.Session.request` documentation for details. + + project: Project ID of the project of where to run the upload. Defaults to the client's project. @@ -2921,29 +2947,39 @@ def _initiate_resumable_upload( return upload, transport def _do_multipart_upload( - self, stream, metadata, size, num_retries, timeout, project=None + self, + stream: IO[bytes], + metadata: Mapping[str, str], + size: int, + num_retries: int, + timeout: Optional[ResumableTimeoutType], + project: Optional[str] = None, ): """Perform a multipart upload. Args: - stream (IO[bytes]): A bytes IO object open for reading. + stream: A bytes IO object open for reading. - metadata (Dict): The metadata associated with the upload. + metadata: The metadata associated with the upload. - size (int): + size: The number of bytes to be uploaded (which will be read from ``stream``). If not provided, the upload will be concluded once ``stream`` is exhausted (or :data:`None`). - num_retries (int): + num_retries: Number of upload retries. (Deprecated: This argument will be removed in a future release.) - timeout (float): + timeout: The number of seconds to wait for the underlying HTTP transport - before using ``retry``. + before using ``retry``. Depending on the retry strategy, a request may + be repeated several times using the same timeout each time. - project (Optional[str]): + Can also be passed as a tuple (connect_timeout, read_timeout). + See :meth:`requests.Session.request` documentation for details. + + project: Project ID of the project of where to run the upload. Defaults to the client's project. diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 97aa2eedb..9c93765e8 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -8235,6 +8235,22 @@ def test__do_resumable_upload_custom_project(self): assert initiation_url is not None assert "projects/custom-project" in initiation_url + def test__do_resumable_upload_custom_timeout(self): + file_obj = self._make_file_obj() + file_obj_len = len(file_obj.getvalue()) + transport = self._make_transport( + self._make_resumable_upload_responses(file_obj_len) + ) + client = self._make_client(transport) + + client._do_resumable_upload( + file_obj, self.EXPECTED_CONFIGURATION, num_retries=0, timeout=3.14 + ) + + # The timeout should be applied to all underlying calls. + for call_args in transport.request.call_args_list: + assert call_args.kwargs.get("timeout") == 3.14 + def test__do_multipart_upload(self): transport = self._make_transport([self._make_response(http.client.OK)]) client = self._make_client(transport) @@ -8442,7 +8458,7 @@ def test_upload_chunksize(client): upload.finished = False - def transmit_next_chunk(transport): + def transmit_next_chunk(transport, *args, **kwargs): upload.finished = True result = mock.MagicMock() result.json.return_value = {} From a1359560d6a54b5c261eadc1559db59b2fa58f1a Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 2 Dec 2021 14:14:11 -0600 Subject: [PATCH 05/26] chore: release 2.31.0 (#1066) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 18 ++++++++++++++++++ google/cloud/bigquery/version.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e10ad826..5ba219d20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,24 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +## [2.31.0](https://www.github.com/googleapis/python-bigquery/compare/v2.30.1...v2.31.0) (2021-11-24) + + +### Features + +* allow cell magic body to be a $variable ([#1053](https://www.github.com/googleapis/python-bigquery/issues/1053)) ([3a681e0](https://www.github.com/googleapis/python-bigquery/commit/3a681e046819df18118aa0b2b5733416d004c9b3)) +* promote `RowIterator.to_arrow_iterable` to public method ([#1073](https://www.github.com/googleapis/python-bigquery/issues/1073)) ([21cd710](https://www.github.com/googleapis/python-bigquery/commit/21cd71022d60c32104f8f90ee2ca445fbb43f7f3)) + + +### Bug Fixes + +* apply timeout to all resumable upload requests ([#1070](https://www.github.com/googleapis/python-bigquery/issues/1070)) ([3314dfb](https://www.github.com/googleapis/python-bigquery/commit/3314dfbed62488503dc41b11e403a672fcf71048)) + + +### Dependencies + +* support OpenTelemetry >= 1.1.0 ([#1050](https://www.github.com/googleapis/python-bigquery/issues/1050)) ([4616cd5](https://www.github.com/googleapis/python-bigquery/commit/4616cd58d3c6da641fb881ce99a87dcdedc20ba2)) + ### [2.30.1](https://www.github.com/googleapis/python-bigquery/compare/v2.30.0...v2.30.1) (2021-11-04) diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index 877ea53d8..6329658af 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.30.1" +__version__ = "2.31.0" From be6eb3429f9fd6ad5839826983ba96a0e0a071d3 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 2 Dec 2021 14:42:14 -0600 Subject: [PATCH 06/26] test: check extreme DATE/DATETIME values can be loaded from pandas DataFrame (#1078) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Towards #1076 🦕 (edit: moved to https://github.com/googleapis/python-db-dtypes-pandas/issues/45 ) --- tests/system/test_pandas.py | 41 ++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/tests/system/test_pandas.py b/tests/system/test_pandas.py index 1541dd3b9..f3534cd19 100644 --- a/tests/system/test_pandas.py +++ b/tests/system/test_pandas.py @@ -268,7 +268,7 @@ def test_load_table_from_dataframe_w_nulls(bigquery_client, dataset_id): See: https://github.com/googleapis/google-cloud-python/issues/7370 """ # Schema with all scalar types. - scalars_schema = ( + table_schema = ( bigquery.SchemaField("bool_col", "BOOLEAN"), bigquery.SchemaField("bytes_col", "BYTES"), bigquery.SchemaField("date_col", "DATE"), @@ -283,15 +283,6 @@ def test_load_table_from_dataframe_w_nulls(bigquery_client, dataset_id): bigquery.SchemaField("ts_col", "TIMESTAMP"), ) - table_schema = scalars_schema + ( - # TODO: Array columns can't be read due to NULLABLE versus REPEATED - # mode mismatch. See: - # https://issuetracker.google.com/133415569#comment3 - # bigquery.SchemaField("array_col", "INTEGER", mode="REPEATED"), - # TODO: Support writing StructArrays to Parquet. See: - # https://jira.apache.org/jira/browse/ARROW-2587 - # bigquery.SchemaField("struct_col", "RECORD", fields=scalars_schema), - ) num_rows = 100 nulls = [None] * num_rows df_data = [ @@ -372,7 +363,8 @@ def test_load_table_from_dataframe_w_explicit_schema(bigquery_client, dataset_id # See: # https://github.com/googleapis/python-bigquery/issues/61 # https://issuetracker.google.com/issues/151765076 - scalars_schema = ( + table_schema = ( + bigquery.SchemaField("row_num", "INTEGER"), bigquery.SchemaField("bool_col", "BOOLEAN"), bigquery.SchemaField("bytes_col", "BYTES"), bigquery.SchemaField("date_col", "DATE"), @@ -387,17 +379,8 @@ def test_load_table_from_dataframe_w_explicit_schema(bigquery_client, dataset_id bigquery.SchemaField("ts_col", "TIMESTAMP"), ) - table_schema = scalars_schema + ( - # TODO: Array columns can't be read due to NULLABLE versus REPEATED - # mode mismatch. See: - # https://issuetracker.google.com/133415569#comment3 - # bigquery.SchemaField("array_col", "INTEGER", mode="REPEATED"), - # TODO: Support writing StructArrays to Parquet. See: - # https://jira.apache.org/jira/browse/ARROW-2587 - # bigquery.SchemaField("struct_col", "RECORD", fields=scalars_schema), - ) - df_data = [ + ("row_num", [1, 2, 3]), ("bool_col", [True, None, False]), ("bytes_col", [b"abc", None, b"def"]), ("date_col", [datetime.date(1, 1, 1), None, datetime.date(9999, 12, 31)]), @@ -464,6 +447,22 @@ def test_load_table_from_dataframe_w_explicit_schema(bigquery_client, dataset_id assert tuple(table.schema) == table_schema assert table.num_rows == 3 + result = bigquery_client.list_rows(table).to_dataframe() + result.sort_values("row_num", inplace=True) + + # Check that extreme DATE/DATETIME values are loaded correctly. + # https://github.com/googleapis/python-bigquery/issues/1076 + assert result["date_col"][0] == datetime.date(1, 1, 1) + assert result["date_col"][2] == datetime.date(9999, 12, 31) + assert result["dt_col"][0] == datetime.datetime(1, 1, 1, 0, 0, 0) + assert result["dt_col"][2] == datetime.datetime(9999, 12, 31, 23, 59, 59, 999999) + assert result["ts_col"][0] == datetime.datetime( + 1, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc + ) + assert result["ts_col"][2] == datetime.datetime( + 9999, 12, 31, 23, 59, 59, 999999, tzinfo=datetime.timezone.utc + ) + def test_load_table_from_dataframe_w_struct_datatype(bigquery_client, dataset_id): """Test that a DataFrame with struct datatype can be uploaded if a From effd6734a08914216482ba370d9433364d278f3a Mon Sep 17 00:00:00 2001 From: Lo Ferris <50979514+loferris@users.noreply.github.com> Date: Mon, 6 Dec 2021 09:50:44 -0800 Subject: [PATCH 07/26] docs: add sample for revoking dataset access (#778) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * revoke dataset access setup * basic template for sample * sample + test * revoke dataset access sample * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/master/packages/owl-bot/README.md * docs: add sample for revoking dataset access - update year and string formatting * docs: add sample for revoking dataset access - move to snippets and change parameter pattern for readibility * moving update_dataset to /snippets and adjusting imports on both revoke_access and update_access * Update samples/snippets/revoke_dataset_access.py removed nested START/END tags Co-authored-by: Tim Swast * Update samples/snippets/revoke_dataset_access.py update readability in API request Co-authored-by: Tim Swast * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/master/packages/owl-bot/README.md * updated test * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/master/packages/owl-bot/README.md * change after running test * resolving linting failure, rewriting test * removed relative import errors * remove relative mport from update_dataset_access * adding fixture to conftest.py * updated sample * updating sample to match new update_access sample * fixing region tags * consolidated tests into one file for both methods * updating test to full_dataset format * updated revoke sample * updating test * refactored sample * Update samples/snippets/conftest.py * Update samples/snippets/revoke_dataset_access.py Co-authored-by: Tim Swast * Update samples/snippets/update_dataset_access.py Co-authored-by: Tim Swast * Update samples/snippets/revoke_dataset_access.py Co-authored-by: Tim Swast * Update samples/snippets/revoke_dataset_access.py Co-authored-by: Tim Swast * refactoring entry * added comment for entry access * Update samples/snippets/README.rst Co-authored-by: Tim Swast * Update samples/snippets/dataset_access_test.py Co-authored-by: Tim Swast * Update samples/snippets/dataset_access_test.py Co-authored-by: Tim Swast * added develper TODO in sample * add comments to samples Co-authored-by: Owl Bot Co-authored-by: Tim Swast Co-authored-by: Peter Lamut Co-authored-by: Anthonios Partheniou Co-authored-by: meredithslota --- samples/snippets/README.rst | 27 ++------- samples/snippets/conftest.py | 5 ++ samples/snippets/dataset_access_test.py | 48 ++++++++++++++++ samples/snippets/revoke_dataset_access.py | 52 +++++++++++++++++ samples/snippets/update_dataset_access.py | 70 +++++++++++++++++++++++ 5 files changed, 180 insertions(+), 22 deletions(-) create mode 100644 samples/snippets/dataset_access_test.py create mode 100644 samples/snippets/revoke_dataset_access.py create mode 100644 samples/snippets/update_dataset_access.py diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index 7c3e19e68..05af1e812 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -1,4 +1,3 @@ - .. This file is automatically generated. Do not edit this file directly. Google BigQuery Python Samples @@ -16,11 +15,14 @@ This directory contains samples for Google BigQuery. `Google BigQuery`_ is Googl .. _Google BigQuery: https://cloud.google.com/bigquery/docs +To run the sample, you need to have the `BigQuery Admin` role. + + + Setup ------------------------------------------------------------------------------- - Authentication ++++++++++++++ @@ -31,9 +33,6 @@ credentials for applications. .. _Authentication Getting Started Guide: https://cloud.google.com/docs/authentication/getting-started - - - Install Dependencies ++++++++++++++++++++ @@ -64,15 +63,9 @@ Install Dependencies .. _pip: https://pip.pypa.io/ .. _virtualenv: https://virtualenv.pypa.io/ - - - - - Samples ------------------------------------------------------------------------------- - Quickstart +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -89,8 +82,6 @@ To run this sample: $ python quickstart.py - - Simple Application +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -107,8 +98,6 @@ To run this sample: $ python simple_app.py - - User Credentials +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -124,7 +113,6 @@ To run this sample: $ python user_credentials.py - usage: user_credentials.py [-h] [--launch-browser] project Command-line application to run a query using user credentials. @@ -143,10 +131,6 @@ To run this sample: - - - - The client library ------------------------------------------------------------------------------- @@ -162,5 +146,4 @@ to `browse the source`_ and `report issues`_. https://github.com/GoogleCloudPlatform/google-cloud-python/issues - -.. _Google Cloud SDK: https://cloud.google.com/sdk/ +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/samples/snippets/conftest.py b/samples/snippets/conftest.py index 74984f902..e8aa08487 100644 --- a/samples/snippets/conftest.py +++ b/samples/snippets/conftest.py @@ -50,6 +50,11 @@ def dataset_id(bigquery_client: bigquery.Client, project_id: str): bigquery_client.delete_dataset(dataset, delete_contents=True, not_found_ok=True) +@pytest.fixture(scope="session") +def entity_id(bigquery_client: bigquery.Client, dataset_id: str): + return "cloud-developer-relations@google.com" + + @pytest.fixture(scope="session") def dataset_id_us_east1(bigquery_client: bigquery.Client, project_id: str): dataset_id = prefixer.create_prefix() diff --git a/samples/snippets/dataset_access_test.py b/samples/snippets/dataset_access_test.py new file mode 100644 index 000000000..21776c149 --- /dev/null +++ b/samples/snippets/dataset_access_test.py @@ -0,0 +1,48 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import revoke_dataset_access +import update_dataset_access + + +def test_dataset_access_permissions(capsys, dataset_id, entity_id, bigquery_client): + original_dataset = bigquery_client.get_dataset(dataset_id) + update_dataset_access.update_dataset_access(dataset_id, entity_id) + full_dataset_id = "{}.{}".format( + original_dataset.project, original_dataset.dataset_id + ) + + out, err = capsys.readouterr() + assert ( + "Updated dataset '{}' with modified user permissions.".format(full_dataset_id) + in out + ) + + updated_dataset = bigquery_client.get_dataset(dataset_id) + updated_dataset_entries = list(updated_dataset.access_entries) + updated_dataset_entity_ids = {entry.entity_id for entry in updated_dataset_entries} + assert entity_id in updated_dataset_entity_ids + revoke_dataset_access.revoke_dataset_access(dataset_id, entity_id) + revoked_dataset = bigquery_client.get_dataset(dataset_id) + revoked_dataset_entries = list(revoked_dataset.access_entries) + + full_dataset_id = f"{updated_dataset.project}.{updated_dataset.dataset_id}" + out, err = capsys.readouterr() + assert ( + f"Revoked dataset access for '{entity_id}' to ' dataset '{full_dataset_id}.'" + in out + ) + assert len(revoked_dataset_entries) == len(updated_dataset_entries) - 1 + revoked_dataset_entity_ids = {entry.entity_id for entry in revoked_dataset_entries} + assert entity_id not in revoked_dataset_entity_ids diff --git a/samples/snippets/revoke_dataset_access.py b/samples/snippets/revoke_dataset_access.py new file mode 100644 index 000000000..ce78f5750 --- /dev/null +++ b/samples/snippets/revoke_dataset_access.py @@ -0,0 +1,52 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def revoke_dataset_access(dataset_id: str, entity_id: str): + original_dataset_id = dataset_id + original_entity_id = entity_id + + # [START bigquery_revoke_dataset_access] + + # TODO(developer): Set dataset_id to the ID of the dataset to fetch. + dataset_id = "your-project.your_dataset" + + # TODO(developer): Set entity_id to the ID of the email or group from whom you are revoking access. + entity_id = "user-or-group-to-remove@example.com" + # [END bigquery_revoke_dataset_access] + dataset_id = original_dataset_id + entity_id = original_entity_id + # [START bigquery_revoke_dataset_access] + + from google.cloud import bigquery + + # Construct a BigQuery client object. + client = bigquery.Client() + + dataset = client.get_dataset(dataset_id) # Make an API request. + + entries = list(dataset.access_entries) + dataset.access_entries = [ + entry for entry in entries if entry.entity_id != entity_id + ] + + dataset = client.update_dataset( + dataset, + # Update just the `access_entries` property of the dataset. + ["access_entries"], + ) # Make an API request. + + full_dataset_id = f"{dataset.project}.{dataset.dataset_id}" + print(f"Revoked dataset access for '{entity_id}' to ' dataset '{full_dataset_id}.'") + # [END bigquery_revoke_dataset_access] diff --git a/samples/snippets/update_dataset_access.py b/samples/snippets/update_dataset_access.py new file mode 100644 index 000000000..fb3bfa14f --- /dev/null +++ b/samples/snippets/update_dataset_access.py @@ -0,0 +1,70 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def update_dataset_access(dataset_id: str, entity_id: str): + original_dataset_id = dataset_id + original_entity_id = entity_id + + # [START bigquery_update_dataset_access] + + # TODO(developer): Set dataset_id to the ID of the dataset to fetch. + dataset_id = "your-project.your_dataset" + + # TODO(developer): Set entity_id to the ID of the email or group from whom + # you are adding access. Alternatively, to the JSON REST API representation + # of the entity, such as a view's table reference. + entity_id = "user-or-group-to-add@example.com" + + # TODO(developer): Set entity_type to the type of entity you are granting access to. + # Common types include: + # + # * "userByEmail" -- A single user or service account. For example "fred@example.com" + # * "groupByEmail" -- A group of users. For example "example@googlegroups.com" + # * "view" -- An authorized view. For example + # {"projectId": "p", "datasetId": "d", "tableId": "v"} + # + # For a complete reference, see the REST API reference documentation: + # https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#Dataset.FIELDS.access + entity_type = "groupByEmail" + + # TODO(developer): Set role to a one of the "Basic roles for datasets" + # described here: + # https://cloud.google.com/bigquery/docs/access-control-basic-roles#dataset-basic-roles + role = "READER" + # [END bigquery_update_dataset_access] + dataset_id = original_dataset_id + entity_id = original_entity_id + # [START bigquery_update_dataset_access] + + from google.cloud import bigquery + + # Construct a BigQuery client object. + client = bigquery.Client() + + dataset = client.get_dataset(dataset_id) # Make an API request. + + entries = list(dataset.access_entries) + entries.append( + bigquery.AccessEntry(role=role, entity_type=entity_type, entity_id=entity_id,) + ) + dataset.access_entries = entries + + dataset = client.update_dataset(dataset, ["access_entries"]) # Make an API request. + + full_dataset_id = "{}.{}".format(dataset.project, dataset.dataset_id) + print( + "Updated dataset '{}' with modified user permissions.".format(full_dataset_id) + ) + # [END bigquery_update_dataset_access] From d1a9902198da1f9b5e5c77f51e0adf10027e049c Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Sat, 27 Nov 2021 16:52:38 +0200 Subject: [PATCH 08/26] Add type hints to magics samples --- samples/magics/_helpers.py | 2 +- samples/magics/conftest.py | 12 ++++++++++-- samples/magics/query.py | 7 ++++++- samples/magics/query_params_scalars.py | 7 ++++++- samples/magics/query_params_scalars_test.py | 2 +- samples/magics/query_test.py | 2 +- 6 files changed, 25 insertions(+), 7 deletions(-) diff --git a/samples/magics/_helpers.py b/samples/magics/_helpers.py index 18a513b99..c7248ee3d 100644 --- a/samples/magics/_helpers.py +++ b/samples/magics/_helpers.py @@ -13,7 +13,7 @@ # limitations under the License. -def strip_region_tags(sample_text): +def strip_region_tags(sample_text: str) -> str: """Remove blank lines and region tags from sample text""" magic_lines = [ line for line in sample_text.split("\n") if len(line) > 0 and "# [" not in line diff --git a/samples/magics/conftest.py b/samples/magics/conftest.py index bf8602235..55ea30f90 100644 --- a/samples/magics/conftest.py +++ b/samples/magics/conftest.py @@ -12,14 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing +from typing import Iterator + import pytest +if typing.TYPE_CHECKING: + from IPython.core.interactiveshell import TerminalInteractiveShell + interactiveshell = pytest.importorskip("IPython.terminal.interactiveshell") tools = pytest.importorskip("IPython.testing.tools") @pytest.fixture(scope="session") -def ipython(): +def ipython() -> "TerminalInteractiveShell": config = tools.default_config() config.TerminalInteractiveShell.simple_prompt = True shell = interactiveshell.TerminalInteractiveShell.instance(config=config) @@ -27,7 +33,9 @@ def ipython(): @pytest.fixture(autouse=True) -def ipython_interactive(ipython): +def ipython_interactive( + ipython: "TerminalInteractiveShell", +) -> Iterator["TerminalInteractiveShell"]: """Activate IPython's builtin hooks for the duration of the test scope. diff --git a/samples/magics/query.py b/samples/magics/query.py index c2739eace..4d3b4418b 100644 --- a/samples/magics/query.py +++ b/samples/magics/query.py @@ -12,12 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + import IPython from . import _helpers +if typing.TYPE_CHECKING: + import pandas + -def query(): +def query() -> "pandas.DataFrame": ip = IPython.get_ipython() ip.extension_manager.load_extension("google.cloud.bigquery") diff --git a/samples/magics/query_params_scalars.py b/samples/magics/query_params_scalars.py index a26f25aea..e833ef93b 100644 --- a/samples/magics/query_params_scalars.py +++ b/samples/magics/query_params_scalars.py @@ -12,12 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + import IPython from . import _helpers +if typing.TYPE_CHECKING: + import pandas + -def query_with_parameters(): +def query_with_parameters() -> "pandas.DataFrame": ip = IPython.get_ipython() ip.extension_manager.load_extension("google.cloud.bigquery") diff --git a/samples/magics/query_params_scalars_test.py b/samples/magics/query_params_scalars_test.py index 9b4159667..4f481cbe9 100644 --- a/samples/magics/query_params_scalars_test.py +++ b/samples/magics/query_params_scalars_test.py @@ -17,7 +17,7 @@ from . import query_params_scalars -def test_query_with_parameters(): +def test_query_with_parameters() -> None: df = query_params_scalars.query_with_parameters() assert isinstance(df, pandas.DataFrame) assert len(df) == 10 diff --git a/samples/magics/query_test.py b/samples/magics/query_test.py index d20797908..1aaa9c1bb 100644 --- a/samples/magics/query_test.py +++ b/samples/magics/query_test.py @@ -17,7 +17,7 @@ from . import query -def test_query(): +def test_query() -> None: df = query.query() assert isinstance(df, pandas.DataFrame) assert len(df) == 3 From 8141548b58af85cf8df3fd3b399635f5d4a8ba83 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Sun, 28 Nov 2021 15:38:57 +0200 Subject: [PATCH 09/26] Add type hints to geography samples --- samples/geography/conftest.py | 13 ++++++++----- samples/geography/insert_geojson.py | 9 ++++++++- samples/geography/insert_geojson_test.py | 2 +- samples/geography/insert_wkt.py | 9 ++++++++- samples/geography/insert_wkt_test.py | 2 +- samples/geography/to_geodataframe.py | 10 ++++++++-- samples/geography/to_geodataframe_test.py | 2 +- 7 files changed, 35 insertions(+), 12 deletions(-) diff --git a/samples/geography/conftest.py b/samples/geography/conftest.py index 265900f5a..14823d10a 100644 --- a/samples/geography/conftest.py +++ b/samples/geography/conftest.py @@ -13,30 +13,31 @@ # limitations under the License. import datetime +from typing import Iterator import uuid from google.cloud import bigquery import pytest -def temp_suffix(): +def temp_suffix() -> str: now = datetime.datetime.now() return f"{now.strftime('%Y%m%d%H%M%S')}_{uuid.uuid4().hex[:8]}" @pytest.fixture(scope="session") -def bigquery_client(): +def bigquery_client() -> bigquery.Client: bigquery_client = bigquery.Client() return bigquery_client @pytest.fixture(scope="session") -def project_id(bigquery_client): +def project_id(bigquery_client: bigquery.Client) -> str: return bigquery_client.project @pytest.fixture -def dataset_id(bigquery_client): +def dataset_id(bigquery_client: bigquery.Client) -> Iterator[str]: dataset_id = f"geography_{temp_suffix()}" bigquery_client.create_dataset(dataset_id) yield dataset_id @@ -44,7 +45,9 @@ def dataset_id(bigquery_client): @pytest.fixture -def table_id(bigquery_client, project_id, dataset_id): +def table_id( + bigquery_client: bigquery.Client, project_id: str, dataset_id: str +) -> Iterator[str]: table_id = f"{project_id}.{dataset_id}.geography_{temp_suffix()}" table = bigquery.Table(table_id) table.schema = [ diff --git a/samples/geography/insert_geojson.py b/samples/geography/insert_geojson.py index 23f249c15..e98d72cb5 100644 --- a/samples/geography/insert_geojson.py +++ b/samples/geography/insert_geojson.py @@ -12,14 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Dict, Mapping, Optional, Sequence -def insert_geojson(override_values={}): + +def insert_geojson( + override_values: Optional[Mapping[str, str]] = None +) -> Sequence[Dict[str, object]]: # [START bigquery_insert_geojson] import geojson from google.cloud import bigquery bigquery_client = bigquery.Client() + if override_values is None: + override_values = {} + # This example uses a table containing a column named "geo" with the # GEOGRAPHY data type. table_id = "my-project.my_dataset.my_table" diff --git a/samples/geography/insert_geojson_test.py b/samples/geography/insert_geojson_test.py index 5ef15ee13..507201872 100644 --- a/samples/geography/insert_geojson_test.py +++ b/samples/geography/insert_geojson_test.py @@ -15,6 +15,6 @@ from . import insert_geojson -def test_insert_geojson(table_id): +def test_insert_geojson(table_id: str) -> None: errors = insert_geojson.insert_geojson(override_values={"table_id": table_id}) assert not errors diff --git a/samples/geography/insert_wkt.py b/samples/geography/insert_wkt.py index d7d3accde..d95ed4b87 100644 --- a/samples/geography/insert_wkt.py +++ b/samples/geography/insert_wkt.py @@ -12,8 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Dict, Mapping, Optional, Sequence -def insert_wkt(override_values={}): + +def insert_wkt( + override_values: Optional[Mapping[str, str]] = None +) -> Sequence[Dict[str, object]]: # [START bigquery_insert_geography_wkt] from google.cloud import bigquery import shapely.geometry @@ -21,6 +25,9 @@ def insert_wkt(override_values={}): bigquery_client = bigquery.Client() + if override_values is None: + override_values = {} + # This example uses a table containing a column named "geo" with the # GEOGRAPHY data type. table_id = "my-project.my_dataset.my_table" diff --git a/samples/geography/insert_wkt_test.py b/samples/geography/insert_wkt_test.py index 8bcb62cec..a7c3d4ed3 100644 --- a/samples/geography/insert_wkt_test.py +++ b/samples/geography/insert_wkt_test.py @@ -15,6 +15,6 @@ from . import insert_wkt -def test_insert_wkt(table_id): +def test_insert_wkt(table_id: str) -> None: errors = insert_wkt.insert_wkt(override_values={"table_id": table_id}) assert not errors diff --git a/samples/geography/to_geodataframe.py b/samples/geography/to_geodataframe.py index fa8073fef..e36331f27 100644 --- a/samples/geography/to_geodataframe.py +++ b/samples/geography/to_geodataframe.py @@ -12,12 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from google.cloud import bigquery -client = bigquery.Client() +if typing.TYPE_CHECKING: + import pandas + + +client: bigquery.Client = bigquery.Client() -def get_austin_service_requests_as_geography(): +def get_austin_service_requests_as_geography() -> "pandas.DataFrame": # [START bigquery_query_results_geodataframe] sql = """ diff --git a/samples/geography/to_geodataframe_test.py b/samples/geography/to_geodataframe_test.py index 7a2ba6937..7499d7001 100644 --- a/samples/geography/to_geodataframe_test.py +++ b/samples/geography/to_geodataframe_test.py @@ -17,7 +17,7 @@ from .to_geodataframe import get_austin_service_requests_as_geography -def test_get_austin_service_requests_as_geography(): +def test_get_austin_service_requests_as_geography() -> None: geopandas = pytest.importorskip("geopandas") df = get_austin_service_requests_as_geography() assert isinstance(df, geopandas.GeoDataFrame) From 0b58d9ddc5686fc4cfb5a4c94373c8e7391e0ba5 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Mon, 29 Nov 2021 16:56:17 +0200 Subject: [PATCH 10/26] Add type hints to snippets --- .../snippets/authenticate_service_account.py | 6 ++- .../authenticate_service_account_test.py | 9 ++++- samples/snippets/authorized_view_tutorial.py | 9 ++++- .../snippets/authorized_view_tutorial_test.py | 11 ++++-- samples/snippets/conftest.py | 24 ++++++++---- .../create_table_external_hive_partitioned.py | 7 +++- ...te_table_external_hive_partitioned_test.py | 9 ++++- samples/snippets/delete_job.py | 2 +- samples/snippets/delete_job_test.py | 11 +++++- samples/snippets/jupyter_tutorial_test.py | 17 +++++++-- samples/snippets/load_table_uri_firestore.py | 2 +- .../snippets/load_table_uri_firestore_test.py | 9 ++++- samples/snippets/materialized_view.py | 25 ++++++++++-- samples/snippets/materialized_view_test.py | 24 +++++++++--- samples/snippets/natality_tutorial.py | 7 +++- samples/snippets/natality_tutorial_test.py | 11 ++++-- samples/snippets/quickstart.py | 8 +++- samples/snippets/quickstart_test.py | 13 +++++-- samples/snippets/simple_app.py | 2 +- samples/snippets/simple_app_test.py | 7 +++- samples/snippets/test_update_with_dml.py | 10 ++++- samples/snippets/update_with_dml.py | 13 +++++-- samples/snippets/user_credentials.py | 2 +- samples/snippets/user_credentials_test.py | 9 ++++- samples/snippets/view.py | 38 +++++++++++++++++-- samples/snippets/view_test.py | 31 ++++++++++----- 26 files changed, 246 insertions(+), 70 deletions(-) diff --git a/samples/snippets/authenticate_service_account.py b/samples/snippets/authenticate_service_account.py index c07848bee..e44766886 100644 --- a/samples/snippets/authenticate_service_account.py +++ b/samples/snippets/authenticate_service_account.py @@ -13,9 +13,13 @@ # limitations under the License. import os +import typing + +if typing.TYPE_CHECKING: + from google.cloud import bigquery -def main(): +def main() -> "bigquery.Client": key_path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") # [START bigquery_client_json_credentials] diff --git a/samples/snippets/authenticate_service_account_test.py b/samples/snippets/authenticate_service_account_test.py index 131c69d2c..0c4d582c7 100644 --- a/samples/snippets/authenticate_service_account_test.py +++ b/samples/snippets/authenticate_service_account_test.py @@ -12,19 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + import google.auth import authenticate_service_account +if typing.TYPE_CHECKING: + import pytest + -def mock_credentials(*args, **kwargs): +def mock_credentials() -> google.auth.credentials.Credentials: credentials, _ = google.auth.default( ["https://www.googleapis.com/auth/cloud-platform"] ) return credentials -def test_main(monkeypatch): +def test_main(monkeypatch: "pytest.MonkeyPatch") -> None: monkeypatch.setattr( "google.oauth2.service_account.Credentials.from_service_account_file", mock_credentials, diff --git a/samples/snippets/authorized_view_tutorial.py b/samples/snippets/authorized_view_tutorial.py index b6a20c6ec..d3aeb1da1 100644 --- a/samples/snippets/authorized_view_tutorial.py +++ b/samples/snippets/authorized_view_tutorial.py @@ -14,12 +14,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Dict, Optional -def run_authorized_view_tutorial(override_values={}): + +def run_authorized_view_tutorial( + override_values: Optional[Dict[str, str]] = None +) -> None: # Note to user: This is a group email for testing purposes. Replace with # your own group email address when running this code. analyst_group_email = "example-analyst-group@google.com" + if override_values is None: + override_values = {} + # [START bigquery_authorized_view_tutorial] # Create a source dataset # [START bigquery_avt_create_source_dataset] diff --git a/samples/snippets/authorized_view_tutorial_test.py b/samples/snippets/authorized_view_tutorial_test.py index eb247c5eb..cae870486 100644 --- a/samples/snippets/authorized_view_tutorial_test.py +++ b/samples/snippets/authorized_view_tutorial_test.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Iterator, List import uuid from google.cloud import bigquery @@ -21,19 +22,21 @@ @pytest.fixture(scope="module") -def client(): +def client() -> bigquery.Client: return bigquery.Client() @pytest.fixture -def datasets_to_delete(client): - doomed = [] +def datasets_to_delete(client: bigquery.Client) -> Iterator[List[str]]: + doomed: List[str] = [] yield doomed for item in doomed: client.delete_dataset(item, delete_contents=True, not_found_ok=True) -def test_authorized_view_tutorial(client, datasets_to_delete): +def test_authorized_view_tutorial( + client: bigquery.Client, datasets_to_delete: List[str] +) -> None: override_values = { "source_dataset_id": "github_source_data_{}".format( str(uuid.uuid4()).replace("-", "_") diff --git a/samples/snippets/conftest.py b/samples/snippets/conftest.py index 74984f902..05c73c17c 100644 --- a/samples/snippets/conftest.py +++ b/samples/snippets/conftest.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Iterator + from google.cloud import bigquery import pytest import test_utils.prefixer @@ -21,7 +23,7 @@ @pytest.fixture(scope="session", autouse=True) -def cleanup_datasets(bigquery_client: bigquery.Client): +def cleanup_datasets(bigquery_client: bigquery.Client) -> None: for dataset in bigquery_client.list_datasets(): if prefixer.should_cleanup(dataset.dataset_id): bigquery_client.delete_dataset( @@ -30,18 +32,18 @@ def cleanup_datasets(bigquery_client: bigquery.Client): @pytest.fixture(scope="session") -def bigquery_client(): +def bigquery_client() -> bigquery.Client: bigquery_client = bigquery.Client() return bigquery_client @pytest.fixture(scope="session") -def project_id(bigquery_client): +def project_id(bigquery_client: bigquery.Client) -> str: return bigquery_client.project @pytest.fixture(scope="session") -def dataset_id(bigquery_client: bigquery.Client, project_id: str): +def dataset_id(bigquery_client: bigquery.Client, project_id: str) -> Iterator[str]: dataset_id = prefixer.create_prefix() full_dataset_id = f"{project_id}.{dataset_id}" dataset = bigquery.Dataset(full_dataset_id) @@ -51,7 +53,9 @@ def dataset_id(bigquery_client: bigquery.Client, project_id: str): @pytest.fixture(scope="session") -def dataset_id_us_east1(bigquery_client: bigquery.Client, project_id: str): +def dataset_id_us_east1( + bigquery_client: bigquery.Client, project_id: str +) -> Iterator[str]: dataset_id = prefixer.create_prefix() full_dataset_id = f"{project_id}.{dataset_id}" dataset = bigquery.Dataset(full_dataset_id) @@ -64,7 +68,7 @@ def dataset_id_us_east1(bigquery_client: bigquery.Client, project_id: str): @pytest.fixture(scope="session") def table_id_us_east1( bigquery_client: bigquery.Client, project_id: str, dataset_id_us_east1: str -): +) -> Iterator[str]: table_id = prefixer.create_prefix() full_table_id = f"{project_id}.{dataset_id_us_east1}.{table_id}" table = bigquery.Table( @@ -76,7 +80,9 @@ def table_id_us_east1( @pytest.fixture -def random_table_id(bigquery_client: bigquery.Client, project_id: str, dataset_id: str): +def random_table_id( + bigquery_client: bigquery.Client, project_id: str, dataset_id: str +) -> Iterator[str]: """Create a new table ID each time, so random_table_id can be used as target for load jobs. """ @@ -87,5 +93,7 @@ def random_table_id(bigquery_client: bigquery.Client, project_id: str, dataset_i @pytest.fixture -def bigquery_client_patch(monkeypatch, bigquery_client): +def bigquery_client_patch( + monkeypatch: pytest.MonkeyPatch, bigquery_client: bigquery.Client +) -> None: monkeypatch.setattr(bigquery, "Client", lambda: bigquery_client) diff --git a/samples/snippets/create_table_external_hive_partitioned.py b/samples/snippets/create_table_external_hive_partitioned.py index 2ff8a2220..1170c57da 100644 --- a/samples/snippets/create_table_external_hive_partitioned.py +++ b/samples/snippets/create_table_external_hive_partitioned.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing -def create_table_external_hive_partitioned(table_id: str): +if typing.TYPE_CHECKING: + from google.cloud import bigquery + + +def create_table_external_hive_partitioned(table_id: str) -> "bigquery.Table": original_table_id = table_id # [START bigquery_create_table_external_hivepartitioned] # Demonstrates creating an external table with hive partitioning. diff --git a/samples/snippets/create_table_external_hive_partitioned_test.py b/samples/snippets/create_table_external_hive_partitioned_test.py index c3cdddb55..3ff39c881 100644 --- a/samples/snippets/create_table_external_hive_partitioned_test.py +++ b/samples/snippets/create_table_external_hive_partitioned_test.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + import create_table_external_hive_partitioned +if typing.TYPE_CHECKING: + import pytest + -def test_create_table_external_hive_partitioned(capsys, random_table_id): +def test_create_table_external_hive_partitioned( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: table = create_table_external_hive_partitioned.create_table_external_hive_partitioned( random_table_id ) diff --git a/samples/snippets/delete_job.py b/samples/snippets/delete_job.py index abed0c90d..7c8640baf 100644 --- a/samples/snippets/delete_job.py +++ b/samples/snippets/delete_job.py @@ -13,7 +13,7 @@ # limitations under the License. -def delete_job_metadata(job_id: str, location: str): +def delete_job_metadata(job_id: str, location: str) -> None: orig_job_id = job_id orig_location = location # [START bigquery_delete_job] diff --git a/samples/snippets/delete_job_test.py b/samples/snippets/delete_job_test.py index c9baa817d..0bc83e4a6 100644 --- a/samples/snippets/delete_job_test.py +++ b/samples/snippets/delete_job_test.py @@ -12,14 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from google.cloud import bigquery import delete_job +if typing.TYPE_CHECKING: + import pytest + def test_delete_job_metadata( - capsys, bigquery_client: bigquery.Client, table_id_us_east1: str -): + capsys: "pytest.CaptureFixture[str]", + bigquery_client: bigquery.Client, + table_id_us_east1: str, +) -> None: query_job: bigquery.QueryJob = bigquery_client.query( f"SELECT COUNT(*) FROM `{table_id_us_east1}`", location="us-east1", ) diff --git a/samples/snippets/jupyter_tutorial_test.py b/samples/snippets/jupyter_tutorial_test.py index 7fe1cde85..9d42a4eda 100644 --- a/samples/snippets/jupyter_tutorial_test.py +++ b/samples/snippets/jupyter_tutorial_test.py @@ -11,8 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +import typing +from typing import Iterator + import pytest +if typing.TYPE_CHECKING: + from IPython.terminal.interactiveshell import TerminalInteractiveShell + IPython = pytest.importorskip("IPython") interactiveshell = pytest.importorskip("IPython.terminal.interactiveshell") tools = pytest.importorskip("IPython.testing.tools") @@ -23,7 +30,7 @@ @pytest.fixture(scope="session") -def ipython(): +def ipython() -> "TerminalInteractiveShell": config = tools.default_config() config.TerminalInteractiveShell.simple_prompt = True shell = interactiveshell.TerminalInteractiveShell.instance(config=config) @@ -31,7 +38,9 @@ def ipython(): @pytest.fixture() -def ipython_interactive(request, ipython): +def ipython_interactive( + request: pytest.FixtureRequest, ipython: "TerminalInteractiveShell" +) -> Iterator["TerminalInteractiveShell"]: """Activate IPython's builtin hooks for the duration of the test scope. @@ -40,7 +49,7 @@ def ipython_interactive(request, ipython): yield ipython -def _strip_region_tags(sample_text): +def _strip_region_tags(sample_text: str) -> str: """Remove blank lines and region tags from sample text""" magic_lines = [ line for line in sample_text.split("\n") if len(line) > 0 and "# [" not in line @@ -48,7 +57,7 @@ def _strip_region_tags(sample_text): return "\n".join(magic_lines) -def test_jupyter_tutorial(ipython): +def test_jupyter_tutorial(ipython: "TerminalInteractiveShell") -> None: matplotlib.use("agg") ip = IPython.get_ipython() ip.extension_manager.load_extension("google.cloud.bigquery") diff --git a/samples/snippets/load_table_uri_firestore.py b/samples/snippets/load_table_uri_firestore.py index bf9d01349..6c33fd0ff 100644 --- a/samples/snippets/load_table_uri_firestore.py +++ b/samples/snippets/load_table_uri_firestore.py @@ -13,7 +13,7 @@ # limitations under the License. -def load_table_uri_firestore(table_id): +def load_table_uri_firestore(table_id: str) -> None: orig_table_id = table_id # [START bigquery_load_table_gcs_firestore] # TODO(developer): Set table_id to the ID of the table to create. diff --git a/samples/snippets/load_table_uri_firestore_test.py b/samples/snippets/load_table_uri_firestore_test.py index ffa02cdf9..552fa2e35 100644 --- a/samples/snippets/load_table_uri_firestore_test.py +++ b/samples/snippets/load_table_uri_firestore_test.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + import load_table_uri_firestore +if typing.TYPE_CHECKING: + import pytest + -def test_load_table_uri_firestore(capsys, random_table_id): +def test_load_table_uri_firestore( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: load_table_uri_firestore.load_table_uri_firestore(random_table_id) out, _ = capsys.readouterr() assert "Loaded 50 rows." in out diff --git a/samples/snippets/materialized_view.py b/samples/snippets/materialized_view.py index 429bd98b4..adb3688a4 100644 --- a/samples/snippets/materialized_view.py +++ b/samples/snippets/materialized_view.py @@ -12,8 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing +from typing import Dict, Optional + +if typing.TYPE_CHECKING: + from google.cloud import bigquery + + +def create_materialized_view( + override_values: Optional[Dict[str, str]] = None +) -> "bigquery.Table": + if override_values is None: + override_values = {} -def create_materialized_view(override_values={}): # [START bigquery_create_materialized_view] from google.cloud import bigquery @@ -41,7 +52,12 @@ def create_materialized_view(override_values={}): return view -def update_materialized_view(override_values={}): +def update_materialized_view( + override_values: Optional[Dict[str, str]] = None +) -> "bigquery.Table": + if override_values is None: + override_values = {} + # [START bigquery_update_materialized_view] import datetime from google.cloud import bigquery @@ -69,7 +85,10 @@ def update_materialized_view(override_values={}): return view -def delete_materialized_view(override_values={}): +def delete_materialized_view(override_values: Optional[Dict[str, str]] = None) -> None: + if override_values is None: + override_values = {} + # [START bigquery_delete_materialized_view] from google.cloud import bigquery diff --git a/samples/snippets/materialized_view_test.py b/samples/snippets/materialized_view_test.py index 75c6b2106..70869346f 100644 --- a/samples/snippets/materialized_view_test.py +++ b/samples/snippets/materialized_view_test.py @@ -13,6 +13,7 @@ # limitations under the License. import datetime +from typing import Iterator import uuid from google.api_core import exceptions @@ -22,18 +23,20 @@ import materialized_view -def temp_suffix(): +def temp_suffix() -> str: now = datetime.datetime.now() return f"{now.strftime('%Y%m%d%H%M%S')}_{uuid.uuid4().hex[:8]}" @pytest.fixture(autouse=True) -def bigquery_client_patch(monkeypatch, bigquery_client): +def bigquery_client_patch( + monkeypatch: pytest.MonkeyPatch, bigquery_client: bigquery.Client +) -> None: monkeypatch.setattr(bigquery, "Client", lambda: bigquery_client) @pytest.fixture(scope="module") -def dataset_id(bigquery_client): +def dataset_id(bigquery_client: bigquery.Client) -> Iterator[str]: dataset_id = f"mvdataset_{temp_suffix()}" bigquery_client.create_dataset(dataset_id) yield dataset_id @@ -41,7 +44,9 @@ def dataset_id(bigquery_client): @pytest.fixture(scope="module") -def base_table_id(bigquery_client, project_id, dataset_id): +def base_table_id( + bigquery_client: bigquery.Client, project_id: str, dataset_id: str +) -> Iterator[str]: base_table_id = f"{project_id}.{dataset_id}.base_{temp_suffix()}" # Schema from materialized views guide: # https://cloud.google.com/bigquery/docs/materialized-views#create @@ -56,13 +61,20 @@ def base_table_id(bigquery_client, project_id, dataset_id): @pytest.fixture(scope="module") -def view_id(bigquery_client, project_id, dataset_id): +def view_id( + bigquery_client: bigquery.Client, project_id: str, dataset_id: str +) -> Iterator[str]: view_id = f"{project_id}.{dataset_id}.mview_{temp_suffix()}" yield view_id bigquery_client.delete_table(view_id, not_found_ok=True) -def test_materialized_view(capsys, bigquery_client, base_table_id, view_id): +def test_materialized_view( + capsys: pytest.CaptureFixture[str], + bigquery_client: bigquery.Client, + base_table_id: str, + view_id: str, +) -> None: override_values = { "base_table_id": base_table_id, "view_id": view_id, diff --git a/samples/snippets/natality_tutorial.py b/samples/snippets/natality_tutorial.py index ed08b279a..b330a3c21 100644 --- a/samples/snippets/natality_tutorial.py +++ b/samples/snippets/natality_tutorial.py @@ -14,8 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Dict, Optional + + +def run_natality_tutorial(override_values: Optional[Dict[str, str]] = None) -> None: + if override_values is None: + override_values = {} -def run_natality_tutorial(override_values={}): # [START bigquery_query_natality_tutorial] """Create a Google BigQuery linear regression input table. diff --git a/samples/snippets/natality_tutorial_test.py b/samples/snippets/natality_tutorial_test.py index d9c89bef2..f56738528 100644 --- a/samples/snippets/natality_tutorial_test.py +++ b/samples/snippets/natality_tutorial_test.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Iterator, List import uuid from google.cloud import bigquery @@ -21,19 +22,21 @@ @pytest.fixture(scope="module") -def client(): +def client() -> bigquery.Client: return bigquery.Client() @pytest.fixture -def datasets_to_delete(client): - doomed = [] +def datasets_to_delete(client: bigquery.Client) -> Iterator[List[str]]: + doomed: List[str] = [] yield doomed for item in doomed: client.delete_dataset(item, delete_contents=True) -def test_natality_tutorial(client, datasets_to_delete): +def test_natality_tutorial( + client: bigquery.Client, datasets_to_delete: List[str] +) -> None: override_values = { "dataset_id": "natality_regression_{}".format( str(uuid.uuid4()).replace("-", "_") diff --git a/samples/snippets/quickstart.py b/samples/snippets/quickstart.py index 1b0ef5b3a..f9628da7d 100644 --- a/samples/snippets/quickstart.py +++ b/samples/snippets/quickstart.py @@ -14,8 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Dict, Optional + + +def run_quickstart(override_values: Optional[Dict[str, str]] = None) -> None: + + if override_values is None: + override_values = {} -def run_quickstart(override_values={}): # [START bigquery_quickstart] # Imports the Google Cloud client library from google.cloud import bigquery diff --git a/samples/snippets/quickstart_test.py b/samples/snippets/quickstart_test.py index a5e3a13e3..b0bad5ee5 100644 --- a/samples/snippets/quickstart_test.py +++ b/samples/snippets/quickstart_test.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Iterator, List import uuid from google.cloud import bigquery @@ -26,19 +27,23 @@ @pytest.fixture(scope="module") -def client(): +def client() -> bigquery.Client: return bigquery.Client() @pytest.fixture -def datasets_to_delete(client): - doomed = [] +def datasets_to_delete(client: bigquery.Client) -> Iterator[List[str]]: + doomed: List[str] = [] yield doomed for item in doomed: client.delete_dataset(item, delete_contents=True) -def test_quickstart(capsys, client, datasets_to_delete): +def test_quickstart( + capsys: "pytest.CaptureFixture[str]", + client: bigquery.Client, + datasets_to_delete: List[str], +) -> None: override_values = { "dataset_id": "my_new_dataset_{}".format(str(uuid.uuid4()).replace("-", "_")), diff --git a/samples/snippets/simple_app.py b/samples/snippets/simple_app.py index c21ae86f4..3d856d4bb 100644 --- a/samples/snippets/simple_app.py +++ b/samples/snippets/simple_app.py @@ -22,7 +22,7 @@ # [END bigquery_simple_app_deps] -def query_stackoverflow(): +def query_stackoverflow() -> None: # [START bigquery_simple_app_client] client = bigquery.Client() # [END bigquery_simple_app_client] diff --git a/samples/snippets/simple_app_test.py b/samples/snippets/simple_app_test.py index 5c608e1fd..de4e1ce34 100644 --- a/samples/snippets/simple_app_test.py +++ b/samples/snippets/simple_app_test.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + import simple_app +if typing.TYPE_CHECKING: + import pytest + -def test_query_stackoverflow(capsys): +def test_query_stackoverflow(capsys: "pytest.CaptureFixture[str]") -> None: simple_app.query_stackoverflow() out, _ = capsys.readouterr() assert "views" in out diff --git a/samples/snippets/test_update_with_dml.py b/samples/snippets/test_update_with_dml.py index 912fd76e2..ef5ec196a 100644 --- a/samples/snippets/test_update_with_dml.py +++ b/samples/snippets/test_update_with_dml.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Iterator + from google.cloud import bigquery import pytest @@ -20,14 +22,18 @@ @pytest.fixture -def table_id(bigquery_client: bigquery.Client, project_id: str, dataset_id: str): +def table_id( + bigquery_client: bigquery.Client, project_id: str, dataset_id: str +) -> Iterator[str]: table_id = f"{prefixer.create_prefix()}_update_with_dml" yield table_id full_table_id = f"{project_id}.{dataset_id}.{table_id}" bigquery_client.delete_table(full_table_id, not_found_ok=True) -def test_update_with_dml(bigquery_client_patch, dataset_id, table_id): +def test_update_with_dml( + bigquery_client_patch: None, dataset_id: str, table_id: str +) -> None: override_values = { "dataset_id": dataset_id, "table_id": table_id, diff --git a/samples/snippets/update_with_dml.py b/samples/snippets/update_with_dml.py index 7fd09dd80..b6408072c 100644 --- a/samples/snippets/update_with_dml.py +++ b/samples/snippets/update_with_dml.py @@ -14,6 +14,8 @@ # [START bigquery_update_with_dml] import pathlib +import typing +from typing import Dict, Optional from google.cloud import bigquery from google.cloud.bigquery import enums @@ -25,7 +27,7 @@ def load_from_newline_delimited_json( project_id: str, dataset_id: str, table_id: str, -): +) -> None: full_table_id = f"{project_id}.{dataset_id}.{table_id}" job_config = bigquery.LoadJobConfig() job_config.source_format = enums.SourceFormat.NEWLINE_DELIMITED_JSON @@ -48,7 +50,7 @@ def load_from_newline_delimited_json( def update_with_dml( client: bigquery.Client, project_id: str, dataset_id: str, table_id: str -): +) -> int: query_text = f""" UPDATE `{project_id}.{dataset_id}.{table_id}` SET ip_address = REGEXP_REPLACE(ip_address, r"(\\.[0-9]+)$", ".0") @@ -60,10 +62,13 @@ def update_with_dml( query_job.result() print(f"DML query modified {query_job.num_dml_affected_rows} rows.") - return query_job.num_dml_affected_rows + return typing.cast(int, query_job.num_dml_affected_rows) -def run_sample(override_values={}): +def run_sample(override_values: Optional[Dict[str, str]] = None) -> int: + if override_values is None: + override_values = {} + client = bigquery.Client() filepath = pathlib.Path(__file__).parent / "user_sessions_data.json" project_id = client.project diff --git a/samples/snippets/user_credentials.py b/samples/snippets/user_credentials.py index 6089d9fd9..dcd498c42 100644 --- a/samples/snippets/user_credentials.py +++ b/samples/snippets/user_credentials.py @@ -23,7 +23,7 @@ import argparse -def main(project, launch_browser=True): +def main(project: str, launch_browser: bool = True) -> None: # [START bigquery_auth_user_flow] from google_auth_oauthlib import flow diff --git a/samples/snippets/user_credentials_test.py b/samples/snippets/user_credentials_test.py index 829502d25..79ebb2538 100644 --- a/samples/snippets/user_credentials_test.py +++ b/samples/snippets/user_credentials_test.py @@ -13,6 +13,7 @@ # limitations under the License. import os +from typing import Iterator, Union import google.auth import mock @@ -23,9 +24,11 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +MockType = Union[mock.mock.MagicMock, mock.mock.AsyncMock] + @pytest.fixture -def mock_flow(): +def mock_flow() -> Iterator[MockType]: flow_patch = mock.patch("google_auth_oauthlib.flow.InstalledAppFlow", autospec=True) with flow_patch as flow_mock: @@ -34,7 +37,9 @@ def mock_flow(): yield flow_mock -def test_auth_query_console(mock_flow, capsys): +def test_auth_query_console( + mock_flow: MockType, capsys: pytest.CaptureFixture[str] +) -> None: main(PROJECT, launch_browser=False) out, _ = capsys.readouterr() # Fun fact: William P. Wood was the 1st director of the US Secret Service. diff --git a/samples/snippets/view.py b/samples/snippets/view.py index ad3f11717..443c5b75e 100644 --- a/samples/snippets/view.py +++ b/samples/snippets/view.py @@ -12,8 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing +from typing import Dict, Optional, Tuple, TypedDict + +if typing.TYPE_CHECKING: + from google.cloud import bigquery + + +class OverridesDict(TypedDict, total=False): + analyst_group_email: str + view_dataset_id: str + view_id: str + view_reference: Dict[str, str] + source_dataset_id: str + source_id: str + + +def create_view(override_values: Optional[Dict[str, str]] = None) -> "bigquery.Table": + if override_values is None: + override_values = {} -def create_view(override_values={}): # [START bigquery_create_view] from google.cloud import bigquery @@ -43,7 +61,10 @@ def create_view(override_values={}): return view -def get_view(override_values={}): +def get_view(override_values: Optional[Dict[str, str]] = None) -> "bigquery.Table": + if override_values is None: + override_values = {} + # [START bigquery_get_view] from google.cloud import bigquery @@ -65,7 +86,10 @@ def get_view(override_values={}): return view -def update_view(override_values={}): +def update_view(override_values: Optional[Dict[str, str]] = None) -> "bigquery.Table": + if override_values is None: + override_values = {} + # [START bigquery_update_view_query] from google.cloud import bigquery @@ -95,7 +119,13 @@ def update_view(override_values={}): return view -def grant_access(override_values={}): +def grant_access( + override_values: Optional[OverridesDict] = None, +) -> Tuple["bigquery.Dataset", "bigquery.Dataset"]: + + if override_values is None: + override_values = {} + # [START bigquery_grant_view_access] from google.cloud import bigquery diff --git a/samples/snippets/view_test.py b/samples/snippets/view_test.py index 77105b61a..4d0d43b77 100644 --- a/samples/snippets/view_test.py +++ b/samples/snippets/view_test.py @@ -13,6 +13,7 @@ # limitations under the License. import datetime +from typing import Iterator import uuid from google.cloud import bigquery @@ -21,18 +22,20 @@ import view -def temp_suffix(): +def temp_suffix() -> str: now = datetime.datetime.now() return f"{now.strftime('%Y%m%d%H%M%S')}_{uuid.uuid4().hex[:8]}" @pytest.fixture(autouse=True) -def bigquery_client_patch(monkeypatch, bigquery_client): +def bigquery_client_patch( + monkeypatch: pytest.MonkeyPatch, bigquery_client: bigquery.Client +) -> None: monkeypatch.setattr(bigquery, "Client", lambda: bigquery_client) @pytest.fixture(scope="module") -def view_dataset_id(bigquery_client, project_id): +def view_dataset_id(bigquery_client: bigquery.Client, project_id: str) -> Iterator[str]: dataset_id = f"{project_id}.view_{temp_suffix()}" bigquery_client.create_dataset(dataset_id) yield dataset_id @@ -40,14 +43,16 @@ def view_dataset_id(bigquery_client, project_id): @pytest.fixture(scope="module") -def view_id(bigquery_client, view_dataset_id): +def view_id(bigquery_client: bigquery.Client, view_dataset_id: str) -> Iterator[str]: view_id = f"{view_dataset_id}.my_view" yield view_id bigquery_client.delete_table(view_id, not_found_ok=True) @pytest.fixture(scope="module") -def source_dataset_id(bigquery_client, project_id): +def source_dataset_id( + bigquery_client: bigquery.Client, project_id: str +) -> Iterator[str]: dataset_id = f"{project_id}.view_{temp_suffix()}" bigquery_client.create_dataset(dataset_id) yield dataset_id @@ -55,7 +60,9 @@ def source_dataset_id(bigquery_client, project_id): @pytest.fixture(scope="module") -def source_table_id(bigquery_client, source_dataset_id): +def source_table_id( + bigquery_client: bigquery.Client, source_dataset_id: str +) -> Iterator[str]: source_table_id = f"{source_dataset_id}.us_states" job_config = bigquery.LoadJobConfig( schema=[ @@ -74,7 +81,13 @@ def source_table_id(bigquery_client, source_dataset_id): bigquery_client.delete_table(source_table_id, not_found_ok=True) -def test_view(capsys, view_id, view_dataset_id, source_table_id, source_dataset_id): +def test_view( + capsys: pytest.CaptureFixture[str], + view_id: str, + view_dataset_id: str, + source_table_id: str, + source_dataset_id: str, +) -> None: override_values = { "view_id": view_id, "source_id": source_table_id, @@ -99,7 +112,7 @@ def test_view(capsys, view_id, view_dataset_id, source_table_id, source_dataset_ assert view_id in out project_id, dataset_id, table_id = view_id.split(".") - override_values = { + overrides: view.OverridesDict = { "analyst_group_email": "cloud-dpes-bigquery@google.com", "view_dataset_id": view_dataset_id, "source_dataset_id": source_dataset_id, @@ -109,7 +122,7 @@ def test_view(capsys, view_id, view_dataset_id, source_table_id, source_dataset_ "tableId": table_id, }, } - view_dataset, source_dataset = view.grant_access(override_values) + view_dataset, source_dataset = view.grant_access(overrides) assert len(view_dataset.access_entries) != 0 assert len(source_dataset.access_entries) != 0 out, _ = capsys.readouterr() From 452f15e8cd1db4b6373c0a3977632e7b61fc4778 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Sat, 4 Dec 2021 17:59:56 +0200 Subject: [PATCH 11/26] Add type hints to top level samples (part 1) --- samples/create_dataset.py | 2 +- samples/create_job.py | 7 +++++- samples/create_routine.py | 7 +++++- samples/create_routine_ddl.py | 2 +- samples/create_table.py | 7 +++++- samples/create_table_clustered.py | 7 +++++- samples/create_table_range_partitioned.py | 7 +++++- samples/dataset_exists.py | 2 +- samples/delete_dataset.py | 2 +- samples/delete_dataset_labels.py | 7 +++++- samples/delete_model.py | 2 +- samples/delete_routine.py | 2 +- samples/delete_table.py | 2 +- samples/download_public_data.py | 2 +- samples/download_public_data_sandbox.py | 2 +- samples/get_dataset.py | 2 +- samples/get_dataset_labels.py | 2 +- samples/get_model.py | 2 +- samples/get_routine.py | 7 +++++- samples/get_table.py | 2 +- samples/label_dataset.py | 2 +- samples/list_datasets.py | 2 +- samples/list_datasets_by_label.py | 2 +- samples/list_models.py | 2 +- samples/list_routines.py | 2 +- samples/list_tables.py | 2 +- samples/load_table_clustered.py | 7 +++++- samples/load_table_dataframe.py | 7 +++++- samples/load_table_file.py | 7 +++++- samples/load_table_uri_autodetect_csv.py | 2 +- samples/load_table_uri_autodetect_json.py | 2 +- samples/load_table_uri_avro.py | 2 +- samples/load_table_uri_cmek.py | 2 +- samples/load_table_uri_csv.py | 2 +- samples/load_table_uri_json.py | 2 +- samples/load_table_uri_orc.py | 2 +- samples/load_table_uri_parquet.py | 2 +- samples/load_table_uri_truncate_avro.py | 2 +- samples/load_table_uri_truncate_csv.py | 2 +- samples/load_table_uri_truncate_json.py | 2 +- samples/load_table_uri_truncate_orc.py | 2 +- samples/load_table_uri_truncate_parquet.py | 2 +- samples/table_exists.py | 2 +- samples/table_insert_rows.py | 2 +- ...le_insert_rows_explicit_none_insert_ids.py | 2 +- samples/tests/conftest.py | 25 ++++++++++--------- samples/tests/test_dataset_exists.py | 11 +++++++- samples/tests/test_dataset_label_samples.py | 9 ++++++- samples/tests/test_get_dataset.py | 7 +++++- samples/tests/test_get_table.py | 9 ++++++- samples/tests/test_load_table_file.py | 7 +++++- samples/tests/test_load_table_uri_avro.py | 9 ++++++- samples/tests/test_load_table_uri_cmek.py | 9 ++++++- samples/tests/test_load_table_uri_csv.py | 9 ++++++- samples/tests/test_load_table_uri_json.py | 9 ++++++- samples/tests/test_load_table_uri_orc.py | 9 ++++++- samples/tests/test_load_table_uri_parquet.py | 9 ++++++- samples/tests/test_model_samples.py | 9 ++++++- samples/tests/test_query_no_cache.py | 6 ++++- samples/tests/test_query_pagination.py | 7 +++++- samples/tests/test_query_script.py | 7 +++++- samples/tests/test_query_to_arrow.py | 2 +- samples/tests/test_routine_samples.py | 2 +- samples/tests/test_table_exists.py | 9 ++++++- samples/tests/test_table_insert_rows.py | 9 ++++++- ...le_insert_rows_explicit_none_insert_ids.py | 9 ++++++- samples/tests/test_undelete_table.py | 11 +++++++- samples/undelete_table.py | 2 +- samples/update_model.py | 2 +- 69 files changed, 257 insertions(+), 80 deletions(-) diff --git a/samples/create_dataset.py b/samples/create_dataset.py index 6af3c67eb..dea91798d 100644 --- a/samples/create_dataset.py +++ b/samples/create_dataset.py @@ -13,7 +13,7 @@ # limitations under the License. -def create_dataset(dataset_id): +def create_dataset(dataset_id: str) -> None: # [START bigquery_create_dataset] from google.cloud import bigquery diff --git a/samples/create_job.py b/samples/create_job.py index feed04ca0..39922f7ae 100644 --- a/samples/create_job.py +++ b/samples/create_job.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing -def create_job(): +if typing.TYPE_CHECKING: + from google.cloud import bigquery + + +def create_job() -> "bigquery.QueryJob": # [START bigquery_create_job] from google.cloud import bigquery diff --git a/samples/create_routine.py b/samples/create_routine.py index b8746905d..96dc24210 100644 --- a/samples/create_routine.py +++ b/samples/create_routine.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing -def create_routine(routine_id): +if typing.TYPE_CHECKING: + from google.cloud import bigquery + + +def create_routine(routine_id: str) -> "bigquery.Routine": # [START bigquery_create_routine] from google.cloud import bigquery diff --git a/samples/create_routine_ddl.py b/samples/create_routine_ddl.py index c191bd385..56c7cfe24 100644 --- a/samples/create_routine_ddl.py +++ b/samples/create_routine_ddl.py @@ -13,7 +13,7 @@ # limitations under the License. -def create_routine_ddl(routine_id): +def create_routine_ddl(routine_id: str) -> None: # [START bigquery_create_routine_ddl] diff --git a/samples/create_table.py b/samples/create_table.py index d62e86681..7bbe4cef0 100644 --- a/samples/create_table.py +++ b/samples/create_table.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing -def create_table(table_id): +if typing.TYPE_CHECKING: + from google.cloud import bigquery + + +def create_table(table_id: str) -> "bigquery.Table": # [START bigquery_create_table] from google.cloud import bigquery diff --git a/samples/create_table_clustered.py b/samples/create_table_clustered.py index 2b45b747e..1686c519a 100644 --- a/samples/create_table_clustered.py +++ b/samples/create_table_clustered.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing -def create_table_clustered(table_id): +if typing.TYPE_CHECKING: + from google.cloud import bigquery + + +def create_table_clustered(table_id: str) -> "bigquery.Table": # [START bigquery_create_table_clustered] from google.cloud import bigquery diff --git a/samples/create_table_range_partitioned.py b/samples/create_table_range_partitioned.py index 260041aa5..2f2a19f71 100644 --- a/samples/create_table_range_partitioned.py +++ b/samples/create_table_range_partitioned.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing -def create_table_range_partitioned(table_id): +if typing.TYPE_CHECKING: + from google.cloud import bigquery + + +def create_table_range_partitioned(table_id: str) -> bigquery.Table: # [START bigquery_create_table_range_partitioned] from google.cloud import bigquery diff --git a/samples/dataset_exists.py b/samples/dataset_exists.py index b4db9353b..221899a65 100644 --- a/samples/dataset_exists.py +++ b/samples/dataset_exists.py @@ -13,7 +13,7 @@ # limitations under the License. -def dataset_exists(dataset_id): +def dataset_exists(dataset_id: str) -> None: # [START bigquery_dataset_exists] from google.cloud import bigquery diff --git a/samples/delete_dataset.py b/samples/delete_dataset.py index e25740baa..b340ed57a 100644 --- a/samples/delete_dataset.py +++ b/samples/delete_dataset.py @@ -13,7 +13,7 @@ # limitations under the License. -def delete_dataset(dataset_id): +def delete_dataset(dataset_id: str) -> None: # [START bigquery_delete_dataset] diff --git a/samples/delete_dataset_labels.py b/samples/delete_dataset_labels.py index a52de2967..27dea0079 100644 --- a/samples/delete_dataset_labels.py +++ b/samples/delete_dataset_labels.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing -def delete_dataset_labels(dataset_id): +if typing.TYPE_CHECKING: + from google.cloud import bigquery + + +def delete_dataset_labels(dataset_id: str) -> bigquery.Dataset: # [START bigquery_delete_label_dataset] diff --git a/samples/delete_model.py b/samples/delete_model.py index 0190315c6..2703ba3f5 100644 --- a/samples/delete_model.py +++ b/samples/delete_model.py @@ -13,7 +13,7 @@ # limitations under the License. -def delete_model(model_id): +def delete_model(model_id: str) -> None: """Sample ID: go/samples-tracker/1534""" # [START bigquery_delete_model] diff --git a/samples/delete_routine.py b/samples/delete_routine.py index 679cbee4b..7362a5fea 100644 --- a/samples/delete_routine.py +++ b/samples/delete_routine.py @@ -13,7 +13,7 @@ # limitations under the License. -def delete_routine(routine_id): +def delete_routine(routine_id: str) -> None: # [START bigquery_delete_routine] diff --git a/samples/delete_table.py b/samples/delete_table.py index 3d0a6f0ba..9e7ee170a 100644 --- a/samples/delete_table.py +++ b/samples/delete_table.py @@ -13,7 +13,7 @@ # limitations under the License. -def delete_table(table_id): +def delete_table(table_id: str) -> None: # [START bigquery_delete_table] diff --git a/samples/download_public_data.py b/samples/download_public_data.py index d10ed161a..a488bbbb5 100644 --- a/samples/download_public_data.py +++ b/samples/download_public_data.py @@ -13,7 +13,7 @@ # limitations under the License. -def download_public_data(): +def download_public_data() -> None: # [START bigquery_pandas_public_data] diff --git a/samples/download_public_data_sandbox.py b/samples/download_public_data_sandbox.py index afb50b15c..ce5200b4e 100644 --- a/samples/download_public_data_sandbox.py +++ b/samples/download_public_data_sandbox.py @@ -13,7 +13,7 @@ # limitations under the License. -def download_public_data_sandbox(): +def download_public_data_sandbox() -> None: # [START bigquery_pandas_public_data_sandbox] diff --git a/samples/get_dataset.py b/samples/get_dataset.py index 54ba05781..5654cbdce 100644 --- a/samples/get_dataset.py +++ b/samples/get_dataset.py @@ -13,7 +13,7 @@ # limitations under the License. -def get_dataset(dataset_id): +def get_dataset(dataset_id: str) -> None: # [START bigquery_get_dataset] diff --git a/samples/get_dataset_labels.py b/samples/get_dataset_labels.py index 18a9ca985..d97ee3c01 100644 --- a/samples/get_dataset_labels.py +++ b/samples/get_dataset_labels.py @@ -13,7 +13,7 @@ # limitations under the License. -def get_dataset_labels(dataset_id): +def get_dataset_labels(dataset_id: str) -> None: # [START bigquery_get_dataset_labels] diff --git a/samples/get_model.py b/samples/get_model.py index 1570ef816..dab4146ab 100644 --- a/samples/get_model.py +++ b/samples/get_model.py @@ -13,7 +13,7 @@ # limitations under the License. -def get_model(model_id): +def get_model(model_id: str) -> None: """Sample ID: go/samples-tracker/1510""" # [START bigquery_get_model] diff --git a/samples/get_routine.py b/samples/get_routine.py index 72715ee1b..031d9a127 100644 --- a/samples/get_routine.py +++ b/samples/get_routine.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing -def get_routine(routine_id): +if typing.TYPE_CHECKING: + from google.cloud import bigquery + + +def get_routine(routine_id: str) -> "bigquery.Routine": # [START bigquery_get_routine] diff --git a/samples/get_table.py b/samples/get_table.py index 0d1d809ba..6195aaf9a 100644 --- a/samples/get_table.py +++ b/samples/get_table.py @@ -13,7 +13,7 @@ # limitations under the License. -def get_table(table_id): +def get_table(table_id: str) -> None: # [START bigquery_get_table] diff --git a/samples/label_dataset.py b/samples/label_dataset.py index bd4cd6721..a59743e5d 100644 --- a/samples/label_dataset.py +++ b/samples/label_dataset.py @@ -13,7 +13,7 @@ # limitations under the License. -def label_dataset(dataset_id): +def label_dataset(dataset_id: str) -> None: # [START bigquery_label_dataset] diff --git a/samples/list_datasets.py b/samples/list_datasets.py index 6a1b93d00..c1b6639a9 100644 --- a/samples/list_datasets.py +++ b/samples/list_datasets.py @@ -13,7 +13,7 @@ # limitations under the License. -def list_datasets(): +def list_datasets() -> None: # [START bigquery_list_datasets] diff --git a/samples/list_datasets_by_label.py b/samples/list_datasets_by_label.py index 1b310049b..d1f264872 100644 --- a/samples/list_datasets_by_label.py +++ b/samples/list_datasets_by_label.py @@ -13,7 +13,7 @@ # limitations under the License. -def list_datasets_by_label(): +def list_datasets_by_label() -> None: # [START bigquery_list_datasets_by_label] diff --git a/samples/list_models.py b/samples/list_models.py index 7251c001a..df8ae0e1b 100644 --- a/samples/list_models.py +++ b/samples/list_models.py @@ -13,7 +13,7 @@ # limitations under the License. -def list_models(dataset_id): +def list_models(dataset_id: str) -> None: """Sample ID: go/samples-tracker/1512""" # [START bigquery_list_models] diff --git a/samples/list_routines.py b/samples/list_routines.py index 718d40d68..bee7c23be 100644 --- a/samples/list_routines.py +++ b/samples/list_routines.py @@ -13,7 +13,7 @@ # limitations under the License. -def list_routines(dataset_id): +def list_routines(dataset_id: str) -> None: # [START bigquery_list_routines] diff --git a/samples/list_tables.py b/samples/list_tables.py index 9ab527a49..df846961d 100644 --- a/samples/list_tables.py +++ b/samples/list_tables.py @@ -13,7 +13,7 @@ # limitations under the License. -def list_tables(dataset_id): +def list_tables(dataset_id: str) -> None: # [START bigquery_list_tables] diff --git a/samples/load_table_clustered.py b/samples/load_table_clustered.py index 20d412cb3..87b6c76ce 100644 --- a/samples/load_table_clustered.py +++ b/samples/load_table_clustered.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing -def load_table_clustered(table_id): +if typing.TYPE_CHECKING: + from google.cloud import bigquery + + +def load_table_clustered(table_id: str) -> "bigquery.Table": # [START bigquery_load_table_clustered] from google.cloud import bigquery diff --git a/samples/load_table_dataframe.py b/samples/load_table_dataframe.py index 91dd6e9f0..6cd06f266 100644 --- a/samples/load_table_dataframe.py +++ b/samples/load_table_dataframe.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing -def load_table_dataframe(table_id): +if typing.TYPE_CHECKING: + from google.cloud import bigquery + + +def load_table_dataframe(table_id: str) -> "bigquery.Table": # [START bigquery_load_table_dataframe] import datetime diff --git a/samples/load_table_file.py b/samples/load_table_file.py index b7e45dac3..81df368f0 100644 --- a/samples/load_table_file.py +++ b/samples/load_table_file.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing -def load_table_file(file_path, table_id): +if typing.TYPE_CHECKING: + from google.cloud import bigquery + + +def load_table_file(file_path: str, table_id: str) -> "bigquery.Table": # [START bigquery_load_from_file] from google.cloud import bigquery diff --git a/samples/load_table_uri_autodetect_csv.py b/samples/load_table_uri_autodetect_csv.py index 09a5d708d..c412c63f1 100644 --- a/samples/load_table_uri_autodetect_csv.py +++ b/samples/load_table_uri_autodetect_csv.py @@ -13,7 +13,7 @@ # limitations under the License. -def load_table_uri_autodetect_csv(table_id): +def load_table_uri_autodetect_csv(table_id: str) -> None: # [START bigquery_load_table_gcs_csv_autodetect] from google.cloud import bigquery diff --git a/samples/load_table_uri_autodetect_json.py b/samples/load_table_uri_autodetect_json.py index 61b7aab12..9d0bc3f22 100644 --- a/samples/load_table_uri_autodetect_json.py +++ b/samples/load_table_uri_autodetect_json.py @@ -13,7 +13,7 @@ # limitations under the License. -def load_table_uri_autodetect_json(table_id): +def load_table_uri_autodetect_json(table_id: str) -> None: # [START bigquery_load_table_gcs_json_autodetect] from google.cloud import bigquery diff --git a/samples/load_table_uri_avro.py b/samples/load_table_uri_avro.py index 5c25eed22..e9f7c39ed 100644 --- a/samples/load_table_uri_avro.py +++ b/samples/load_table_uri_avro.py @@ -13,7 +13,7 @@ # limitations under the License. -def load_table_uri_avro(table_id): +def load_table_uri_avro(table_id: str) -> None: # [START bigquery_load_table_gcs_avro] from google.cloud import bigquery diff --git a/samples/load_table_uri_cmek.py b/samples/load_table_uri_cmek.py index 8bd84993c..4dfc0d3b4 100644 --- a/samples/load_table_uri_cmek.py +++ b/samples/load_table_uri_cmek.py @@ -13,7 +13,7 @@ # limitations under the License. -def load_table_uri_cmek(table_id, kms_key_name): +def load_table_uri_cmek(table_id: str, kms_key_name: str) -> None: # [START bigquery_load_table_gcs_json_cmek] from google.cloud import bigquery diff --git a/samples/load_table_uri_csv.py b/samples/load_table_uri_csv.py index 0736a560c..9cb8c6f20 100644 --- a/samples/load_table_uri_csv.py +++ b/samples/load_table_uri_csv.py @@ -13,7 +13,7 @@ # limitations under the License. -def load_table_uri_csv(table_id): +def load_table_uri_csv(table_id: str) -> None: # [START bigquery_load_table_gcs_csv] from google.cloud import bigquery diff --git a/samples/load_table_uri_json.py b/samples/load_table_uri_json.py index 3c21972c8..409a83e8e 100644 --- a/samples/load_table_uri_json.py +++ b/samples/load_table_uri_json.py @@ -13,7 +13,7 @@ # limitations under the License. -def load_table_uri_json(table_id): +def load_table_uri_json(table_id: str) -> None: # [START bigquery_load_table_gcs_json] from google.cloud import bigquery diff --git a/samples/load_table_uri_orc.py b/samples/load_table_uri_orc.py index 3ab6ff45a..7babd2630 100644 --- a/samples/load_table_uri_orc.py +++ b/samples/load_table_uri_orc.py @@ -13,7 +13,7 @@ # limitations under the License. -def load_table_uri_orc(table_id): +def load_table_uri_orc(table_id: str) -> None: # [START bigquery_load_table_gcs_orc] from google.cloud import bigquery diff --git a/samples/load_table_uri_parquet.py b/samples/load_table_uri_parquet.py index 3dce5e8ef..6ea032f71 100644 --- a/samples/load_table_uri_parquet.py +++ b/samples/load_table_uri_parquet.py @@ -13,7 +13,7 @@ # limitations under the License. -def load_table_uri_parquet(table_id): +def load_table_uri_parquet(table_id: str) -> None: # [START bigquery_load_table_gcs_parquet] from google.cloud import bigquery diff --git a/samples/load_table_uri_truncate_avro.py b/samples/load_table_uri_truncate_avro.py index 1aa0aa49c..51c6636fa 100644 --- a/samples/load_table_uri_truncate_avro.py +++ b/samples/load_table_uri_truncate_avro.py @@ -13,7 +13,7 @@ # limitations under the License. -def load_table_uri_truncate_avro(table_id): +def load_table_uri_truncate_avro(table_id: str) -> None: # [START bigquery_load_table_gcs_avro_truncate] import io diff --git a/samples/load_table_uri_truncate_csv.py b/samples/load_table_uri_truncate_csv.py index 198cdc281..ee8b34043 100644 --- a/samples/load_table_uri_truncate_csv.py +++ b/samples/load_table_uri_truncate_csv.py @@ -13,7 +13,7 @@ # limitations under the License. -def load_table_uri_truncate_csv(table_id): +def load_table_uri_truncate_csv(table_id: str) -> None: # [START bigquery_load_table_gcs_csv_truncate] import io diff --git a/samples/load_table_uri_truncate_json.py b/samples/load_table_uri_truncate_json.py index d67d93e7b..e85e0808e 100644 --- a/samples/load_table_uri_truncate_json.py +++ b/samples/load_table_uri_truncate_json.py @@ -13,7 +13,7 @@ # limitations under the License. -def load_table_uri_truncate_json(table_id): +def load_table_uri_truncate_json(table_id: str) -> None: # [START bigquery_load_table_gcs_json_truncate] import io diff --git a/samples/load_table_uri_truncate_orc.py b/samples/load_table_uri_truncate_orc.py index 90543b791..c730099d1 100644 --- a/samples/load_table_uri_truncate_orc.py +++ b/samples/load_table_uri_truncate_orc.py @@ -13,7 +13,7 @@ # limitations under the License. -def load_table_uri_truncate_orc(table_id): +def load_table_uri_truncate_orc(table_id: str) -> None: # [START bigquery_load_table_gcs_orc_truncate] import io diff --git a/samples/load_table_uri_truncate_parquet.py b/samples/load_table_uri_truncate_parquet.py index e036fc180..3a0a55c8a 100644 --- a/samples/load_table_uri_truncate_parquet.py +++ b/samples/load_table_uri_truncate_parquet.py @@ -13,7 +13,7 @@ # limitations under the License. -def load_table_uri_truncate_parquet(table_id): +def load_table_uri_truncate_parquet(table_id: str) -> None: # [START bigquery_load_table_gcs_parquet_truncate] import io diff --git a/samples/table_exists.py b/samples/table_exists.py index 152d95534..6edba9239 100644 --- a/samples/table_exists.py +++ b/samples/table_exists.py @@ -13,7 +13,7 @@ # limitations under the License. -def table_exists(table_id): +def table_exists(table_id: str) -> None: # [START bigquery_table_exists] from google.cloud import bigquery diff --git a/samples/table_insert_rows.py b/samples/table_insert_rows.py index 24d739871..897133330 100644 --- a/samples/table_insert_rows.py +++ b/samples/table_insert_rows.py @@ -13,7 +13,7 @@ # limitations under the License. -def table_insert_rows(table_id): +def table_insert_rows(table_id: str) -> None: # [START bigquery_table_insert_rows] from google.cloud import bigquery diff --git a/samples/table_insert_rows_explicit_none_insert_ids.py b/samples/table_insert_rows_explicit_none_insert_ids.py index d91792b82..1ccb1acc4 100644 --- a/samples/table_insert_rows_explicit_none_insert_ids.py +++ b/samples/table_insert_rows_explicit_none_insert_ids.py @@ -13,7 +13,7 @@ # limitations under the License. -def table_insert_rows_explicit_none_insert_ids(table_id): +def table_insert_rows_explicit_none_insert_ids(table_id: str) -> None: # [START bigquery_table_insert_rows_explicit_none_insert_ids] from google.cloud import bigquery diff --git a/samples/tests/conftest.py b/samples/tests/conftest.py index 4764a571f..b7a2ad587 100644 --- a/samples/tests/conftest.py +++ b/samples/tests/conftest.py @@ -13,6 +13,7 @@ # limitations under the License. import datetime +from typing import Iterator import uuid import google.auth @@ -23,7 +24,7 @@ @pytest.fixture(scope="session", autouse=True) -def client(): +def client() -> bigquery.Client: credentials, project = google.auth.default( scopes=[ "https://www.googleapis.com/auth/drive", @@ -33,12 +34,12 @@ def client(): real_client = bigquery.Client(credentials=credentials, project=project) mock_client = mock.create_autospec(bigquery.Client) mock_client.return_value = real_client - bigquery.Client = mock_client + bigquery.Client = mock_client # type: ignore return real_client @pytest.fixture -def random_table_id(dataset_id): +def random_table_id(dataset_id: str) -> str: now = datetime.datetime.now() random_table_id = "example_table_{}_{}".format( now.strftime("%Y%m%d%H%M%S"), uuid.uuid4().hex[:8] @@ -47,7 +48,7 @@ def random_table_id(dataset_id): @pytest.fixture -def random_dataset_id(client): +def random_dataset_id(client: bigquery.Client) -> Iterator[str]: now = datetime.datetime.now() random_dataset_id = "example_dataset_{}_{}".format( now.strftime("%Y%m%d%H%M%S"), uuid.uuid4().hex[:8] @@ -57,7 +58,7 @@ def random_dataset_id(client): @pytest.fixture -def random_routine_id(dataset_id): +def random_routine_id(dataset_id: str) -> str: now = datetime.datetime.now() random_routine_id = "example_routine_{}_{}".format( now.strftime("%Y%m%d%H%M%S"), uuid.uuid4().hex[:8] @@ -66,7 +67,7 @@ def random_routine_id(dataset_id): @pytest.fixture -def dataset_id(client): +def dataset_id(client: bigquery.Client) -> Iterator[str]: now = datetime.datetime.now() dataset_id = "python_dataset_sample_{}_{}".format( now.strftime("%Y%m%d%H%M%S"), uuid.uuid4().hex[:8] @@ -77,7 +78,7 @@ def dataset_id(client): @pytest.fixture -def table_id(client, dataset_id): +def table_id(client: bigquery.Client, dataset_id: str) -> Iterator[str]: now = datetime.datetime.now() table_id = "python_table_sample_{}_{}".format( now.strftime("%Y%m%d%H%M%S"), uuid.uuid4().hex[:8] @@ -90,7 +91,7 @@ def table_id(client, dataset_id): @pytest.fixture -def table_with_schema_id(client, dataset_id): +def table_with_schema_id(client: bigquery.Client, dataset_id: str) -> Iterator[str]: now = datetime.datetime.now() table_id = "python_table_with_schema_{}_{}".format( now.strftime("%Y%m%d%H%M%S"), uuid.uuid4().hex[:8] @@ -106,12 +107,12 @@ def table_with_schema_id(client, dataset_id): @pytest.fixture -def table_with_data_id(): +def table_with_data_id() -> str: return "bigquery-public-data.samples.shakespeare" @pytest.fixture -def routine_id(client, dataset_id): +def routine_id(client: bigquery.Client, dataset_id: str) -> Iterator[str]: now = datetime.datetime.now() routine_id = "python_routine_sample_{}_{}".format( now.strftime("%Y%m%d%H%M%S"), uuid.uuid4().hex[:8] @@ -136,7 +137,7 @@ def routine_id(client, dataset_id): @pytest.fixture -def model_id(client, dataset_id): +def model_id(client: bigquery.Client, dataset_id: str) -> str: model_id = "{}.{}".format(dataset_id, uuid.uuid4().hex) # The only way to create a model resource is via SQL. @@ -162,5 +163,5 @@ def model_id(client, dataset_id): @pytest.fixture -def kms_key_name(): +def kms_key_name() -> str: return "projects/cloud-samples-tests/locations/us/keyRings/test/cryptoKeys/test" diff --git a/samples/tests/test_dataset_exists.py b/samples/tests/test_dataset_exists.py index 6bc38b4d2..bfef4368f 100644 --- a/samples/tests/test_dataset_exists.py +++ b/samples/tests/test_dataset_exists.py @@ -12,12 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from google.cloud import bigquery from .. import dataset_exists +if typing.TYPE_CHECKING: + import pytest + -def test_dataset_exists(capsys, random_dataset_id, client): +def test_dataset_exists( + capsys: "pytest.CaptureFixture[str]", + random_dataset_id: str, + client: bigquery.Client, +) -> None: dataset_exists.dataset_exists(random_dataset_id) out, err = capsys.readouterr() diff --git a/samples/tests/test_dataset_label_samples.py b/samples/tests/test_dataset_label_samples.py index 0dbb2a76b..75a024856 100644 --- a/samples/tests/test_dataset_label_samples.py +++ b/samples/tests/test_dataset_label_samples.py @@ -12,12 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import delete_dataset_labels from .. import get_dataset_labels from .. import label_dataset +if typing.TYPE_CHECKING: + import pytest + -def test_dataset_label_samples(capsys, dataset_id): +def test_dataset_label_samples( + capsys: "pytest.CaptureFixture[str]", dataset_id: str +) -> None: label_dataset.label_dataset(dataset_id) out, err = capsys.readouterr() diff --git a/samples/tests/test_get_dataset.py b/samples/tests/test_get_dataset.py index 3afdb00d3..97b30541b 100644 --- a/samples/tests/test_get_dataset.py +++ b/samples/tests/test_get_dataset.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import get_dataset +if typing.TYPE_CHECKING: + import pytest + -def test_get_dataset(capsys, dataset_id): +def test_get_dataset(capsys: "pytest.CaptureFixture[str]", dataset_id: str) -> None: get_dataset.get_dataset(dataset_id) out, err = capsys.readouterr() diff --git a/samples/tests/test_get_table.py b/samples/tests/test_get_table.py index 8bbd0681b..e6383010f 100644 --- a/samples/tests/test_get_table.py +++ b/samples/tests/test_get_table.py @@ -12,12 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from google.cloud import bigquery from .. import get_table +if typing.TYPE_CHECKING: + import pytest + -def test_get_table(capsys, random_table_id, client): +def test_get_table( + capsys: "pytest.CaptureFixture[str]", random_table_id: str, client: bigquery.Client +) -> None: schema = [ bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), diff --git a/samples/tests/test_load_table_file.py b/samples/tests/test_load_table_file.py index a7ebe7682..5be093881 100644 --- a/samples/tests/test_load_table_file.py +++ b/samples/tests/test_load_table_file.py @@ -13,14 +13,19 @@ # limitations under the License. import os +import typing from google.cloud import bigquery from .. import load_table_file +if typing.TYPE_CHECKING: + import pytest -def test_load_table_file(capsys, random_table_id, client): +def test_load_table_file( + capsys: "pytest.CaptureFixture[str]", random_table_id: str, client: bigquery.Client +): samples_test_dir = os.path.abspath(os.path.dirname(__file__)) file_path = os.path.join( samples_test_dir, "..", "..", "tests", "data", "people.csv" diff --git a/samples/tests/test_load_table_uri_avro.py b/samples/tests/test_load_table_uri_avro.py index 0be29d6b3..d0be44aca 100644 --- a/samples/tests/test_load_table_uri_avro.py +++ b/samples/tests/test_load_table_uri_avro.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import load_table_uri_avro +if typing.TYPE_CHECKING: + import pytest + -def test_load_table_uri_avro(capsys, random_table_id): +def test_load_table_uri_avro( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: load_table_uri_avro.load_table_uri_avro(random_table_id) out, _ = capsys.readouterr() assert "Loaded 50 rows." in out diff --git a/samples/tests/test_load_table_uri_cmek.py b/samples/tests/test_load_table_uri_cmek.py index c15dad9a7..1eb873843 100644 --- a/samples/tests/test_load_table_uri_cmek.py +++ b/samples/tests/test_load_table_uri_cmek.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import load_table_uri_cmek +if typing.TYPE_CHECKING: + import pytest + -def test_load_table_uri_cmek(capsys, random_table_id, kms_key_name): +def test_load_table_uri_cmek( + capsys: "pytest.CaptureFixture[str]", random_table_id: str, kms_key_name: str +) -> None: load_table_uri_cmek.load_table_uri_cmek(random_table_id, kms_key_name) out, _ = capsys.readouterr() diff --git a/samples/tests/test_load_table_uri_csv.py b/samples/tests/test_load_table_uri_csv.py index fbcc69358..a57224c84 100644 --- a/samples/tests/test_load_table_uri_csv.py +++ b/samples/tests/test_load_table_uri_csv.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import load_table_uri_csv +if typing.TYPE_CHECKING: + import pytest + -def test_load_table_uri_csv(capsys, random_table_id): +def test_load_table_uri_csv( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: load_table_uri_csv.load_table_uri_csv(random_table_id) out, _ = capsys.readouterr() diff --git a/samples/tests/test_load_table_uri_json.py b/samples/tests/test_load_table_uri_json.py index e054cb07a..3ad0ce29b 100644 --- a/samples/tests/test_load_table_uri_json.py +++ b/samples/tests/test_load_table_uri_json.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import load_table_uri_json +if typing.TYPE_CHECKING: + import pytest + -def test_load_table_uri_json(capsys, random_table_id): +def test_load_table_uri_json( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: load_table_uri_json.load_table_uri_json(random_table_id) out, _ = capsys.readouterr() diff --git a/samples/tests/test_load_table_uri_orc.py b/samples/tests/test_load_table_uri_orc.py index 96dc72022..f31e8cabb 100644 --- a/samples/tests/test_load_table_uri_orc.py +++ b/samples/tests/test_load_table_uri_orc.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import load_table_uri_orc +if typing.TYPE_CHECKING: + import pytest + -def test_load_table_uri_orc(capsys, random_table_id): +def test_load_table_uri_orc( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: load_table_uri_orc.load_table_uri_orc(random_table_id) out, _ = capsys.readouterr() diff --git a/samples/tests/test_load_table_uri_parquet.py b/samples/tests/test_load_table_uri_parquet.py index 81ba3fcef..5404e8584 100644 --- a/samples/tests/test_load_table_uri_parquet.py +++ b/samples/tests/test_load_table_uri_parquet.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import load_table_uri_parquet +if typing.TYPE_CHECKING: + import pytest + -def test_load_table_uri_json(capsys, random_table_id): +def test_load_table_uri_json( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: load_table_uri_parquet.load_table_uri_parquet(random_table_id) out, _ = capsys.readouterr() diff --git a/samples/tests/test_model_samples.py b/samples/tests/test_model_samples.py index ebefad846..ed82dd678 100644 --- a/samples/tests/test_model_samples.py +++ b/samples/tests/test_model_samples.py @@ -12,13 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import delete_model from .. import get_model from .. import list_models from .. import update_model +if typing.TYPE_CHECKING: + import pytest + -def test_model_samples(capsys, dataset_id, model_id): +def test_model_samples( + capsys: "pytest.CaptureFixture[str]", dataset_id: str, model_id: str +) -> None: """Since creating a model is a long operation, test all model samples in the same test, following a typical end-to-end flow. """ diff --git a/samples/tests/test_query_no_cache.py b/samples/tests/test_query_no_cache.py index df17d0d0b..f3fb039c9 100644 --- a/samples/tests/test_query_no_cache.py +++ b/samples/tests/test_query_no_cache.py @@ -13,11 +13,15 @@ # limitations under the License. import re +import typing from .. import query_no_cache +if typing.TYPE_CHECKING: + import pytest -def test_query_no_cache(capsys,): + +def test_query_no_cache(capsys: "pytest.CaptureFixture[str]") -> None: query_no_cache.query_no_cache() out, err = capsys.readouterr() diff --git a/samples/tests/test_query_pagination.py b/samples/tests/test_query_pagination.py index 7ab049c8c..daf711e49 100644 --- a/samples/tests/test_query_pagination.py +++ b/samples/tests/test_query_pagination.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import query_pagination +if typing.TYPE_CHECKING: + import pytest + -def test_query_pagination(capsys,): +def test_query_pagination(capsys: "pytest.CaptureFixture[str]") -> None: query_pagination.query_pagination() out, _ = capsys.readouterr() diff --git a/samples/tests/test_query_script.py b/samples/tests/test_query_script.py index 037664d36..98dd1253b 100644 --- a/samples/tests/test_query_script.py +++ b/samples/tests/test_query_script.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import query_script +if typing.TYPE_CHECKING: + import pytest + -def test_query_script(capsys,): +def test_query_script(capsys: "pytest.CaptureFixture[str]") -> None: query_script.query_script() out, _ = capsys.readouterr() diff --git a/samples/tests/test_query_to_arrow.py b/samples/tests/test_query_to_arrow.py index f14ce5561..d9b1aeb73 100644 --- a/samples/tests/test_query_to_arrow.py +++ b/samples/tests/test_query_to_arrow.py @@ -19,7 +19,7 @@ pyarrow = pytest.importorskip("pyarrow") -def test_query_to_arrow(capsys,): +def test_query_to_arrow(capsys: "pytest.CaptureFixture[str]") -> None: arrow_table = query_to_arrow.query_to_arrow() out, err = capsys.readouterr() diff --git a/samples/tests/test_routine_samples.py b/samples/tests/test_routine_samples.py index b457c464a..446758f1e 100644 --- a/samples/tests/test_routine_samples.py +++ b/samples/tests/test_routine_samples.py @@ -15,7 +15,7 @@ from google.cloud import bigquery -def test_create_routine(capsys, random_routine_id): +def test_create_routine(capsys, random_routine_id: str) -> None: from .. import create_routine create_routine.create_routine(random_routine_id) diff --git a/samples/tests/test_table_exists.py b/samples/tests/test_table_exists.py index d1f579a64..7317ba747 100644 --- a/samples/tests/test_table_exists.py +++ b/samples/tests/test_table_exists.py @@ -12,12 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from google.cloud import bigquery from .. import table_exists +if typing.TYPE_CHECKING: + import pytest + -def test_table_exists(capsys, random_table_id, client): +def test_table_exists( + capsys: "pytest.CaptureFixture[str]", random_table_id: str, client: bigquery.Client +) -> None: table_exists.table_exists(random_table_id) out, err = capsys.readouterr() diff --git a/samples/tests/test_table_insert_rows.py b/samples/tests/test_table_insert_rows.py index 72b51df9c..410137631 100644 --- a/samples/tests/test_table_insert_rows.py +++ b/samples/tests/test_table_insert_rows.py @@ -12,12 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from google.cloud import bigquery from .. import table_insert_rows +if typing.TYPE_CHECKING: + import pytest + -def test_table_insert_rows(capsys, random_table_id, client): +def test_table_insert_rows( + capsys: "pytest.CaptureFixture[str]", random_table_id: str, client: bigquery.Client, +) -> None: schema = [ bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), diff --git a/samples/tests/test_table_insert_rows_explicit_none_insert_ids.py b/samples/tests/test_table_insert_rows_explicit_none_insert_ids.py index c6199894a..00456ce84 100644 --- a/samples/tests/test_table_insert_rows_explicit_none_insert_ids.py +++ b/samples/tests/test_table_insert_rows_explicit_none_insert_ids.py @@ -12,12 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from google.cloud import bigquery from .. import table_insert_rows_explicit_none_insert_ids as mut +if typing.TYPE_CHECKING: + import pytest + -def test_table_insert_rows_explicit_none_insert_ids(capsys, random_table_id, client): +def test_table_insert_rows_explicit_none_insert_ids( + capsys: "pytest.CaptureFixture[str]", random_table_id: str, client: bigquery.Client +) -> None: schema = [ bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), diff --git a/samples/tests/test_undelete_table.py b/samples/tests/test_undelete_table.py index a070abdbd..08841ad72 100644 --- a/samples/tests/test_undelete_table.py +++ b/samples/tests/test_undelete_table.py @@ -12,10 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import undelete_table +if typing.TYPE_CHECKING: + import pytest + -def test_undelete_table(capsys, table_with_schema_id, random_table_id): +def test_undelete_table( + capsys: "pytest.CaptureFixture[str]", + table_with_schema_id: str, + random_table_id: str, +) -> None: undelete_table.undelete_table(table_with_schema_id, random_table_id) out, _ = capsys.readouterr() assert ( diff --git a/samples/undelete_table.py b/samples/undelete_table.py index 18b15801f..7c4bba134 100644 --- a/samples/undelete_table.py +++ b/samples/undelete_table.py @@ -15,7 +15,7 @@ from google.api_core import datetime_helpers -def undelete_table(table_id, recovered_table_id): +def undelete_table(table_id: str, recovered_table_id: str) -> None: # [START bigquery_undelete_table] import time diff --git a/samples/update_model.py b/samples/update_model.py index db262d8cc..e11b6d5af 100644 --- a/samples/update_model.py +++ b/samples/update_model.py @@ -13,7 +13,7 @@ # limitations under the License. -def update_model(model_id): +def update_model(model_id: str) -> None: """Sample ID: go/samples-tracker/1533""" # [START bigquery_update_model_description] From ee1179adb5d6fa1b12efc3086636ed000f1af16a Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Sun, 5 Dec 2021 11:44:28 +0200 Subject: [PATCH 12/26] Add type hints to top level samples (part 2) --- samples/add_empty_column.py | 2 +- samples/browse_table_data.py | 2 +- samples/client_list_jobs.py | 2 +- samples/client_load_partitioned_table.py | 2 +- samples/client_query.py | 2 +- samples/client_query_add_column.py | 2 +- samples/client_query_batch.py | 7 +++++- samples/client_query_destination_table.py | 2 +- ...lient_query_destination_table_clustered.py | 2 +- .../client_query_destination_table_cmek.py | 2 +- .../client_query_destination_table_legacy.py | 2 +- samples/client_query_dry_run.py | 7 +++++- samples/client_query_legacy_sql.py | 2 +- samples/client_query_relax_column.py | 2 +- samples/client_query_w_array_params.py | 2 +- samples/client_query_w_named_params.py | 2 +- samples/client_query_w_positional_params.py | 2 +- samples/client_query_w_struct_params.py | 2 +- samples/client_query_w_timestamp_params.py | 2 +- samples/copy_table.py | 2 +- samples/copy_table_cmek.py | 2 +- samples/copy_table_multiple_source.py | 4 ++- samples/query_external_gcs_temporary_table.py | 2 +- .../query_external_sheets_permanent_table.py | 2 +- .../query_external_sheets_temporary_table.py | 2 +- samples/query_no_cache.py | 2 +- samples/query_pagination.py | 2 +- samples/query_script.py | 2 +- samples/query_to_arrow.py | 7 +++++- samples/tests/test_add_empty_column.py | 7 +++++- samples/tests/test_browse_table_data.py | 9 ++++++- samples/tests/test_client_list_jobs.py | 10 +++++++- .../test_client_load_partitioned_table.py | 9 ++++++- samples/tests/test_client_query.py | 7 +++++- samples/tests/test_client_query_add_column.py | 9 ++++++- samples/tests/test_client_query_batch.py | 7 +++++- .../test_client_query_destination_table.py | 9 ++++++- ...lient_query_destination_table_clustered.py | 9 ++++++- ...est_client_query_destination_table_cmek.py | 9 ++++++- ...t_client_query_destination_table_legacy.py | 9 ++++++- samples/tests/test_client_query_dry_run.py | 7 +++++- samples/tests/test_client_query_legacy_sql.py | 6 ++++- .../tests/test_client_query_relax_column.py | 9 ++++++- .../tests/test_client_query_w_array_params.py | 7 +++++- .../tests/test_client_query_w_named_params.py | 7 +++++- .../test_client_query_w_positional_params.py | 7 +++++- .../test_client_query_w_struct_params.py | 7 +++++- .../test_client_query_w_timestamp_params.py | 7 +++++- samples/tests/test_copy_table.py | 13 +++++++++- samples/tests/test_copy_table_cmek.py | 12 ++++++++- .../tests/test_copy_table_multiple_source.py | 12 ++++++++- samples/tests/test_create_dataset.py | 9 ++++++- samples/tests/test_create_job.py | 10 +++++++- samples/tests/test_create_table.py | 9 ++++++- samples/tests/test_create_table_clustered.py | 9 ++++++- .../test_create_table_range_partitioned.py | 9 ++++++- samples/tests/test_delete_dataset.py | 7 +++++- samples/tests/test_delete_table.py | 7 +++++- samples/tests/test_download_public_data.py | 4 ++- .../test_download_public_data_sandbox.py | 4 ++- samples/tests/test_list_datasets.py | 10 +++++++- samples/tests/test_list_datasets_by_label.py | 10 +++++++- samples/tests/test_list_tables.py | 9 ++++++- samples/tests/test_load_table_clustered.py | 12 ++++++++- samples/tests/test_load_table_dataframe.py | 9 ++++++- samples/tests/test_load_table_file.py | 2 +- .../test_load_table_uri_autodetect_csv.py | 9 ++++++- .../test_load_table_uri_autodetect_json.py | 9 ++++++- .../test_load_table_uri_truncate_avro.py | 9 ++++++- .../tests/test_load_table_uri_truncate_csv.py | 9 ++++++- .../test_load_table_uri_truncate_json.py | 9 ++++++- .../tests/test_load_table_uri_truncate_orc.py | 9 ++++++- .../test_load_table_uri_truncate_parquet.py | 9 ++++++- ...test_query_external_gcs_temporary_table.py | 9 ++++++- ...t_query_external_sheets_permanent_table.py | 9 ++++++- ...t_query_external_sheets_temporary_table.py | 9 ++++++- samples/tests/test_routine_samples.py | 25 ++++++++++++++----- samples/tests/test_update_dataset_access.py | 9 ++++++- ...te_dataset_default_partition_expiration.py | 9 ++++++- ...update_dataset_default_table_expiration.py | 9 ++++++- .../tests/test_update_dataset_description.py | 9 ++++++- ...t_update_table_require_partition_filter.py | 9 ++++++- samples/update_dataset_access.py | 2 +- ...te_dataset_default_partition_expiration.py | 2 +- ...update_dataset_default_table_expiration.py | 2 +- samples/update_dataset_description.py | 2 +- samples/update_routine.py | 7 +++++- .../update_table_require_partition_filter.py | 2 +- 88 files changed, 467 insertions(+), 93 deletions(-) diff --git a/samples/add_empty_column.py b/samples/add_empty_column.py index cd7cf5018..6d449d6e2 100644 --- a/samples/add_empty_column.py +++ b/samples/add_empty_column.py @@ -13,7 +13,7 @@ # limitations under the License. -def add_empty_column(table_id): +def add_empty_column(table_id: str) -> None: # [START bigquery_add_empty_column] from google.cloud import bigquery diff --git a/samples/browse_table_data.py b/samples/browse_table_data.py index 29a1c2ff6..7b0ace7ac 100644 --- a/samples/browse_table_data.py +++ b/samples/browse_table_data.py @@ -13,7 +13,7 @@ # limitations under the License. -def browse_table_data(table_id): +def browse_table_data(table_id: str) -> None: # [START bigquery_browse_table] diff --git a/samples/client_list_jobs.py b/samples/client_list_jobs.py index b2344e23c..7f1e39cb8 100644 --- a/samples/client_list_jobs.py +++ b/samples/client_list_jobs.py @@ -13,7 +13,7 @@ # limitations under the License. -def client_list_jobs(): +def client_list_jobs() -> None: # [START bigquery_list_jobs] diff --git a/samples/client_load_partitioned_table.py b/samples/client_load_partitioned_table.py index e4e8a296c..9956f3f00 100644 --- a/samples/client_load_partitioned_table.py +++ b/samples/client_load_partitioned_table.py @@ -13,7 +13,7 @@ # limitations under the License. -def client_load_partitioned_table(table_id): +def client_load_partitioned_table(table_id: str) -> None: # [START bigquery_load_table_partitioned] from google.cloud import bigquery diff --git a/samples/client_query.py b/samples/client_query.py index 7fedc3f90..091d3f98b 100644 --- a/samples/client_query.py +++ b/samples/client_query.py @@ -13,7 +13,7 @@ # limitations under the License. -def client_query(): +def client_query() -> None: # [START bigquery_query] diff --git a/samples/client_query_add_column.py b/samples/client_query_add_column.py index ff7d5aa68..2da200bc5 100644 --- a/samples/client_query_add_column.py +++ b/samples/client_query_add_column.py @@ -13,7 +13,7 @@ # limitations under the License. -def client_query_add_column(table_id): +def client_query_add_column(table_id: str) -> None: # [START bigquery_add_column_query_append] from google.cloud import bigquery diff --git a/samples/client_query_batch.py b/samples/client_query_batch.py index e1680f4a1..3be97a780 100644 --- a/samples/client_query_batch.py +++ b/samples/client_query_batch.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing -def client_query_batch(): +if typing.TYPE_CHECKING: + from google.cloud import bigquery + + +def client_query_batch() -> "bigquery.QueryJob": # [START bigquery_query_batch] from google.cloud import bigquery diff --git a/samples/client_query_destination_table.py b/samples/client_query_destination_table.py index 303ce5a0c..b200f1cc6 100644 --- a/samples/client_query_destination_table.py +++ b/samples/client_query_destination_table.py @@ -13,7 +13,7 @@ # limitations under the License. -def client_query_destination_table(table_id): +def client_query_destination_table(table_id: str) -> None: # [START bigquery_query_destination_table] from google.cloud import bigquery diff --git a/samples/client_query_destination_table_clustered.py b/samples/client_query_destination_table_clustered.py index 5a109ed10..c4ab305f5 100644 --- a/samples/client_query_destination_table_clustered.py +++ b/samples/client_query_destination_table_clustered.py @@ -13,7 +13,7 @@ # limitations under the License. -def client_query_destination_table_clustered(table_id): +def client_query_destination_table_clustered(table_id: str) -> None: # [START bigquery_query_clustered_table] from google.cloud import bigquery diff --git a/samples/client_query_destination_table_cmek.py b/samples/client_query_destination_table_cmek.py index 24d4f2222..0fd44d189 100644 --- a/samples/client_query_destination_table_cmek.py +++ b/samples/client_query_destination_table_cmek.py @@ -13,7 +13,7 @@ # limitations under the License. -def client_query_destination_table_cmek(table_id, kms_key_name): +def client_query_destination_table_cmek(table_id: str, kms_key_name: str) -> None: # [START bigquery_query_destination_table_cmek] from google.cloud import bigquery diff --git a/samples/client_query_destination_table_legacy.py b/samples/client_query_destination_table_legacy.py index c8fdd606f..ee45d9a01 100644 --- a/samples/client_query_destination_table_legacy.py +++ b/samples/client_query_destination_table_legacy.py @@ -13,7 +13,7 @@ # limitations under the License. -def client_query_destination_table_legacy(table_id): +def client_query_destination_table_legacy(table_id: str) -> None: # [START bigquery_query_legacy_large_results] from google.cloud import bigquery diff --git a/samples/client_query_dry_run.py b/samples/client_query_dry_run.py index 1f7bd0c9c..418b43cb5 100644 --- a/samples/client_query_dry_run.py +++ b/samples/client_query_dry_run.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing -def client_query_dry_run(): +if typing.TYPE_CHECKING: + from google.cloud import bigquery + + +def client_query_dry_run() -> "bigquery.QueryJob": # [START bigquery_query_dry_run] from google.cloud import bigquery diff --git a/samples/client_query_legacy_sql.py b/samples/client_query_legacy_sql.py index 3f9465779..c054e1f28 100644 --- a/samples/client_query_legacy_sql.py +++ b/samples/client_query_legacy_sql.py @@ -13,7 +13,7 @@ # limitations under the License. -def client_query_legacy_sql(): +def client_query_legacy_sql() -> None: # [START bigquery_query_legacy] from google.cloud import bigquery diff --git a/samples/client_query_relax_column.py b/samples/client_query_relax_column.py index 5e2ec8056..c96a1e7aa 100644 --- a/samples/client_query_relax_column.py +++ b/samples/client_query_relax_column.py @@ -13,7 +13,7 @@ # limitations under the License. -def client_query_relax_column(table_id): +def client_query_relax_column(table_id: str) -> None: # [START bigquery_relax_column_query_append] from google.cloud import bigquery diff --git a/samples/client_query_w_array_params.py b/samples/client_query_w_array_params.py index 4077be2c7..669713182 100644 --- a/samples/client_query_w_array_params.py +++ b/samples/client_query_w_array_params.py @@ -13,7 +13,7 @@ # limitations under the License. -def client_query_w_array_params(): +def client_query_w_array_params() -> None: # [START bigquery_query_params_arrays] from google.cloud import bigquery diff --git a/samples/client_query_w_named_params.py b/samples/client_query_w_named_params.py index a0de8f63a..f42be1dc8 100644 --- a/samples/client_query_w_named_params.py +++ b/samples/client_query_w_named_params.py @@ -13,7 +13,7 @@ # limitations under the License. -def client_query_w_named_params(): +def client_query_w_named_params() -> None: # [START bigquery_query_params_named] from google.cloud import bigquery diff --git a/samples/client_query_w_positional_params.py b/samples/client_query_w_positional_params.py index ee316044b..b088b305e 100644 --- a/samples/client_query_w_positional_params.py +++ b/samples/client_query_w_positional_params.py @@ -13,7 +13,7 @@ # limitations under the License. -def client_query_w_positional_params(): +def client_query_w_positional_params() -> None: # [START bigquery_query_params_positional] from google.cloud import bigquery diff --git a/samples/client_query_w_struct_params.py b/samples/client_query_w_struct_params.py index 041a3a0e3..6c5b78113 100644 --- a/samples/client_query_w_struct_params.py +++ b/samples/client_query_w_struct_params.py @@ -13,7 +13,7 @@ # limitations under the License. -def client_query_w_struct_params(): +def client_query_w_struct_params() -> None: # [START bigquery_query_params_structs] from google.cloud import bigquery diff --git a/samples/client_query_w_timestamp_params.py b/samples/client_query_w_timestamp_params.py index 41a27770e..07d64cc94 100644 --- a/samples/client_query_w_timestamp_params.py +++ b/samples/client_query_w_timestamp_params.py @@ -13,7 +13,7 @@ # limitations under the License. -def client_query_w_timestamp_params(): +def client_query_w_timestamp_params() -> None: # [START bigquery_query_params_timestamps] import datetime diff --git a/samples/copy_table.py b/samples/copy_table.py index 91c58e109..8c6153fef 100644 --- a/samples/copy_table.py +++ b/samples/copy_table.py @@ -13,7 +13,7 @@ # limitations under the License. -def copy_table(source_table_id, destination_table_id): +def copy_table(source_table_id: str, destination_table_id: str) -> None: # [START bigquery_copy_table] diff --git a/samples/copy_table_cmek.py b/samples/copy_table_cmek.py index 52ccb5f7b..f2e8a90f9 100644 --- a/samples/copy_table_cmek.py +++ b/samples/copy_table_cmek.py @@ -13,7 +13,7 @@ # limitations under the License. -def copy_table_cmek(dest_table_id, orig_table_id, kms_key_name): +def copy_table_cmek(dest_table_id: str, orig_table_id: str, kms_key_name: str) -> None: # [START bigquery_copy_table_cmek] from google.cloud import bigquery diff --git a/samples/copy_table_multiple_source.py b/samples/copy_table_multiple_source.py index d86e380d0..1163b1664 100644 --- a/samples/copy_table_multiple_source.py +++ b/samples/copy_table_multiple_source.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Sequence -def copy_table_multiple_source(dest_table_id, table_ids): + +def copy_table_multiple_source(dest_table_id: str, table_ids: Sequence[str]) -> None: # [START bigquery_copy_table_multiple_source] diff --git a/samples/query_external_gcs_temporary_table.py b/samples/query_external_gcs_temporary_table.py index 3c3caf695..8b826e061 100644 --- a/samples/query_external_gcs_temporary_table.py +++ b/samples/query_external_gcs_temporary_table.py @@ -13,7 +13,7 @@ # limitations under the License. -def query_external_gcs_temporary_table(): +def query_external_gcs_temporary_table() -> None: # [START bigquery_query_external_gcs_temp] from google.cloud import bigquery diff --git a/samples/query_external_sheets_permanent_table.py b/samples/query_external_sheets_permanent_table.py index 31143d1b0..97a358014 100644 --- a/samples/query_external_sheets_permanent_table.py +++ b/samples/query_external_sheets_permanent_table.py @@ -13,7 +13,7 @@ # limitations under the License. -def query_external_sheets_permanent_table(dataset_id): +def query_external_sheets_permanent_table(dataset_id: str) -> None: # [START bigquery_query_external_sheets_perm] from google.cloud import bigquery diff --git a/samples/query_external_sheets_temporary_table.py b/samples/query_external_sheets_temporary_table.py index a9d58e388..07c183b3b 100644 --- a/samples/query_external_sheets_temporary_table.py +++ b/samples/query_external_sheets_temporary_table.py @@ -13,7 +13,7 @@ # limitations under the License. -def query_external_sheets_temporary_table(): +def query_external_sheets_temporary_table() -> None: # [START bigquery_query_external_sheets_temp] # [START bigquery_auth_drive_scope] diff --git a/samples/query_no_cache.py b/samples/query_no_cache.py index e380f0b15..f39c01dbc 100644 --- a/samples/query_no_cache.py +++ b/samples/query_no_cache.py @@ -13,7 +13,7 @@ # limitations under the License. -def query_no_cache(): +def query_no_cache() -> None: # [START bigquery_query_no_cache] from google.cloud import bigquery diff --git a/samples/query_pagination.py b/samples/query_pagination.py index 57a4212cf..2e1654050 100644 --- a/samples/query_pagination.py +++ b/samples/query_pagination.py @@ -13,7 +13,7 @@ # limitations under the License. -def query_pagination(): +def query_pagination() -> None: # [START bigquery_query_pagination] diff --git a/samples/query_script.py b/samples/query_script.py index 9390d352d..89ff55187 100644 --- a/samples/query_script.py +++ b/samples/query_script.py @@ -13,7 +13,7 @@ # limitations under the License. -def query_script(): +def query_script() -> None: # [START bigquery_query_script] from google.cloud import bigquery diff --git a/samples/query_to_arrow.py b/samples/query_to_arrow.py index 4a57992d1..157a93638 100644 --- a/samples/query_to_arrow.py +++ b/samples/query_to_arrow.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing -def query_to_arrow(): +if typing.TYPE_CHECKING: + import pyarrow + + +def query_to_arrow() -> "pyarrow.Table": # [START bigquery_query_to_arrow] diff --git a/samples/tests/test_add_empty_column.py b/samples/tests/test_add_empty_column.py index d89fcb6b7..5c7184766 100644 --- a/samples/tests/test_add_empty_column.py +++ b/samples/tests/test_add_empty_column.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import add_empty_column +if typing.TYPE_CHECKING: + import pytest + -def test_add_empty_column(capsys, table_id): +def test_add_empty_column(capsys: "pytest.CaptureFixture[str]", table_id: str) -> None: add_empty_column.add_empty_column(table_id) out, err = capsys.readouterr() diff --git a/samples/tests/test_browse_table_data.py b/samples/tests/test_browse_table_data.py index a5f647bdb..368e5cad6 100644 --- a/samples/tests/test_browse_table_data.py +++ b/samples/tests/test_browse_table_data.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import browse_table_data +if typing.TYPE_CHECKING: + import pytest + -def test_browse_table_data(capsys, table_with_data_id): +def test_browse_table_data( + capsys: "pytest.CaptureFixture[str]", table_with_data_id: str +) -> None: browse_table_data.browse_table_data(table_with_data_id) out, err = capsys.readouterr() diff --git a/samples/tests/test_client_list_jobs.py b/samples/tests/test_client_list_jobs.py index 896950a82..a2845b7ad 100644 --- a/samples/tests/test_client_list_jobs.py +++ b/samples/tests/test_client_list_jobs.py @@ -12,11 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import client_list_jobs from .. import create_job +if typing.TYPE_CHECKING: + from google.cloud import bigquery + import pytest + -def test_client_list_jobs(capsys, client): +def test_client_list_jobs( + capsys: "pytest.CaptureFixture[str]", client: "bigquery.Client" +) -> None: job = create_job.create_job() client.cancel_job(job.job_id) diff --git a/samples/tests/test_client_load_partitioned_table.py b/samples/tests/test_client_load_partitioned_table.py index f1d72a858..24f86c700 100644 --- a/samples/tests/test_client_load_partitioned_table.py +++ b/samples/tests/test_client_load_partitioned_table.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import client_load_partitioned_table +if typing.TYPE_CHECKING: + import pytest + -def test_client_load_partitioned_table(capsys, random_table_id): +def test_client_load_partitioned_table( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: client_load_partitioned_table.client_load_partitioned_table(random_table_id) out, err = capsys.readouterr() diff --git a/samples/tests/test_client_query.py b/samples/tests/test_client_query.py index 810c46a17..a8e3c343e 100644 --- a/samples/tests/test_client_query.py +++ b/samples/tests/test_client_query.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import client_query +if typing.TYPE_CHECKING: + import pytest + -def test_client_query(capsys,): +def test_client_query(capsys: "pytest.CaptureFixture[str]") -> None: client_query.client_query() out, err = capsys.readouterr() diff --git a/samples/tests/test_client_query_add_column.py b/samples/tests/test_client_query_add_column.py index 254533f78..1eb5a1ed6 100644 --- a/samples/tests/test_client_query_add_column.py +++ b/samples/tests/test_client_query_add_column.py @@ -12,12 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from google.cloud import bigquery from .. import client_query_add_column +if typing.TYPE_CHECKING: + import pytest + -def test_client_query_add_column(capsys, random_table_id, client): +def test_client_query_add_column( + capsys: "pytest.CaptureFixture[str]", random_table_id: str, client: bigquery.Client +) -> None: schema = [ bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), diff --git a/samples/tests/test_client_query_batch.py b/samples/tests/test_client_query_batch.py index c5e19985d..548fe3ac3 100644 --- a/samples/tests/test_client_query_batch.py +++ b/samples/tests/test_client_query_batch.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import client_query_batch +if typing.TYPE_CHECKING: + import pytest + -def test_client_query_batch(capsys,): +def test_client_query_batch(capsys: "pytest.CaptureFixture[str]") -> None: job = client_query_batch.client_query_batch() out, err = capsys.readouterr() diff --git a/samples/tests/test_client_query_destination_table.py b/samples/tests/test_client_query_destination_table.py index 6bcdd498a..067bc16ec 100644 --- a/samples/tests/test_client_query_destination_table.py +++ b/samples/tests/test_client_query_destination_table.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import client_query_destination_table +if typing.TYPE_CHECKING: + import pytest + -def test_client_query_destination_table(capsys, table_id): +def test_client_query_destination_table( + capsys: "pytest.CaptureFixture[str]", table_id: str +) -> None: client_query_destination_table.client_query_destination_table(table_id) out, err = capsys.readouterr() diff --git a/samples/tests/test_client_query_destination_table_clustered.py b/samples/tests/test_client_query_destination_table_clustered.py index b4bdd588c..02b131531 100644 --- a/samples/tests/test_client_query_destination_table_clustered.py +++ b/samples/tests/test_client_query_destination_table_clustered.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import client_query_destination_table_clustered +if typing.TYPE_CHECKING: + import pytest + -def test_client_query_destination_table_clustered(capsys, random_table_id): +def test_client_query_destination_table_clustered( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: client_query_destination_table_clustered.client_query_destination_table_clustered( random_table_id diff --git a/samples/tests/test_client_query_destination_table_cmek.py b/samples/tests/test_client_query_destination_table_cmek.py index 4f9e3bc9a..f2fe3bc39 100644 --- a/samples/tests/test_client_query_destination_table_cmek.py +++ b/samples/tests/test_client_query_destination_table_cmek.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import client_query_destination_table_cmek +if typing.TYPE_CHECKING: + import pytest + -def test_client_query_destination_table_cmek(capsys, random_table_id, kms_key_name): +def test_client_query_destination_table_cmek( + capsys: "pytest.CaptureFixture[str]", random_table_id: str, kms_key_name: str +) -> None: client_query_destination_table_cmek.client_query_destination_table_cmek( random_table_id, kms_key_name diff --git a/samples/tests/test_client_query_destination_table_legacy.py b/samples/tests/test_client_query_destination_table_legacy.py index 46077497b..0071ee4a4 100644 --- a/samples/tests/test_client_query_destination_table_legacy.py +++ b/samples/tests/test_client_query_destination_table_legacy.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import client_query_destination_table_legacy +if typing.TYPE_CHECKING: + import pytest + -def test_client_query_destination_table_legacy(capsys, random_table_id): +def test_client_query_destination_table_legacy( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: client_query_destination_table_legacy.client_query_destination_table_legacy( random_table_id diff --git a/samples/tests/test_client_query_dry_run.py b/samples/tests/test_client_query_dry_run.py index 5cbf2e3fa..cffb152ef 100644 --- a/samples/tests/test_client_query_dry_run.py +++ b/samples/tests/test_client_query_dry_run.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import client_query_dry_run +if typing.TYPE_CHECKING: + import pytest + -def test_client_query_dry_run(capsys,): +def test_client_query_dry_run(capsys: "pytest.CaptureFixture[str]") -> None: query_job = client_query_dry_run.client_query_dry_run() out, err = capsys.readouterr() diff --git a/samples/tests/test_client_query_legacy_sql.py b/samples/tests/test_client_query_legacy_sql.py index ab240fad1..b12b5a934 100644 --- a/samples/tests/test_client_query_legacy_sql.py +++ b/samples/tests/test_client_query_legacy_sql.py @@ -13,11 +13,15 @@ # limitations under the License. import re +import typing from .. import client_query_legacy_sql +if typing.TYPE_CHECKING: + import pytest -def test_client_query_legacy_sql(capsys,): + +def test_client_query_legacy_sql(capsys: "pytest.CaptureFixture[str]") -> None: client_query_legacy_sql.client_query_legacy_sql() out, err = capsys.readouterr() diff --git a/samples/tests/test_client_query_relax_column.py b/samples/tests/test_client_query_relax_column.py index 0c5b7aa6f..f910d61f0 100644 --- a/samples/tests/test_client_query_relax_column.py +++ b/samples/tests/test_client_query_relax_column.py @@ -12,12 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from google.cloud import bigquery from .. import client_query_relax_column +if typing.TYPE_CHECKING: + import pytest + -def test_client_query_relax_column(capsys, random_table_id, client): +def test_client_query_relax_column( + capsys: "pytest.CaptureFixture[str]", random_table_id: str, client: bigquery.Client, +) -> None: schema = [ bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), diff --git a/samples/tests/test_client_query_w_array_params.py b/samples/tests/test_client_query_w_array_params.py index 07e0294e9..fcd3f6972 100644 --- a/samples/tests/test_client_query_w_array_params.py +++ b/samples/tests/test_client_query_w_array_params.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import client_query_w_array_params +if typing.TYPE_CHECKING: + import pytest + -def test_client_query_w_array_params(capsys,): +def test_client_query_w_array_params(capsys: "pytest.CaptureFixture[str]") -> None: client_query_w_array_params.client_query_w_array_params() out, err = capsys.readouterr() diff --git a/samples/tests/test_client_query_w_named_params.py b/samples/tests/test_client_query_w_named_params.py index 2970dfdc4..85ef1dc4a 100644 --- a/samples/tests/test_client_query_w_named_params.py +++ b/samples/tests/test_client_query_w_named_params.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import client_query_w_named_params +if typing.TYPE_CHECKING: + import pytest + -def test_client_query_w_named_params(capsys,): +def test_client_query_w_named_params(capsys: "pytest.CaptureFixture[str]") -> None: client_query_w_named_params.client_query_w_named_params() out, err = capsys.readouterr() diff --git a/samples/tests/test_client_query_w_positional_params.py b/samples/tests/test_client_query_w_positional_params.py index e41ffa825..8ade676ab 100644 --- a/samples/tests/test_client_query_w_positional_params.py +++ b/samples/tests/test_client_query_w_positional_params.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import client_query_w_positional_params +if typing.TYPE_CHECKING: + import pytest + -def test_client_query_w_positional_params(capsys,): +def test_client_query_w_positional_params(capsys: "pytest.CaptureFixture[str]") -> None: client_query_w_positional_params.client_query_w_positional_params() out, err = capsys.readouterr() diff --git a/samples/tests/test_client_query_w_struct_params.py b/samples/tests/test_client_query_w_struct_params.py index 03083a3a7..3198dbad5 100644 --- a/samples/tests/test_client_query_w_struct_params.py +++ b/samples/tests/test_client_query_w_struct_params.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import client_query_w_struct_params +if typing.TYPE_CHECKING: + import pytest + -def test_client_query_w_struct_params(capsys,): +def test_client_query_w_struct_params(capsys: "pytest.CaptureFixture[str]") -> None: client_query_w_struct_params.client_query_w_struct_params() out, err = capsys.readouterr() diff --git a/samples/tests/test_client_query_w_timestamp_params.py b/samples/tests/test_client_query_w_timestamp_params.py index 9dddcb9a0..a3bbccdd4 100644 --- a/samples/tests/test_client_query_w_timestamp_params.py +++ b/samples/tests/test_client_query_w_timestamp_params.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import client_query_w_timestamp_params +if typing.TYPE_CHECKING: + import pytest + -def test_client_query_w_timestamp_params(capsys,): +def test_client_query_w_timestamp_params(capsys: "pytest.CaptureFixture[str]") -> None: client_query_w_timestamp_params.client_query_w_timestamp_params() out, err = capsys.readouterr() diff --git a/samples/tests/test_copy_table.py b/samples/tests/test_copy_table.py index 0b95c5443..7d77b1745 100644 --- a/samples/tests/test_copy_table.py +++ b/samples/tests/test_copy_table.py @@ -12,10 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import copy_table +if typing.TYPE_CHECKING: + import pytest + from google.cloud import bigquery + -def test_copy_table(capsys, table_with_data_id, random_table_id, client): +def test_copy_table( + capsys: "pytest.CaptureFixture[str]", + table_with_data_id: str, + random_table_id: str, + client: bigquery.Client, +) -> None: copy_table.copy_table(table_with_data_id, random_table_id) out, err = capsys.readouterr() diff --git a/samples/tests/test_copy_table_cmek.py b/samples/tests/test_copy_table_cmek.py index ac04675c9..061410b99 100644 --- a/samples/tests/test_copy_table_cmek.py +++ b/samples/tests/test_copy_table_cmek.py @@ -12,10 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import copy_table_cmek +if typing.TYPE_CHECKING: + import pytest + -def test_copy_table_cmek(capsys, random_table_id, table_with_data_id, kms_key_name): +def test_copy_table_cmek( + capsys: "pytest.CaptureFixture[str]", + random_table_id: str, + table_with_data_id: str, + kms_key_name: str, +) -> None: copy_table_cmek.copy_table_cmek(random_table_id, table_with_data_id, kms_key_name) out, err = capsys.readouterr() diff --git a/samples/tests/test_copy_table_multiple_source.py b/samples/tests/test_copy_table_multiple_source.py index 5bc4668b0..e8b27d2a9 100644 --- a/samples/tests/test_copy_table_multiple_source.py +++ b/samples/tests/test_copy_table_multiple_source.py @@ -13,12 +13,22 @@ # limitations under the License. import io +import typing + from google.cloud import bigquery from .. import copy_table_multiple_source +if typing.TYPE_CHECKING: + import pytest + -def test_copy_table_multiple_source(capsys, random_table_id, random_dataset_id, client): +def test_copy_table_multiple_source( + capsys: "pytest.CaptureFixture[str]", + random_table_id: str, + random_dataset_id: str, + client: bigquery.Client, +) -> None: dataset = bigquery.Dataset(random_dataset_id) dataset.location = "US" diff --git a/samples/tests/test_create_dataset.py b/samples/tests/test_create_dataset.py index a00003803..e7a897f8f 100644 --- a/samples/tests/test_create_dataset.py +++ b/samples/tests/test_create_dataset.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import create_dataset +if typing.TYPE_CHECKING: + import pytest + -def test_create_dataset(capsys, random_dataset_id): +def test_create_dataset( + capsys: "pytest.CaptureFixture[str]", random_dataset_id: str +) -> None: create_dataset.create_dataset(random_dataset_id) out, err = capsys.readouterr() diff --git a/samples/tests/test_create_job.py b/samples/tests/test_create_job.py index eab4b3e48..9e6621e91 100644 --- a/samples/tests/test_create_job.py +++ b/samples/tests/test_create_job.py @@ -12,10 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import create_job +if typing.TYPE_CHECKING: + import pytest + from google.cloud import bigquery + -def test_create_job(capsys, client): +def test_create_job( + capsys: "pytest.CaptureFixture[str]", client: "bigquery.Client" +) -> None: query_job = create_job.create_job() client.cancel_job(query_job.job_id, location=query_job.location) out, err = capsys.readouterr() diff --git a/samples/tests/test_create_table.py b/samples/tests/test_create_table.py index 48e52889a..98a0fa936 100644 --- a/samples/tests/test_create_table.py +++ b/samples/tests/test_create_table.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import create_table +if typing.TYPE_CHECKING: + import pytest + -def test_create_table(capsys, random_table_id): +def test_create_table( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: create_table.create_table(random_table_id) out, err = capsys.readouterr() assert "Created table {}".format(random_table_id) in out diff --git a/samples/tests/test_create_table_clustered.py b/samples/tests/test_create_table_clustered.py index 8eab5d48b..a3e483441 100644 --- a/samples/tests/test_create_table_clustered.py +++ b/samples/tests/test_create_table_clustered.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import create_table_clustered +if typing.TYPE_CHECKING: + import pytest + -def test_create_table_clustered(capsys, random_table_id): +def test_create_table_clustered( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: table = create_table_clustered.create_table_clustered(random_table_id) out, _ = capsys.readouterr() assert "Created clustered table {}".format(random_table_id) in out diff --git a/samples/tests/test_create_table_range_partitioned.py b/samples/tests/test_create_table_range_partitioned.py index 9745966bf..1c06b66fe 100644 --- a/samples/tests/test_create_table_range_partitioned.py +++ b/samples/tests/test_create_table_range_partitioned.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import create_table_range_partitioned +if typing.TYPE_CHECKING: + import pytest + -def test_create_table_range_partitioned(capsys, random_table_id): +def test_create_table_range_partitioned( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: table = create_table_range_partitioned.create_table_range_partitioned( random_table_id ) diff --git a/samples/tests/test_delete_dataset.py b/samples/tests/test_delete_dataset.py index 1f9b3c823..9347bf185 100644 --- a/samples/tests/test_delete_dataset.py +++ b/samples/tests/test_delete_dataset.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import delete_dataset +if typing.TYPE_CHECKING: + import pytest + -def test_delete_dataset(capsys, dataset_id): +def test_delete_dataset(capsys: "pytest.CaptureFixture[str]", dataset_id: str) -> None: delete_dataset.delete_dataset(dataset_id) out, err = capsys.readouterr() diff --git a/samples/tests/test_delete_table.py b/samples/tests/test_delete_table.py index 7065743b0..aca2df62f 100644 --- a/samples/tests/test_delete_table.py +++ b/samples/tests/test_delete_table.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import delete_table +if typing.TYPE_CHECKING: + import pytest + -def test_delete_table(capsys, table_id): +def test_delete_table(capsys: "pytest.CaptureFixture[str]", table_id: str) -> None: delete_table.delete_table(table_id) out, err = capsys.readouterr() diff --git a/samples/tests/test_download_public_data.py b/samples/tests/test_download_public_data.py index 2412c147f..02c2c6f9c 100644 --- a/samples/tests/test_download_public_data.py +++ b/samples/tests/test_download_public_data.py @@ -21,7 +21,9 @@ pytest.importorskip("google.cloud.bigquery_storage_v1") -def test_download_public_data(caplog, capsys): +def test_download_public_data( + caplog: pytest.LogCaptureFixture, capsys: pytest.CaptureFixture[str] +) -> None: # Enable debug-level logging to verify the BigQuery Storage API is used. caplog.set_level(logging.DEBUG) diff --git a/samples/tests/test_download_public_data_sandbox.py b/samples/tests/test_download_public_data_sandbox.py index 08e1aab73..e86f604ad 100644 --- a/samples/tests/test_download_public_data_sandbox.py +++ b/samples/tests/test_download_public_data_sandbox.py @@ -21,7 +21,9 @@ pytest.importorskip("google.cloud.bigquery_storage_v1") -def test_download_public_data_sandbox(caplog, capsys): +def test_download_public_data_sandbox( + caplog: pytest.LogCaptureFixture, capsys: pytest.CaptureFixture[str] +) -> None: # Enable debug-level logging to verify the BigQuery Storage API is used. caplog.set_level(logging.DEBUG) diff --git a/samples/tests/test_list_datasets.py b/samples/tests/test_list_datasets.py index 1610d0e4a..f51fe18f1 100644 --- a/samples/tests/test_list_datasets.py +++ b/samples/tests/test_list_datasets.py @@ -12,10 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import list_datasets +if typing.TYPE_CHECKING: + import pytest + from google.cloud import bigquery + -def test_list_datasets(capsys, dataset_id, client): +def test_list_datasets( + capsys: "pytest.CaptureFixture[str]", dataset_id: str, client: "bigquery.Client" +) -> None: list_datasets.list_datasets() out, err = capsys.readouterr() assert "Datasets in project {}:".format(client.project) in out diff --git a/samples/tests/test_list_datasets_by_label.py b/samples/tests/test_list_datasets_by_label.py index 5b375f4f4..ee6b9a999 100644 --- a/samples/tests/test_list_datasets_by_label.py +++ b/samples/tests/test_list_datasets_by_label.py @@ -12,10 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import list_datasets_by_label +if typing.TYPE_CHECKING: + import pytest + from google.cloud import bigquery + -def test_list_datasets_by_label(capsys, dataset_id, client): +def test_list_datasets_by_label( + capsys: "pytest.CaptureFixture[str]", dataset_id: str, client: "bigquery.Client" +) -> None: dataset = client.get_dataset(dataset_id) dataset.labels = {"color": "green"} dataset = client.update_dataset(dataset, ["labels"]) diff --git a/samples/tests/test_list_tables.py b/samples/tests/test_list_tables.py index f9426aa53..7c726accc 100644 --- a/samples/tests/test_list_tables.py +++ b/samples/tests/test_list_tables.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import list_tables +if typing.TYPE_CHECKING: + import pytest + -def test_list_tables(capsys, dataset_id, table_id): +def test_list_tables( + capsys: "pytest.CaptureFixture[str]", dataset_id: str, table_id: str +) -> None: list_tables.list_tables(dataset_id) out, err = capsys.readouterr() diff --git a/samples/tests/test_load_table_clustered.py b/samples/tests/test_load_table_clustered.py index bafdc2051..bbf3c671f 100644 --- a/samples/tests/test_load_table_clustered.py +++ b/samples/tests/test_load_table_clustered.py @@ -12,10 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import load_table_clustered +if typing.TYPE_CHECKING: + import pytest + from google.cloud import bigquery + -def test_load_table_clustered(capsys, random_table_id, client): +def test_load_table_clustered( + capsys: "pytest.CaptureFixture[str]", + random_table_id: str, + client: "bigquery.Client", +) -> None: table = load_table_clustered.load_table_clustered(random_table_id) diff --git a/samples/tests/test_load_table_dataframe.py b/samples/tests/test_load_table_dataframe.py index 777967959..152c82f8c 100644 --- a/samples/tests/test_load_table_dataframe.py +++ b/samples/tests/test_load_table_dataframe.py @@ -12,16 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + import pytest from .. import load_table_dataframe +if typing.TYPE_CHECKING: + from google.cloud import bigquery + pandas = pytest.importorskip("pandas") pyarrow = pytest.importorskip("pyarrow") -def test_load_table_dataframe(capsys, client, random_table_id): +def test_load_table_dataframe( + capsys: pytest.CaptureFixture[str], client: "bigquery.Client", random_table_id: str, +) -> None: table = load_table_dataframe.load_table_dataframe(random_table_id) out, _ = capsys.readouterr() diff --git a/samples/tests/test_load_table_file.py b/samples/tests/test_load_table_file.py index 5be093881..95b06c7f6 100644 --- a/samples/tests/test_load_table_file.py +++ b/samples/tests/test_load_table_file.py @@ -25,7 +25,7 @@ def test_load_table_file( capsys: "pytest.CaptureFixture[str]", random_table_id: str, client: bigquery.Client -): +) -> None: samples_test_dir = os.path.abspath(os.path.dirname(__file__)) file_path = os.path.join( samples_test_dir, "..", "..", "tests", "data", "people.csv" diff --git a/samples/tests/test_load_table_uri_autodetect_csv.py b/samples/tests/test_load_table_uri_autodetect_csv.py index a40719783..c9b410850 100644 --- a/samples/tests/test_load_table_uri_autodetect_csv.py +++ b/samples/tests/test_load_table_uri_autodetect_csv.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import load_table_uri_autodetect_csv +if typing.TYPE_CHECKING: + import pytest + -def test_load_table_uri_autodetect_csv(capsys, random_table_id): +def test_load_table_uri_autodetect_csv( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: load_table_uri_autodetect_csv.load_table_uri_autodetect_csv(random_table_id) out, err = capsys.readouterr() diff --git a/samples/tests/test_load_table_uri_autodetect_json.py b/samples/tests/test_load_table_uri_autodetect_json.py index df14d26ed..2c68a13db 100644 --- a/samples/tests/test_load_table_uri_autodetect_json.py +++ b/samples/tests/test_load_table_uri_autodetect_json.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import load_table_uri_autodetect_json +if typing.TYPE_CHECKING: + import pytest + -def test_load_table_uri_autodetect_csv(capsys, random_table_id): +def test_load_table_uri_autodetect_csv( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: load_table_uri_autodetect_json.load_table_uri_autodetect_json(random_table_id) out, err = capsys.readouterr() diff --git a/samples/tests/test_load_table_uri_truncate_avro.py b/samples/tests/test_load_table_uri_truncate_avro.py index ba680cabd..19b62fe7e 100644 --- a/samples/tests/test_load_table_uri_truncate_avro.py +++ b/samples/tests/test_load_table_uri_truncate_avro.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import load_table_uri_truncate_avro +if typing.TYPE_CHECKING: + import pytest + -def test_load_table_uri_truncate_avro(capsys, random_table_id): +def test_load_table_uri_truncate_avro( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: load_table_uri_truncate_avro.load_table_uri_truncate_avro(random_table_id) out, _ = capsys.readouterr() assert "Loaded 50 rows." in out diff --git a/samples/tests/test_load_table_uri_truncate_csv.py b/samples/tests/test_load_table_uri_truncate_csv.py index 5c1da7dce..9bc467cd0 100644 --- a/samples/tests/test_load_table_uri_truncate_csv.py +++ b/samples/tests/test_load_table_uri_truncate_csv.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import load_table_uri_truncate_csv +if typing.TYPE_CHECKING: + import pytest + -def test_load_table_uri_truncate_csv(capsys, random_table_id): +def test_load_table_uri_truncate_csv( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: load_table_uri_truncate_csv.load_table_uri_truncate_csv(random_table_id) out, _ = capsys.readouterr() assert "Loaded 50 rows." in out diff --git a/samples/tests/test_load_table_uri_truncate_json.py b/samples/tests/test_load_table_uri_truncate_json.py index 180ca7f40..cdf96454b 100644 --- a/samples/tests/test_load_table_uri_truncate_json.py +++ b/samples/tests/test_load_table_uri_truncate_json.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import load_table_uri_truncate_json +if typing.TYPE_CHECKING: + import pytest + -def test_load_table_uri_truncate_json(capsys, random_table_id): +def test_load_table_uri_truncate_json( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: load_table_uri_truncate_json.load_table_uri_truncate_json(random_table_id) out, _ = capsys.readouterr() assert "Loaded 50 rows." in out diff --git a/samples/tests/test_load_table_uri_truncate_orc.py b/samples/tests/test_load_table_uri_truncate_orc.py index 322bf3127..041923da9 100644 --- a/samples/tests/test_load_table_uri_truncate_orc.py +++ b/samples/tests/test_load_table_uri_truncate_orc.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import load_table_uri_truncate_orc +if typing.TYPE_CHECKING: + import pytest + -def test_load_table_uri_truncate_orc(capsys, random_table_id): +def test_load_table_uri_truncate_orc( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: load_table_uri_truncate_orc.load_table_uri_truncate_orc(random_table_id) out, _ = capsys.readouterr() assert "Loaded 50 rows." in out diff --git a/samples/tests/test_load_table_uri_truncate_parquet.py b/samples/tests/test_load_table_uri_truncate_parquet.py index ca901defa..2139f316f 100644 --- a/samples/tests/test_load_table_uri_truncate_parquet.py +++ b/samples/tests/test_load_table_uri_truncate_parquet.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import load_table_uri_truncate_parquet +if typing.TYPE_CHECKING: + import pytest + -def test_load_table_uri_truncate_parquet(capsys, random_table_id): +def test_load_table_uri_truncate_parquet( + capsys: "pytest.CaptureFixture[str]", random_table_id: str +) -> None: load_table_uri_truncate_parquet.load_table_uri_truncate_parquet(random_table_id) out, _ = capsys.readouterr() assert "Loaded 50 rows." in out diff --git a/samples/tests/test_query_external_gcs_temporary_table.py b/samples/tests/test_query_external_gcs_temporary_table.py index 022b327be..9590f3d7a 100644 --- a/samples/tests/test_query_external_gcs_temporary_table.py +++ b/samples/tests/test_query_external_gcs_temporary_table.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import query_external_gcs_temporary_table +if typing.TYPE_CHECKING: + import pytest + -def test_query_external_gcs_temporary_table(capsys,): +def test_query_external_gcs_temporary_table( + capsys: "pytest.CaptureFixture[str]", +) -> None: query_external_gcs_temporary_table.query_external_gcs_temporary_table() out, err = capsys.readouterr() diff --git a/samples/tests/test_query_external_sheets_permanent_table.py b/samples/tests/test_query_external_sheets_permanent_table.py index a00930cad..851839054 100644 --- a/samples/tests/test_query_external_sheets_permanent_table.py +++ b/samples/tests/test_query_external_sheets_permanent_table.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import query_external_sheets_permanent_table +if typing.TYPE_CHECKING: + import pytest + -def test_query_external_sheets_permanent_table(capsys, dataset_id): +def test_query_external_sheets_permanent_table( + capsys: "pytest.CaptureFixture[str]", dataset_id: str +) -> None: query_external_sheets_permanent_table.query_external_sheets_permanent_table( dataset_id diff --git a/samples/tests/test_query_external_sheets_temporary_table.py b/samples/tests/test_query_external_sheets_temporary_table.py index 8274787cb..58e0cb394 100644 --- a/samples/tests/test_query_external_sheets_temporary_table.py +++ b/samples/tests/test_query_external_sheets_temporary_table.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import query_external_sheets_temporary_table +if typing.TYPE_CHECKING: + import pytest + -def test_query_external_sheets_temporary_table(capsys): +def test_query_external_sheets_temporary_table( + capsys: "pytest.CaptureFixture[str]", +) -> None: query_external_sheets_temporary_table.query_external_sheets_temporary_table() out, err = capsys.readouterr() diff --git a/samples/tests/test_routine_samples.py b/samples/tests/test_routine_samples.py index 446758f1e..57bca074a 100644 --- a/samples/tests/test_routine_samples.py +++ b/samples/tests/test_routine_samples.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from google.cloud import bigquery +if typing.TYPE_CHECKING: + import pytest + -def test_create_routine(capsys, random_routine_id: str) -> None: +def test_create_routine( + capsys: "pytest.CaptureFixture[str]", random_routine_id: str +) -> None: from .. import create_routine create_routine.create_routine(random_routine_id) @@ -23,7 +30,11 @@ def test_create_routine(capsys, random_routine_id: str) -> None: assert "Created routine {}".format(random_routine_id) in out -def test_create_routine_ddl(capsys, random_routine_id, client): +def test_create_routine_ddl( + capsys: "pytest.CaptureFixture[str]", + random_routine_id: str, + client: bigquery.Client, +) -> None: from .. import create_routine_ddl create_routine_ddl.create_routine_ddl(random_routine_id) @@ -63,7 +74,9 @@ def test_create_routine_ddl(capsys, random_routine_id, client): assert routine.arguments == expected_arguments -def test_list_routines(capsys, dataset_id, routine_id): +def test_list_routines( + capsys: "pytest.CaptureFixture[str]", dataset_id: str, routine_id: str +) -> None: from .. import list_routines list_routines.list_routines(dataset_id) @@ -72,7 +85,7 @@ def test_list_routines(capsys, dataset_id, routine_id): assert routine_id in out -def test_get_routine(capsys, routine_id): +def test_get_routine(capsys: "pytest.CaptureFixture[str]", routine_id: str) -> None: from .. import get_routine get_routine.get_routine(routine_id) @@ -84,7 +97,7 @@ def test_get_routine(capsys, routine_id): assert "type_kind=" in out -def test_delete_routine(capsys, routine_id): +def test_delete_routine(capsys: "pytest.CaptureFixture[str]", routine_id: str) -> None: from .. import delete_routine delete_routine.delete_routine(routine_id) @@ -92,7 +105,7 @@ def test_delete_routine(capsys, routine_id): assert "Deleted routine {}.".format(routine_id) in out -def test_update_routine(routine_id): +def test_update_routine(routine_id: str) -> None: from .. import update_routine routine = update_routine.update_routine(routine_id) diff --git a/samples/tests/test_update_dataset_access.py b/samples/tests/test_update_dataset_access.py index 4c0aa835b..186a3b575 100644 --- a/samples/tests/test_update_dataset_access.py +++ b/samples/tests/test_update_dataset_access.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import update_dataset_access +if typing.TYPE_CHECKING: + import pytest + -def test_update_dataset_access(capsys, dataset_id): +def test_update_dataset_access( + capsys: "pytest.CaptureFixture[str]", dataset_id: str +) -> None: update_dataset_access.update_dataset_access(dataset_id) out, err = capsys.readouterr() diff --git a/samples/tests/test_update_dataset_default_partition_expiration.py b/samples/tests/test_update_dataset_default_partition_expiration.py index a5a8e6b52..b7787dde3 100644 --- a/samples/tests/test_update_dataset_default_partition_expiration.py +++ b/samples/tests/test_update_dataset_default_partition_expiration.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import update_dataset_default_partition_expiration +if typing.TYPE_CHECKING: + import pytest + -def test_update_dataset_default_partition_expiration(capsys, dataset_id): +def test_update_dataset_default_partition_expiration( + capsys: "pytest.CaptureFixture[str]", dataset_id: str +) -> None: ninety_days_ms = 90 * 24 * 60 * 60 * 1000 # in milliseconds diff --git a/samples/tests/test_update_dataset_default_table_expiration.py b/samples/tests/test_update_dataset_default_table_expiration.py index b0f701322..f780827f2 100644 --- a/samples/tests/test_update_dataset_default_table_expiration.py +++ b/samples/tests/test_update_dataset_default_table_expiration.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import update_dataset_default_table_expiration +if typing.TYPE_CHECKING: + import pytest + -def test_update_dataset_default_table_expiration(capsys, dataset_id): +def test_update_dataset_default_table_expiration( + capsys: "pytest.CaptureFixture[str]", dataset_id: str +) -> None: one_day_ms = 24 * 60 * 60 * 1000 # in milliseconds diff --git a/samples/tests/test_update_dataset_description.py b/samples/tests/test_update_dataset_description.py index e4ff586c7..5d1209e22 100644 --- a/samples/tests/test_update_dataset_description.py +++ b/samples/tests/test_update_dataset_description.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from .. import update_dataset_description +if typing.TYPE_CHECKING: + import pytest + -def test_update_dataset_description(capsys, dataset_id): +def test_update_dataset_description( + capsys: "pytest.CaptureFixture[str]", dataset_id: str +) -> None: update_dataset_description.update_dataset_description(dataset_id) out, err = capsys.readouterr() diff --git a/samples/tests/test_update_table_require_partition_filter.py b/samples/tests/test_update_table_require_partition_filter.py index 7e9ca6f2b..580796ed3 100644 --- a/samples/tests/test_update_table_require_partition_filter.py +++ b/samples/tests/test_update_table_require_partition_filter.py @@ -12,12 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + from google.cloud import bigquery from .. import update_table_require_partition_filter +if typing.TYPE_CHECKING: + import pytest + -def test_update_table_require_partition_filter(capsys, random_table_id, client): +def test_update_table_require_partition_filter( + capsys: "pytest.CaptureFixture[str]", random_table_id: str, client: bigquery.Client, +) -> None: # Make a partitioned table. schema = [bigquery.SchemaField("transaction_timestamp", "TIMESTAMP")] diff --git a/samples/update_dataset_access.py b/samples/update_dataset_access.py index a5c2670e7..fda784da5 100644 --- a/samples/update_dataset_access.py +++ b/samples/update_dataset_access.py @@ -13,7 +13,7 @@ # limitations under the License. -def update_dataset_access(dataset_id): +def update_dataset_access(dataset_id: str) -> None: # [START bigquery_update_dataset_access] from google.cloud import bigquery diff --git a/samples/update_dataset_default_partition_expiration.py b/samples/update_dataset_default_partition_expiration.py index 18cfb92db..37456f3a0 100644 --- a/samples/update_dataset_default_partition_expiration.py +++ b/samples/update_dataset_default_partition_expiration.py @@ -13,7 +13,7 @@ # limitations under the License. -def update_dataset_default_partition_expiration(dataset_id): +def update_dataset_default_partition_expiration(dataset_id: str) -> None: # [START bigquery_update_dataset_partition_expiration] diff --git a/samples/update_dataset_default_table_expiration.py b/samples/update_dataset_default_table_expiration.py index b7e5cea9b..cf6f50d9f 100644 --- a/samples/update_dataset_default_table_expiration.py +++ b/samples/update_dataset_default_table_expiration.py @@ -13,7 +13,7 @@ # limitations under the License. -def update_dataset_default_table_expiration(dataset_id): +def update_dataset_default_table_expiration(dataset_id: str) -> None: # [START bigquery_update_dataset_expiration] diff --git a/samples/update_dataset_description.py b/samples/update_dataset_description.py index 0732b1c61..98c5fed43 100644 --- a/samples/update_dataset_description.py +++ b/samples/update_dataset_description.py @@ -13,7 +13,7 @@ # limitations under the License. -def update_dataset_description(dataset_id): +def update_dataset_description(dataset_id: str) -> None: # [START bigquery_update_dataset_description] diff --git a/samples/update_routine.py b/samples/update_routine.py index 61c6855b5..1a975a253 100644 --- a/samples/update_routine.py +++ b/samples/update_routine.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing -def update_routine(routine_id): +if typing.TYPE_CHECKING: + from google.cloud import bigquery + + +def update_routine(routine_id: str) -> "bigquery.Routine": # [START bigquery_update_routine] diff --git a/samples/update_table_require_partition_filter.py b/samples/update_table_require_partition_filter.py index cf1d53277..8221238a7 100644 --- a/samples/update_table_require_partition_filter.py +++ b/samples/update_table_require_partition_filter.py @@ -13,7 +13,7 @@ # limitations under the License. -def update_table_require_partition_filter(table_id): +def update_table_require_partition_filter(table_id: str) -> None: # [START bigquery_update_table_require_partition_filter] From d77d70eedc74e48cc8f2b7ae7183c0d7d3d210e7 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 7 Dec 2021 18:33:38 +0200 Subject: [PATCH 13/26] Add type hints to top level samples (part 3) --- samples/browse_table_data.py | 12 +++++++----- samples/client_query_batch.py | 9 ++++++--- samples/create_table.py | 7 +------ samples/query_external_gcs_temporary_table.py | 3 ++- samples/query_external_sheets_permanent_table.py | 6 ++++-- samples/query_external_sheets_temporary_table.py | 6 ++++-- samples/undelete_table.py | 2 +- 7 files changed, 25 insertions(+), 20 deletions(-) diff --git a/samples/browse_table_data.py b/samples/browse_table_data.py index 7b0ace7ac..6a56253bf 100644 --- a/samples/browse_table_data.py +++ b/samples/browse_table_data.py @@ -41,15 +41,17 @@ def browse_table_data(table_id: str) -> None: table = client.get_table(table_id) # Make an API request. fields = table.schema[:2] # First two columns. rows_iter = client.list_rows(table_id, selected_fields=fields, max_results=10) - rows = list(rows_iter) print("Selected {} columns from table {}.".format(len(rows_iter.schema), table_id)) + + rows = list(rows_iter) print("Downloaded {} rows from table {}".format(len(rows), table_id)) # Print row data in tabular format. - rows = client.list_rows(table, max_results=10) - format_string = "{!s:<16} " * len(rows.schema) - field_names = [field.name for field in rows.schema] + rows_iter = client.list_rows(table, max_results=10) + format_string = "{!s:<16} " * len(rows_iter.schema) + field_names = [field.name for field in rows_iter.schema] print(format_string.format(*field_names)) # Prints column headers. - for row in rows: + + for row in rows_iter: print(format_string.format(*row)) # Prints row data. # [END bigquery_browse_table] diff --git a/samples/client_query_batch.py b/samples/client_query_batch.py index 3be97a780..df164d1be 100644 --- a/samples/client_query_batch.py +++ b/samples/client_query_batch.py @@ -42,9 +42,12 @@ def client_query_batch() -> "bigquery.QueryJob": # Check on the progress by getting the job's updated state. Once the state # is `DONE`, the results are ready. - query_job = client.get_job( - query_job.job_id, location=query_job.location - ) # Make an API request. + query_job = typing.cast( + "bigquery.QueryJob", + client.get_job( + query_job.job_id, location=query_job.location + ), # Make an API request. + ) print("Job {} is currently in state {}".format(query_job.job_id, query_job.state)) # [END bigquery_query_batch] diff --git a/samples/create_table.py b/samples/create_table.py index 7bbe4cef0..eaac54696 100644 --- a/samples/create_table.py +++ b/samples/create_table.py @@ -12,13 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import typing -if typing.TYPE_CHECKING: - from google.cloud import bigquery - - -def create_table(table_id: str) -> "bigquery.Table": +def create_table(table_id: str) -> None: # [START bigquery_create_table] from google.cloud import bigquery diff --git a/samples/query_external_gcs_temporary_table.py b/samples/query_external_gcs_temporary_table.py index 8b826e061..7f433dee5 100644 --- a/samples/query_external_gcs_temporary_table.py +++ b/samples/query_external_gcs_temporary_table.py @@ -16,6 +16,7 @@ def query_external_gcs_temporary_table() -> None: # [START bigquery_query_external_gcs_temp] + import typing from google.cloud import bigquery # Construct a BigQuery client object. @@ -30,7 +31,7 @@ def query_external_gcs_temporary_table() -> None: bigquery.SchemaField("name", "STRING"), bigquery.SchemaField("post_abbr", "STRING"), ] - external_config.options.skip_leading_rows = 1 + typing.cast(bigquery.CSVOptions, external_config.options).skip_leading_rows = 1 table_id = "us_states" job_config = bigquery.QueryJobConfig(table_definitions={table_id: external_config}) diff --git a/samples/query_external_sheets_permanent_table.py b/samples/query_external_sheets_permanent_table.py index 97a358014..81cde7415 100644 --- a/samples/query_external_sheets_permanent_table.py +++ b/samples/query_external_sheets_permanent_table.py @@ -16,6 +16,7 @@ def query_external_sheets_permanent_table(dataset_id: str) -> None: # [START bigquery_query_external_sheets_perm] + import typing from google.cloud import bigquery import google.auth @@ -56,8 +57,9 @@ def query_external_sheets_permanent_table(dataset_id: str) -> None: "/d/1i_QCL-7HcSyUZmIbP9E6lO_T5u3HnpLe7dnpHaijg_E/edit?usp=sharing" ) external_config.source_uris = [sheet_url] - external_config.options.skip_leading_rows = 1 # Optionally skip header row. - external_config.options.range = ( + options = typing.cast(bigquery.GoogleSheetsOptions, external_config.options) + options.skip_leading_rows = 1 # Optionally skip header row. + options.range = ( "us-states!A20:B49" # Optionally set range of the sheet to query from. ) table.external_data_configuration = external_config diff --git a/samples/query_external_sheets_temporary_table.py b/samples/query_external_sheets_temporary_table.py index 07c183b3b..b743f7cd4 100644 --- a/samples/query_external_sheets_temporary_table.py +++ b/samples/query_external_sheets_temporary_table.py @@ -17,6 +17,7 @@ def query_external_sheets_temporary_table() -> None: # [START bigquery_query_external_sheets_temp] # [START bigquery_auth_drive_scope] + import typing from google.cloud import bigquery import google.auth @@ -53,8 +54,9 @@ def query_external_sheets_temporary_table() -> None: bigquery.SchemaField("name", "STRING"), bigquery.SchemaField("post_abbr", "STRING"), ] - external_config.options.skip_leading_rows = 1 # Optionally skip header row. - external_config.options.range = ( + options = typing.cast(bigquery.GoogleSheetsOptions, external_config.options) + options.skip_leading_rows = 1 # Optionally skip header row. + options.range = ( "us-states!A20:B49" # Optionally set range of the sheet to query from. ) table_id = "us_states" diff --git a/samples/undelete_table.py b/samples/undelete_table.py index 7c4bba134..c230a9230 100644 --- a/samples/undelete_table.py +++ b/samples/undelete_table.py @@ -39,7 +39,7 @@ def undelete_table(table_id: str, recovered_table_id: str) -> None: # Due to very short lifecycle of the table, ensure we're not picking a time # prior to the table creation due to time drift between backend and client. table = client.get_table(table_id) - created_epoch = datetime_helpers.to_milliseconds(table.created) + created_epoch: int = datetime_helpers.to_milliseconds(table.created) # type: ignore if created_epoch > snapshot_epoch: snapshot_epoch = created_epoch # [END_EXCLUDE] From 819589434527682192254e92607dbb00e343455f Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 7 Dec 2021 18:57:35 +0200 Subject: [PATCH 14/26] Add type hints to core libraries for samples' needs Samples use the code from the core library, but mypy complains about calling an untyped function in a typed context, thus some additional type annotations need to be added. --- google/cloud/bigquery/client.py | 6 ++++-- google/cloud/bigquery/dataset.py | 6 +++--- google/cloud/bigquery/encryption_configuration.py | 2 +- google/cloud/bigquery/external_config.py | 8 ++++---- google/cloud/bigquery/job/copy_.py | 2 +- google/cloud/bigquery/job/load.py | 2 +- google/cloud/bigquery/job/query.py | 4 ++-- google/cloud/bigquery/query.py | 11 +++++++---- google/cloud/bigquery/routine/routine.py | 10 +++++----- google/cloud/bigquery/table.py | 14 +++++++------- 10 files changed, 35 insertions(+), 30 deletions(-) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 76ccafaf4..059ac4a53 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -215,7 +215,7 @@ def __init__( default_query_job_config=None, client_info=None, client_options=None, - ): + ) -> None: super(Client, self).__init__( project=project, credentials=credentials, @@ -3426,7 +3426,9 @@ def insert_rows_json( self, table: Union[Table, TableReference, TableListItem, str], json_rows: Sequence[Dict], - row_ids: Union[Iterable[str], AutoRowIDs, None] = AutoRowIDs.GENERATE_UUID, + row_ids: Union[ + Iterable[Optional[str]], AutoRowIDs, None + ] = AutoRowIDs.GENERATE_UUID, skip_invalid_rows: bool = None, ignore_unknown_values: bool = None, template_suffix: str = None, diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index ff015d605..9eabfb6f8 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -27,7 +27,7 @@ from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration -def _get_table_reference(self, table_id): +def _get_table_reference(self, table_id: str) -> TableReference: """Constructs a TableReference. Args: @@ -144,7 +144,7 @@ class AccessEntry(object): ) """Allowed entity types.""" - def __init__(self, role, entity_type, entity_id): + def __init__(self, role, entity_type, entity_id) -> None: if entity_type not in self.ENTITY_TYPES: message = "Entity type %r not among: %s" % ( entity_type, @@ -407,7 +407,7 @@ class Dataset(object): "default_encryption_configuration": "defaultEncryptionConfiguration", } - def __init__(self, dataset_ref): + def __init__(self, dataset_ref) -> None: if isinstance(dataset_ref, str): dataset_ref = DatasetReference.from_string(dataset_ref) self._properties = {"datasetReference": dataset_ref.to_api_repr(), "labels": {}} diff --git a/google/cloud/bigquery/encryption_configuration.py b/google/cloud/bigquery/encryption_configuration.py index ba04ae2c4..d0b6f3677 100644 --- a/google/cloud/bigquery/encryption_configuration.py +++ b/google/cloud/bigquery/encryption_configuration.py @@ -24,7 +24,7 @@ class EncryptionConfiguration(object): kms_key_name (str): resource ID of Cloud KMS key used for encryption """ - def __init__(self, kms_key_name=None): + def __init__(self, kms_key_name=None) -> None: self._properties = {} if kms_key_name is not None: self._properties["kmsKeyName"] = kms_key_name diff --git a/google/cloud/bigquery/external_config.py b/google/cloud/bigquery/external_config.py index e6f6a97c3..cabf2436b 100644 --- a/google/cloud/bigquery/external_config.py +++ b/google/cloud/bigquery/external_config.py @@ -22,7 +22,7 @@ import base64 import copy -from typing import FrozenSet, Iterable, Optional, Union +from typing import Any, Dict, FrozenSet, Iterable, Optional, Union from google.cloud.bigquery._helpers import _to_bytes from google.cloud.bigquery._helpers import _bytes_to_json @@ -572,8 +572,8 @@ class HivePartitioningOptions(object): https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#HivePartitioningOptions """ - def __init__(self): - self._properties = {} + def __init__(self) -> None: + self._properties: Dict[str, Any] = {} @property def mode(self): @@ -654,7 +654,7 @@ class ExternalConfig(object): See :attr:`source_format`. """ - def __init__(self, source_format): + def __init__(self, source_format) -> None: self._properties = {"sourceFormat": source_format} @property diff --git a/google/cloud/bigquery/job/copy_.py b/google/cloud/bigquery/job/copy_.py index f0dd3d668..29558c01f 100644 --- a/google/cloud/bigquery/job/copy_.py +++ b/google/cloud/bigquery/job/copy_.py @@ -52,7 +52,7 @@ class CopyJobConfig(_JobConfig): the property name as the name of a keyword argument. """ - def __init__(self, **kwargs): + def __init__(self, **kwargs) -> None: super(CopyJobConfig, self).__init__("copy", **kwargs) @property diff --git a/google/cloud/bigquery/job/load.py b/google/cloud/bigquery/job/load.py index b12c3e621..e9f8fe14a 100644 --- a/google/cloud/bigquery/job/load.py +++ b/google/cloud/bigquery/job/load.py @@ -38,7 +38,7 @@ class LoadJobConfig(_JobConfig): the property name as the name of a keyword argument. """ - def __init__(self, **kwargs): + def __init__(self, **kwargs) -> None: super(LoadJobConfig, self).__init__("load", **kwargs) @property diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py index 6b8b5ce12..61e860de5 100644 --- a/google/cloud/bigquery/job/query.py +++ b/google/cloud/bigquery/job/query.py @@ -231,7 +231,7 @@ class QueryJobConfig(_JobConfig): the property name as the name of a keyword argument. """ - def __init__(self, **kwargs): + def __init__(self, **kwargs) -> None: super(QueryJobConfig, self).__init__("query", **kwargs) @property @@ -1067,7 +1067,7 @@ def ddl_target_table(self): return prop @property - def num_dml_affected_rows(self): + def num_dml_affected_rows(self) -> Optional[int]: """Return the number of DML rows affected by the job. See: diff --git a/google/cloud/bigquery/query.py b/google/cloud/bigquery/query.py index c2230e0fa..ad7c60f7d 100644 --- a/google/cloud/bigquery/query.py +++ b/google/cloud/bigquery/query.py @@ -520,7 +520,7 @@ class ArrayQueryParameter(_AbstractQueryParameter): values (List[appropriate type]): The parameter array values. """ - def __init__(self, name, array_type, values): + def __init__(self, name, array_type, values) -> None: self.name = name self.values = values @@ -683,10 +683,13 @@ class StructQueryParameter(_AbstractQueryParameter): ]]): The sub-parameters for the struct """ - def __init__(self, name, *sub_params): + def __init__(self, name, *sub_params) -> None: self.name = name - types = self.struct_types = OrderedDict() - values = self.struct_values = {} + self.struct_types: Dict[str, Any] = OrderedDict() + self.struct_values: Dict[str, Any] = {} + + types = self.struct_types + values = self.struct_values for sub in sub_params: if isinstance(sub, self.__class__): types[sub.name] = "STRUCT" diff --git a/google/cloud/bigquery/routine/routine.py b/google/cloud/bigquery/routine/routine.py index 18a38c3cc..3c0919003 100644 --- a/google/cloud/bigquery/routine/routine.py +++ b/google/cloud/bigquery/routine/routine.py @@ -16,7 +16,7 @@ """Define resources for the BigQuery Routines API.""" -from typing import Optional +from typing import Any, Dict, Optional import google.cloud._helpers # type: ignore from google.cloud.bigquery import _helpers @@ -69,7 +69,7 @@ class Routine(object): "determinism_level": "determinismLevel", } - def __init__(self, routine_ref, **kwargs): + def __init__(self, routine_ref, **kwargs) -> None: if isinstance(routine_ref, str): routine_ref = RoutineReference.from_string(routine_ref) @@ -214,7 +214,7 @@ def return_type(self, value: StandardSqlDataType): self._properties[self._PROPERTY_TO_API_FIELD["return_type"]] = resource @property - def return_table_type(self) -> StandardSqlTableType: + def return_table_type(self) -> Optional[StandardSqlTableType]: """The return type of a Table Valued Function (TVF) routine. .. versionadded:: 2.22.0 @@ -352,8 +352,8 @@ class RoutineArgument(object): "mode": "mode", } - def __init__(self, **kwargs): - self._properties = {} + def __init__(self, **kwargs) -> None: + self._properties: Dict[str, Any] = {} for property_name in kwargs: setattr(self, property_name, kwargs[property_name]) diff --git a/google/cloud/bigquery/table.py b/google/cloud/bigquery/table.py index f434688e7..0f0716f9d 100644 --- a/google/cloud/bigquery/table.py +++ b/google/cloud/bigquery/table.py @@ -369,7 +369,7 @@ class Table(_TableBase): "require_partition_filter": "requirePartitionFilter", } - def __init__(self, table_ref, schema=None): + def __init__(self, table_ref, schema=None) -> None: table_ref = _table_arg_to_table_ref(table_ref) self._properties = {"tableReference": table_ref.to_api_repr(), "labels": {}} # Let the @property do validation. @@ -1318,7 +1318,7 @@ class Row(object): # Choose unusual field names to try to avoid conflict with schema fields. __slots__ = ("_xxx_values", "_xxx_field_to_index") - def __init__(self, values, field_to_index): + def __init__(self, values, field_to_index) -> None: self._xxx_values = values self._xxx_field_to_index = field_to_index @@ -2230,7 +2230,7 @@ class PartitionRange(object): Private. Used to construct object from API resource. """ - def __init__(self, start=None, end=None, interval=None, _properties=None): + def __init__(self, start=None, end=None, interval=None, _properties=None) -> None: if _properties is None: _properties = {} self._properties = _properties @@ -2305,10 +2305,10 @@ class RangePartitioning(object): Private. Used to construct object from API resource. """ - def __init__(self, range_=None, field=None, _properties=None): + def __init__(self, range_=None, field=None, _properties=None) -> None: if _properties is None: _properties = {} - self._properties = _properties + self._properties: Dict[str, Any] = _properties if range_ is not None: self.range_ = range_ @@ -2414,8 +2414,8 @@ class TimePartitioning(object): def __init__( self, type_=None, field=None, expiration_ms=None, require_partition_filter=None - ): - self._properties = {} + ) -> None: + self._properties: Dict[str, Any] = {} if type_ is None: self.type_ = TimePartitioningType.DAY else: From 6a3713b261310937c5ba7e5bdc8d5c1181d83981 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 8 Dec 2021 16:14:12 +0200 Subject: [PATCH 15/26] Add top level mypy config and samples session --- noxfile.py | 22 ++++++++++++++++++++++ samples/mypy.ini | 12 ++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 samples/mypy.ini diff --git a/noxfile.py b/noxfile.py index 505911861..d60f56205 100644 --- a/noxfile.py +++ b/noxfile.py @@ -43,6 +43,7 @@ "lint_setup_py", "blacken", "mypy", + "mypy_samples", "pytype", "docs", ] @@ -186,6 +187,27 @@ def system(session): session.run("py.test", "--quiet", os.path.join("tests", "system"), *session.posargs) +@nox.session(python=DEFAULT_PYTHON_VERSION) +def mypy_samples(session): + """Run type checks with mypy.""" + session.install("-e", ".[all]") + + session.install("ipython", "pytest") + session.install(MYPY_VERSION) + + # Just install the dependencies' type info directly, since "mypy --install-types" + # might require an additional pass. + session.install("types-mock", "types-pytz") + + session.run( + "mypy", + "--config-file", + str(CURRENT_DIRECTORY / "samples" / "mypy.ini"), + "--no-incremental", # Required by warn-unused-configs from mypy.ini to work + "samples/", + ) + + @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) def snippets(session): """Run the snippets test suite.""" diff --git a/samples/mypy.ini b/samples/mypy.ini new file mode 100644 index 000000000..29757e47d --- /dev/null +++ b/samples/mypy.ini @@ -0,0 +1,12 @@ +[mypy] +# Should match DEFAULT_PYTHON_VERSION from root noxfile.py +python_version = 3.8 +exclude = noxfile\.py +strict = True +warn_unused_configs = True + +[mypy-google.auth,google.oauth2,geojson,google_auth_oauthlib,IPython.*] +ignore_missing_imports = True + +[mypy-pandas,pyarrow,shapely.*,test_utils.*] +ignore_missing_imports = True From c4d0ab549341f991a7fcfb180f534549c2662c19 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 8 Dec 2021 17:12:30 +0200 Subject: [PATCH 16/26] Add mypy.ini files to samples subdirectories --- samples/geography/mypy.ini | 8 ++++++++ samples/magics/mypy.ini | 8 ++++++++ samples/snippets/mypy.ini | 8 ++++++++ 3 files changed, 24 insertions(+) create mode 100644 samples/geography/mypy.ini create mode 100644 samples/magics/mypy.ini create mode 100644 samples/snippets/mypy.ini diff --git a/samples/geography/mypy.ini b/samples/geography/mypy.ini new file mode 100644 index 000000000..41898432f --- /dev/null +++ b/samples/geography/mypy.ini @@ -0,0 +1,8 @@ +[mypy] +; We require type annotations in all samples. +strict = True +exclude = noxfile\.py +warn_unused_configs = True + +[mypy-geojson,pandas,shapely.*] +ignore_missing_imports = True diff --git a/samples/magics/mypy.ini b/samples/magics/mypy.ini new file mode 100644 index 000000000..af328dc5e --- /dev/null +++ b/samples/magics/mypy.ini @@ -0,0 +1,8 @@ +[mypy] +; We require type annotations in all samples. +strict = True +exclude = noxfile\.py +warn_unused_configs = True + +[mypy-IPython.*,nox,noxfile_config,pandas] +ignore_missing_imports = True diff --git a/samples/snippets/mypy.ini b/samples/snippets/mypy.ini new file mode 100644 index 000000000..3cc4b8965 --- /dev/null +++ b/samples/snippets/mypy.ini @@ -0,0 +1,8 @@ +[mypy] +; We require type annotations in all samples. +strict = True +exclude = noxfile\.py +warn_unused_configs = True + +[mypy-google.auth,google.oauth2,google_auth_oauthlib,IPython.*,test_utils.*] +ignore_missing_imports = True From c0ef9a07d9de1fb934a848c549e3ae8853ae29e9 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 8 Dec 2021 17:44:36 +0200 Subject: [PATCH 17/26] Enforce late evaluation of some annotations --- samples/create_table_range_partitioned.py | 2 +- samples/delete_dataset_labels.py | 2 +- samples/tests/test_copy_table.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/create_table_range_partitioned.py b/samples/create_table_range_partitioned.py index 2f2a19f71..4dc45ed58 100644 --- a/samples/create_table_range_partitioned.py +++ b/samples/create_table_range_partitioned.py @@ -18,7 +18,7 @@ from google.cloud import bigquery -def create_table_range_partitioned(table_id: str) -> bigquery.Table: +def create_table_range_partitioned(table_id: str) -> "bigquery.Table": # [START bigquery_create_table_range_partitioned] from google.cloud import bigquery diff --git a/samples/delete_dataset_labels.py b/samples/delete_dataset_labels.py index 27dea0079..ec5df09c1 100644 --- a/samples/delete_dataset_labels.py +++ b/samples/delete_dataset_labels.py @@ -18,7 +18,7 @@ from google.cloud import bigquery -def delete_dataset_labels(dataset_id: str) -> bigquery.Dataset: +def delete_dataset_labels(dataset_id: str) -> "bigquery.Dataset": # [START bigquery_delete_label_dataset] diff --git a/samples/tests/test_copy_table.py b/samples/tests/test_copy_table.py index 7d77b1745..64fbdd778 100644 --- a/samples/tests/test_copy_table.py +++ b/samples/tests/test_copy_table.py @@ -25,7 +25,7 @@ def test_copy_table( capsys: "pytest.CaptureFixture[str]", table_with_data_id: str, random_table_id: str, - client: bigquery.Client, + client: "bigquery.Client", ) -> None: copy_table.copy_table(table_with_data_id, random_table_id) From 86dd3897b3e2334090a6b74d4ac5073359852801 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 8 Dec 2021 17:56:39 +0200 Subject: [PATCH 18/26] Install typing.TypedDict backport --- noxfile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/noxfile.py b/noxfile.py index d60f56205..c6197bfed 100644 --- a/noxfile.py +++ b/noxfile.py @@ -198,6 +198,7 @@ def mypy_samples(session): # Just install the dependencies' type info directly, since "mypy --install-types" # might require an additional pass. session.install("types-mock", "types-pytz") + session.install("typing-extensions") # for TypedDict in pre-3.8 Python versions session.run( "mypy", From 278dac7c16a5f4248f1f68d39a1b7673d2deef9f Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 8 Dec 2021 18:01:40 +0200 Subject: [PATCH 19/26] Make sure typing.TypedDict is available in samples --- samples/magics/requirements.txt | 1 + samples/snippets/requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index 0d36904c4..70b7936cb 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -10,3 +10,4 @@ pandas==1.1.5; python_version < '3.7' pandas==1.3.4; python_version >= '3.7' pyarrow==6.0.0 pytz==2021.1 +typing-extensions==3.10.0.2 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 4f04611ba..32ac9fbe9 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -10,3 +10,4 @@ pandas==1.1.5; python_version < '3.7' pandas==1.3.4; python_version >= '3.7' pyarrow==6.0.0 pytz==2021.1 +typing-extensions==3.10.0.2 From 2c78c7267b008ad943c42a7a36f8509680bcf12a Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 8 Dec 2021 22:35:26 +0200 Subject: [PATCH 20/26] Use import fallback for TypedDict --- samples/snippets/view.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/samples/snippets/view.py b/samples/snippets/view.py index 443c5b75e..5e976f68a 100644 --- a/samples/snippets/view.py +++ b/samples/snippets/view.py @@ -13,7 +13,12 @@ # limitations under the License. import typing -from typing import Dict, Optional, Tuple, TypedDict +from typing import Dict, Optional, Tuple + +try: + from typing import TypedDict +except ImportError: + from typing_extensions import TypedDict if typing.TYPE_CHECKING: from google.cloud import bigquery From 12e118d664fdbb4aba4b0a3c8425e87b8b758356 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 8 Dec 2021 23:09:16 +0200 Subject: [PATCH 21/26] Fix service account fixture in snippets --- samples/snippets/authenticate_service_account_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/snippets/authenticate_service_account_test.py b/samples/snippets/authenticate_service_account_test.py index 0c4d582c7..4b5711f80 100644 --- a/samples/snippets/authenticate_service_account_test.py +++ b/samples/snippets/authenticate_service_account_test.py @@ -13,6 +13,7 @@ # limitations under the License. import typing +from typing import Any import google.auth @@ -22,7 +23,7 @@ import pytest -def mock_credentials() -> google.auth.credentials.Credentials: +def mock_credentials(*args: Any, **kwargs: Any) -> google.auth.credentials.Credentials: credentials, _ = google.auth.default( ["https://www.googleapis.com/auth/cloud-platform"] ) From 44221f437f93fae2fac10d074b39a28008b9ad01 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 9 Dec 2021 15:46:17 +0000 Subject: [PATCH 22/26] chore: update python-docs-samples link to main branch (#1082) --- .github/.OwlBot.lock.yaml | 2 +- samples/AUTHORING_GUIDE.md | 2 +- samples/CONTRIBUTING.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 108063d4d..0b3c8cd98 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:4ee57a76a176ede9087c14330c625a71553cf9c72828b2c0ca12f5338171ba60 + digest: sha256:2f90537dd7df70f6b663cd654b1fa5dee483cf6a4edcfd46072b2775be8a23ec diff --git a/samples/AUTHORING_GUIDE.md b/samples/AUTHORING_GUIDE.md index 55c97b32f..8249522ff 100644 --- a/samples/AUTHORING_GUIDE.md +++ b/samples/AUTHORING_GUIDE.md @@ -1 +1 @@ -See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/AUTHORING_GUIDE.md \ No newline at end of file +See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/AUTHORING_GUIDE.md \ No newline at end of file diff --git a/samples/CONTRIBUTING.md b/samples/CONTRIBUTING.md index 34c882b6f..f5fe2e6ba 100644 --- a/samples/CONTRIBUTING.md +++ b/samples/CONTRIBUTING.md @@ -1 +1 @@ -See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/CONTRIBUTING.md \ No newline at end of file +See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/CONTRIBUTING.md \ No newline at end of file From 040a21ce260e9ecdfbf0ec2b8c4257f14c29d1b5 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 14 Dec 2021 12:14:46 +0200 Subject: [PATCH 23/26] Add missing annotations to new samples --- samples/snippets/dataset_access_test.py | 13 ++++++++++++- samples/snippets/revoke_dataset_access.py | 2 +- samples/snippets/update_dataset_access.py | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/samples/snippets/dataset_access_test.py b/samples/snippets/dataset_access_test.py index 21776c149..4d1a70eb1 100644 --- a/samples/snippets/dataset_access_test.py +++ b/samples/snippets/dataset_access_test.py @@ -12,11 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + import revoke_dataset_access import update_dataset_access +if typing.TYPE_CHECKING: + import pytest + from google.cloud import bigquery + -def test_dataset_access_permissions(capsys, dataset_id, entity_id, bigquery_client): +def test_dataset_access_permissions( + capsys: "pytest.CaptureFixture[str]", + dataset_id: str, + entity_id: str, + bigquery_client: "bigquery.Client", +) -> None: original_dataset = bigquery_client.get_dataset(dataset_id) update_dataset_access.update_dataset_access(dataset_id, entity_id) full_dataset_id = "{}.{}".format( diff --git a/samples/snippets/revoke_dataset_access.py b/samples/snippets/revoke_dataset_access.py index ce78f5750..c8cb731ac 100644 --- a/samples/snippets/revoke_dataset_access.py +++ b/samples/snippets/revoke_dataset_access.py @@ -13,7 +13,7 @@ # limitations under the License. -def revoke_dataset_access(dataset_id: str, entity_id: str): +def revoke_dataset_access(dataset_id: str, entity_id: str) -> None: original_dataset_id = dataset_id original_entity_id = entity_id diff --git a/samples/snippets/update_dataset_access.py b/samples/snippets/update_dataset_access.py index fb3bfa14f..4d66b8b1f 100644 --- a/samples/snippets/update_dataset_access.py +++ b/samples/snippets/update_dataset_access.py @@ -13,7 +13,7 @@ # limitations under the License. -def update_dataset_access(dataset_id: str, entity_id: str): +def update_dataset_access(dataset_id: str, entity_id: str) -> None: original_dataset_id = dataset_id original_entity_id = entity_id From fdabea32c1c47937510f4db3b7caec8b4850326f Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 14 Dec 2021 16:28:41 +0200 Subject: [PATCH 24/26] Move the override_values default check out --- samples/geography/insert_geojson.py | 7 ++++--- samples/geography/insert_wkt.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/samples/geography/insert_geojson.py b/samples/geography/insert_geojson.py index e98d72cb5..2db407b55 100644 --- a/samples/geography/insert_geojson.py +++ b/samples/geography/insert_geojson.py @@ -18,15 +18,16 @@ def insert_geojson( override_values: Optional[Mapping[str, str]] = None ) -> Sequence[Dict[str, object]]: + + if override_values is None: + override_values = {} + # [START bigquery_insert_geojson] import geojson from google.cloud import bigquery bigquery_client = bigquery.Client() - if override_values is None: - override_values = {} - # This example uses a table containing a column named "geo" with the # GEOGRAPHY data type. table_id = "my-project.my_dataset.my_table" diff --git a/samples/geography/insert_wkt.py b/samples/geography/insert_wkt.py index d95ed4b87..25c7ee727 100644 --- a/samples/geography/insert_wkt.py +++ b/samples/geography/insert_wkt.py @@ -18,6 +18,10 @@ def insert_wkt( override_values: Optional[Mapping[str, str]] = None ) -> Sequence[Dict[str, object]]: + + if override_values is None: + override_values = {} + # [START bigquery_insert_geography_wkt] from google.cloud import bigquery import shapely.geometry @@ -25,9 +29,6 @@ def insert_wkt( bigquery_client = bigquery.Client() - if override_values is None: - override_values = {} - # This example uses a table containing a column named "geo" with the # GEOGRAPHY data type. table_id = "my-project.my_dataset.my_table" From 484a6ee1059c57119eb82ae1fd0e0ba90ce81277 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 14 Dec 2021 17:08:49 +0200 Subject: [PATCH 25/26] Get rid of type cast of an optional int property --- samples/snippets/update_with_dml.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/samples/snippets/update_with_dml.py b/samples/snippets/update_with_dml.py index b6408072c..2d0294ead 100644 --- a/samples/snippets/update_with_dml.py +++ b/samples/snippets/update_with_dml.py @@ -14,7 +14,6 @@ # [START bigquery_update_with_dml] import pathlib -import typing from typing import Dict, Optional from google.cloud import bigquery @@ -61,8 +60,10 @@ def update_with_dml( # Wait for query job to finish. query_job.result() + assert query_job.num_dml_affected_rows is not None + print(f"DML query modified {query_job.num_dml_affected_rows} rows.") - return typing.cast(int, query_job.num_dml_affected_rows) + return query_job.num_dml_affected_rows def run_sample(override_values: Optional[Dict[str, str]] = None) -> int: From 8c40ec724085c38c86a3a4dfe42d9ca8ac21b189 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 14 Dec 2021 17:24:35 +0200 Subject: [PATCH 26/26] Simplify external config type annotations --- samples/query_external_gcs_temporary_table.py | 5 +++-- samples/query_external_sheets_permanent_table.py | 4 ++-- samples/query_external_sheets_temporary_table.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/samples/query_external_gcs_temporary_table.py b/samples/query_external_gcs_temporary_table.py index 7f433dee5..9bcb86aab 100644 --- a/samples/query_external_gcs_temporary_table.py +++ b/samples/query_external_gcs_temporary_table.py @@ -16,7 +16,6 @@ def query_external_gcs_temporary_table() -> None: # [START bigquery_query_external_gcs_temp] - import typing from google.cloud import bigquery # Construct a BigQuery client object. @@ -31,7 +30,9 @@ def query_external_gcs_temporary_table() -> None: bigquery.SchemaField("name", "STRING"), bigquery.SchemaField("post_abbr", "STRING"), ] - typing.cast(bigquery.CSVOptions, external_config.options).skip_leading_rows = 1 + assert external_config.csv_options is not None + external_config.csv_options.skip_leading_rows = 1 + table_id = "us_states" job_config = bigquery.QueryJobConfig(table_definitions={table_id: external_config}) diff --git a/samples/query_external_sheets_permanent_table.py b/samples/query_external_sheets_permanent_table.py index 81cde7415..a5855e66a 100644 --- a/samples/query_external_sheets_permanent_table.py +++ b/samples/query_external_sheets_permanent_table.py @@ -16,7 +16,6 @@ def query_external_sheets_permanent_table(dataset_id: str) -> None: # [START bigquery_query_external_sheets_perm] - import typing from google.cloud import bigquery import google.auth @@ -57,7 +56,8 @@ def query_external_sheets_permanent_table(dataset_id: str) -> None: "/d/1i_QCL-7HcSyUZmIbP9E6lO_T5u3HnpLe7dnpHaijg_E/edit?usp=sharing" ) external_config.source_uris = [sheet_url] - options = typing.cast(bigquery.GoogleSheetsOptions, external_config.options) + options = external_config.google_sheets_options + assert options is not None options.skip_leading_rows = 1 # Optionally skip header row. options.range = ( "us-states!A20:B49" # Optionally set range of the sheet to query from. diff --git a/samples/query_external_sheets_temporary_table.py b/samples/query_external_sheets_temporary_table.py index b743f7cd4..944d3b826 100644 --- a/samples/query_external_sheets_temporary_table.py +++ b/samples/query_external_sheets_temporary_table.py @@ -17,7 +17,6 @@ def query_external_sheets_temporary_table() -> None: # [START bigquery_query_external_sheets_temp] # [START bigquery_auth_drive_scope] - import typing from google.cloud import bigquery import google.auth @@ -54,7 +53,8 @@ def query_external_sheets_temporary_table() -> None: bigquery.SchemaField("name", "STRING"), bigquery.SchemaField("post_abbr", "STRING"), ] - options = typing.cast(bigquery.GoogleSheetsOptions, external_config.options) + options = external_config.google_sheets_options + assert options is not None options.skip_leading_rows = 1 # Optionally skip header row. options.range = ( "us-states!A20:B49" # Optionally set range of the sheet to query from.