diff --git a/bigquery/noxfile.py b/bigquery/noxfile.py index a6d8094ebbc3..60e12c96866e 100644 --- a/bigquery/noxfile.py +++ b/bigquery/noxfile.py @@ -39,7 +39,22 @@ def default(session): session.install("-e", local_dep) session.install("-e", os.path.join("..", "test_utils")) - dev_install = ".[all]" + + coverage_fail_under = "--cov-fail-under=97" + + # fastparquet is not included in .[all] because, in general, it's redundant + # with pyarrow. We still want to run some unit tests with fastparquet + # serialization, though. + dev_install = ".[all,fastparquet]" + + # There is no pyarrow or fastparquet wheel for Python 3.8. + if session.python == "3.8": + # Since many tests are skipped due to missing dependencies, test + # coverage is much lower in Python 3.8. Remove once we can test with + # pyarrow. + coverage_fail_under = "--cov-fail-under=92" + dev_install = ".[pandas,tqdm]" + session.install("-e", dev_install) # IPython does not support Python 2 after version 5.x @@ -57,19 +72,19 @@ def default(session): "--cov-append", "--cov-config=.coveragerc", "--cov-report=", - "--cov-fail-under=97", + coverage_fail_under, os.path.join("tests", "unit"), *session.posargs ) -@nox.session(python=["2.7", "3.5", "3.6", "3.7"]) +@nox.session(python=["2.7", "3.5", "3.6", "3.7", "3.8"]) def unit(session): """Run the unit test suite.""" default(session) -@nox.session(python=["2.7", "3.6"]) +@nox.session(python=["2.7", "3.7"]) def system(session): """Run the system test suite.""" @@ -100,7 +115,7 @@ def system(session): ) -@nox.session(python=["2.7", "3.6"]) +@nox.session(python=["2.7", "3.7"]) def snippets(session): """Run the snippets test suite.""" @@ -121,7 +136,7 @@ def snippets(session): session.run("py.test", "samples", *session.posargs) -@nox.session(python="3.6") +@nox.session(python="3.7") def cover(session): """Run the final coverage report. @@ -133,7 +148,7 @@ def cover(session): session.run("coverage", "erase") -@nox.session(python="3.6") +@nox.session(python="3.7") def lint(session): """Run linters. @@ -152,7 +167,7 @@ def lint(session): session.run("black", "--check", *BLACK_PATHS) -@nox.session(python="3.6") +@nox.session(python="3.7") def lint_setup_py(session): """Verify that setup.py is valid (including RST check).""" @@ -160,7 +175,7 @@ def lint_setup_py(session): session.run("python", "setup.py", "check", "--restructuredtext", "--strict") -@nox.session(python="3.6") +@nox.session(python="3.7") def blacken(session): """Run black. Format code to uniform standard. @@ -169,7 +184,7 @@ def blacken(session): session.run("black", *BLACK_PATHS) -@nox.session(python="3.6") +@nox.session(python="3.7") def docs(session): """Build the docs.""" diff --git a/bigquery/tests/unit/test__pandas_helpers.py b/bigquery/tests/unit/test__pandas_helpers.py index a6ccec2e094f..b2d74d54e120 100644 --- a/bigquery/tests/unit/test__pandas_helpers.py +++ b/bigquery/tests/unit/test__pandas_helpers.py @@ -31,7 +31,9 @@ import pyarrow import pyarrow.types except ImportError: # pragma: NO COVER - pyarrow = None + # Mock out pyarrow when missing, because methods from pyarrow.types are + # used in test parameterization. + pyarrow = mock.Mock() import pytest import pytz @@ -85,7 +87,7 @@ def all_(*functions): return functools.partial(do_all, functions) -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_is_datetime(): assert is_datetime(pyarrow.timestamp("us", tz=None)) assert not is_datetime(pyarrow.timestamp("ms", tz=None)) @@ -249,7 +251,7 @@ def test_all_(): ("UNKNOWN_TYPE", "REPEATED", is_none), ], ) -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_bq_to_arrow_data_type(module_under_test, bq_type, bq_mode, is_correct_type): field = schema.SchemaField("ignored_name", bq_type, mode=bq_mode) actual = module_under_test.bq_to_arrow_data_type(field) @@ -257,7 +259,7 @@ def test_bq_to_arrow_data_type(module_under_test, bq_type, bq_mode, is_correct_t @pytest.mark.parametrize("bq_type", ["RECORD", "record", "STRUCT", "struct"]) -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_bq_to_arrow_data_type_w_struct(module_under_test, bq_type): fields = ( schema.SchemaField("field01", "STRING"), @@ -301,7 +303,7 @@ def test_bq_to_arrow_data_type_w_struct(module_under_test, bq_type): @pytest.mark.parametrize("bq_type", ["RECORD", "record", "STRUCT", "struct"]) -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_bq_to_arrow_data_type_w_array_struct(module_under_test, bq_type): fields = ( schema.SchemaField("field01", "STRING"), @@ -345,7 +347,7 @@ def test_bq_to_arrow_data_type_w_array_struct(module_under_test, bq_type): assert actual.value_type.equals(expected_value_type) -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_bq_to_arrow_data_type_w_struct_unknown_subfield(module_under_test): fields = ( schema.SchemaField("field1", "STRING"), @@ -442,7 +444,7 @@ def test_bq_to_arrow_data_type_w_struct_unknown_subfield(module_under_test): ], ) @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_bq_to_arrow_array_w_nullable_scalars(module_under_test, bq_type, rows): series = pandas.Series(rows, dtype="object") bq_field = schema.SchemaField("field_name", bq_type) @@ -452,7 +454,7 @@ def test_bq_to_arrow_array_w_nullable_scalars(module_under_test, bq_type, rows): @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_bq_to_arrow_array_w_arrays(module_under_test): rows = [[1, 2, 3], [], [4, 5, 6]] series = pandas.Series(rows, dtype="object") @@ -464,7 +466,7 @@ def test_bq_to_arrow_array_w_arrays(module_under_test): @pytest.mark.parametrize("bq_type", ["RECORD", "record", "STRUCT", "struct"]) @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_bq_to_arrow_array_w_structs(module_under_test, bq_type): rows = [ {"int_col": 123, "string_col": "abc"}, @@ -486,7 +488,7 @@ def test_bq_to_arrow_array_w_structs(module_under_test, bq_type): @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_bq_to_arrow_array_w_special_floats(module_under_test): bq_field = schema.SchemaField("field_name", "FLOAT64") rows = [float("-inf"), float("nan"), float("inf"), None] @@ -503,7 +505,7 @@ def test_bq_to_arrow_array_w_special_floats(module_under_test): assert roundtrip[3] is None -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_bq_to_arrow_schema_w_unknown_type(module_under_test): fields = ( schema.SchemaField("field1", "STRING"), @@ -729,7 +731,7 @@ def test_dataframe_to_bq_schema_dict_sequence(module_under_test): @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_dataframe_to_arrow_with_multiindex(module_under_test): bq_schema = ( schema.SchemaField("str_index", "STRING"), @@ -796,7 +798,7 @@ def test_dataframe_to_arrow_with_multiindex(module_under_test): @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_dataframe_to_arrow_with_required_fields(module_under_test): bq_schema = ( schema.SchemaField("field01", "STRING", mode="REQUIRED"), @@ -851,7 +853,7 @@ def test_dataframe_to_arrow_with_required_fields(module_under_test): @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_dataframe_to_arrow_with_unknown_type(module_under_test): bq_schema = ( schema.SchemaField("field00", "UNKNOWN_TYPE"), @@ -884,7 +886,7 @@ def test_dataframe_to_arrow_with_unknown_type(module_under_test): @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_dataframe_to_arrow_dict_sequence_schema(module_under_test): dict_schema = [ {"name": "field01", "type": "STRING", "mode": "REQUIRED"}, @@ -914,7 +916,7 @@ def test_dataframe_to_parquet_without_pyarrow(module_under_test, monkeypatch): @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_dataframe_to_parquet_w_extra_fields(module_under_test, monkeypatch): with pytest.raises(ValueError) as exc_context: module_under_test.dataframe_to_parquet( @@ -926,7 +928,7 @@ def test_dataframe_to_parquet_w_extra_fields(module_under_test, monkeypatch): @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_dataframe_to_parquet_w_missing_fields(module_under_test, monkeypatch): with pytest.raises(ValueError) as exc_context: module_under_test.dataframe_to_parquet( @@ -938,7 +940,7 @@ def test_dataframe_to_parquet_w_missing_fields(module_under_test, monkeypatch): @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_dataframe_to_parquet_compression_method(module_under_test): bq_schema = (schema.SchemaField("field00", "STRING"),) dataframe = pandas.DataFrame({"field00": ["foo", "bar"]}) @@ -985,7 +987,7 @@ def test_dataframe_to_bq_schema_fallback_needed_wo_pyarrow(module_under_test): @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_dataframe_to_bq_schema_fallback_needed_w_pyarrow(module_under_test): dataframe = pandas.DataFrame( data=[ @@ -1015,7 +1017,7 @@ def test_dataframe_to_bq_schema_fallback_needed_w_pyarrow(module_under_test): @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_dataframe_to_bq_schema_pyarrow_fallback_fails(module_under_test): dataframe = pandas.DataFrame( data=[ @@ -1040,7 +1042,7 @@ def test_dataframe_to_bq_schema_pyarrow_fallback_fails(module_under_test): @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_augment_schema_type_detection_succeeds(module_under_test): dataframe = pandas.DataFrame( data=[ @@ -1101,7 +1103,7 @@ def test_augment_schema_type_detection_succeeds(module_under_test): @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_augment_schema_type_detection_fails(module_under_test): dataframe = pandas.DataFrame( data=[ @@ -1137,7 +1139,7 @@ def test_augment_schema_type_detection_fails(module_under_test): assert "struct_field" in warning_msg and "struct_field_2" in warning_msg -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_dataframe_to_parquet_dict_sequence_schema(module_under_test): dict_schema = [ {"name": "field01", "type": "STRING", "mode": "REQUIRED"}, @@ -1166,7 +1168,7 @@ def test_dataframe_to_parquet_dict_sequence_schema(module_under_test): assert schema_arg == expected_schema_arg -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_download_arrow_tabledata_list_unknown_field_type(module_under_test): fake_page = api_core.page_iterator.Page( parent=mock.Mock(), @@ -1202,7 +1204,7 @@ def test_download_arrow_tabledata_list_unknown_field_type(module_under_test): assert list(col) == [2.2, 22.22, 222.222] -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_download_arrow_tabledata_list_known_field_type(module_under_test): fake_page = api_core.page_iterator.Page( parent=mock.Mock(), @@ -1237,7 +1239,7 @@ def test_download_arrow_tabledata_list_known_field_type(module_under_test): assert list(col) == ["2.2", "22.22", "222.222"] -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_download_arrow_tabledata_list_dict_sequence_schema(module_under_test): fake_page = api_core.page_iterator.Page( parent=mock.Mock(), @@ -1265,7 +1267,7 @@ def test_download_arrow_tabledata_list_dict_sequence_schema(module_under_test): @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_download_dataframe_tabledata_list_dict_sequence_schema(module_under_test): fake_page = api_core.page_iterator.Page( parent=mock.Mock(), diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index da3fb2c56689..aa1ee580811b 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -31,6 +31,10 @@ import pytest import pytz +try: + import fastparquet +except (ImportError, AttributeError): # pragma: NO COVER + fastparquet = None try: import pandas except (ImportError, AttributeError): # pragma: NO COVER @@ -6116,6 +6120,7 @@ def test_load_table_from_dataframe_unknown_table(self): ) @unittest.skipIf(pandas is None, "Requires `pandas`") + @unittest.skipIf(fastparquet is None, "Requires `fastparquet`") def test_load_table_from_dataframe_no_schema_warning_wo_pyarrow(self): client = self._make_client() @@ -6306,6 +6311,7 @@ def test_load_table_from_dataframe_w_partial_schema_extra_types(self): assert "unknown_col" in message @unittest.skipIf(pandas is None, "Requires `pandas`") + @unittest.skipIf(fastparquet is None, "Requires `fastparquet`") def test_load_table_from_dataframe_w_partial_schema_missing_types(self): from google.cloud.bigquery.client import _DEFAULT_NUM_RETRIES from google.cloud.bigquery import job diff --git a/bigquery/tests/unit/test_magics.py b/bigquery/tests/unit/test_magics.py index 6ff9819854a8..19a4d40e73fd 100644 --- a/bigquery/tests/unit/test_magics.py +++ b/bigquery/tests/unit/test_magics.py @@ -153,6 +153,7 @@ def test_context_credentials_and_project_can_be_set_explicitly(): @pytest.mark.usefixtures("ipython_interactive") +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_context_connection_can_be_overriden(): ip = IPython.get_ipython() ip.extension_manager.load_extension("google.cloud.bigquery") @@ -196,6 +197,7 @@ def test_context_connection_can_be_overriden(): @pytest.mark.usefixtures("ipython_interactive") +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_context_no_connection(): ip = IPython.get_ipython() ip.extension_manager.load_extension("google.cloud.bigquery") @@ -326,6 +328,10 @@ def test__make_bqstorage_client_true_raises_import_error(missing_bq_storage): assert "pyarrow" in error_msg +@pytest.mark.skipif( + bigquery_storage_v1beta1 is None, reason="Requires `google-cloud-bigquery-storage`" +) +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test__make_bqstorage_client_true_missing_gapic(missing_grpcio_lib): credentials_mock = mock.create_autospec( google.auth.credentials.Credentials, instance=True @@ -758,6 +764,7 @@ def test_bigquery_magic_w_table_id_invalid(): @pytest.mark.usefixtures("ipython_interactive") +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_w_table_id_and_destination_var(): ip = IPython.get_ipython() ip.extension_manager.load_extension("google.cloud.bigquery") @@ -794,6 +801,10 @@ def test_bigquery_magic_w_table_id_and_destination_var(): @pytest.mark.usefixtures("ipython_interactive") +@pytest.mark.skipif( + bigquery_storage_v1beta1 is None, reason="Requires `google-cloud-bigquery-storage`" +) +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_w_table_id_and_bqstorage_client(): ip = IPython.get_ipython() ip.extension_manager.load_extension("google.cloud.bigquery") @@ -989,6 +1000,7 @@ def test_bigquery_magic_w_maximum_bytes_billed_invalid(): "param_value,expected", [("987654321", "987654321"), ("None", "0")] ) @pytest.mark.usefixtures("ipython_interactive") +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_w_maximum_bytes_billed_overrides_context(param_value, expected): ip = IPython.get_ipython() ip.extension_manager.load_extension("google.cloud.bigquery") @@ -1027,6 +1039,7 @@ def test_bigquery_magic_w_maximum_bytes_billed_overrides_context(param_value, ex @pytest.mark.usefixtures("ipython_interactive") +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_w_maximum_bytes_billed_w_context_inplace(): ip = IPython.get_ipython() ip.extension_manager.load_extension("google.cloud.bigquery") @@ -1062,6 +1075,7 @@ def test_bigquery_magic_w_maximum_bytes_billed_w_context_inplace(): @pytest.mark.usefixtures("ipython_interactive") +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_w_maximum_bytes_billed_w_context_setter(): ip = IPython.get_ipython() ip.extension_manager.load_extension("google.cloud.bigquery") diff --git a/bigquery/tests/unit/test_schema.py b/bigquery/tests/unit/test_schema.py index fc8a41c68c46..e1bdd7b2fb73 100644 --- a/bigquery/tests/unit/test_schema.py +++ b/bigquery/tests/unit/test_schema.py @@ -177,7 +177,6 @@ def test_to_standard_sql_simple_type(self): standard_field = field.to_standard_sql() self.assertEqual(standard_field.name, "some_field") self.assertEqual(standard_field.type.type_kind, standard_type) - self.assertFalse(standard_field.type.HasField("sub_type")) def test_to_standard_sql_struct_type(self): from google.cloud.bigquery_v2 import types @@ -316,7 +315,6 @@ def test_to_standard_sql_unknown_type(self): self.assertEqual(standard_field.name, "weird_field") self.assertEqual(standard_field.type.type_kind, sql_type.TYPE_KIND_UNSPECIFIED) - self.assertFalse(standard_field.type.HasField("sub_type")) def test___eq___wrong_type(self): field = self._make_one("test", "STRING")