feat: removed pkg_resources from all test files and moved importlib i…

…nto pandas extra (#1726)

* feat: Introduce compatibility with native namespace packages

* Update copyright year

* removed pkg_resources from all test files and moved importlib into pandas extra

* feat: removed pkg_resources from all test files and moved importlib into pandas extra

* Adding no cover tag to test code

* reformatted with black

* undo revert

* perf: use the first page a results when `query(api_method="QUERY")` (#1723)

* perf: use the first page a results when `query(api_method="QUERY")`

* add tests

* respect max_results with cached page

* respect page_size, also avoid bqstorage if almost fully downloaded

* skip true test if bqstorage not installed

* coverage

* fix: ensure query job retry has longer deadline than API request deadline (#1734)

In cases where we can't disambiguate API failure from job failure,
this ensures we can still retry the job at least once.

* fix: `load_table_from_dataframe` now assumes there may be local null values (#1735)

Even if the remote schema is REQUIRED

Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly:
- [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery/issues/new/choose) before writing your code!  That way we can discuss the change, evaluate designs, and agree on the general idea
- [ ] Ensure the tests and linter pass
- [ ] Code coverage does not decrease (if any source code was changed)
- [ ] Appropriate docs were updated (if necessary)

Fixes #1692 🦕

* chore: standardize samples directory - delete unneeded dependencies (#1732)

* chore: standardize samples directory = delete unneeded dependencies

* Removed unused import for linter

* fix: move grpc, proto-plus and protobuf packages to extras (#1721)

* chore: move grpc, proto-plus and protobuff packages to extras

* formatted with black

* feat: add `job_timeout_ms` to job configuration classes (#1675)

* fix: adds new property and tests

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* updates docs to correct a sphinx failure

* Updates formatting

* Update tests/system/test_query.py

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* Update google/cloud/bigquery/job/base.py

* updates one test and uses int_or_none

* Update tests/system/test_query.py

testing something.

* Update tests/system/test_query.py

* testing coverage feature

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* minor edits

* tweaks to noxfile for testing purposes

* add new test to base as experiment

* adds a test, updates import statements

* add another test

* edit to tests

* formatting fixes

* update noxfile to correct debug code

* removes unneeded comments.

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>

---------

Co-authored-by: Chalmer Lowe <chalmerlowe@google.com>
Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
Co-authored-by: Tim Swast <swast@google.com>

* remove unnecessary version checks

* undo bad commit, remove unneeded version checks

* Revert "undo bad commit, remove unneeded version checks"

This reverts commit 5c82dcf.

* Revert "remove unnecessary version checks"

This reverts commit 9331a7e.

* revert bad changes, remove pkg_resources from file

* after clarification, reimplement changes and ignore 3.12 tests

* reformatted with black

* removed minimum check

* updated pandas installed version check

---------

Co-authored-by: Anthonios Partheniou <partheniou@google.com>
Co-authored-by: Tim Swast <swast@google.com>
Co-authored-by: Chalmer Lowe <chalmerlowe@google.com>
Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>

.coveragerc

@@ -12,8 +12,3 @@ exclude_lines =
     pragma: NO COVER
     # Ignore debug-only repr
     def __repr__
-    # Ignore pkg_resources exceptions.
-    # This is added at the module level as a safeguard for if someone
-    # generates the code and tries to run it without pip installing. This
-    # makes it virtually impossible to test properly.
-    except pkg_resources.DistributionNotFound

noxfile.py

@@ -137,7 +137,7 @@ def mypy(session):
         "types-requests",
         "types-setuptools",
     )
-    session.run("mypy", "google/cloud", "--show-traceback")
+    session.run("mypy", "-p", "google", "--show-traceback")
 
 
 @nox.session(python=DEFAULT_PYTHON_VERSION)
@@ -149,7 +149,8 @@ def pytype(session):
     session.install("attrs==20.3.0")
     session.install("-e", ".[all]")
     session.install(PYTYPE_VERSION)
-    session.run("pytype")
+    # See https://github.com/google/pytype/issues/464
+    session.run("pytype", "-P", ".", "google/cloud/bigquery")
 
 
 @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)

setup.py

@@ -62,6 +62,7 @@
         "pandas>=1.1.0",
         pyarrow_dependency,
         "db-dtypes>=0.3.0,<2.0.0dev",
+        "importlib_metadata>=1.0.0; python_version<'3.8'",
     ],
     "ipywidgets": [
         "ipywidgets>=7.7.0",
@@ -108,16 +109,10 @@
 # benchmarks, etc.
 packages = [
     package
-    for package in setuptools.PEP420PackageFinder.find()
+    for package in setuptools.find_namespace_packages()
     if package.startswith("google")
 ]
 
-# Determine which namespaces are needed.
-namespaces = ["google"]
-if "google.cloud" in packages:
-    namespaces.append("google.cloud")
-
-
 setuptools.setup(
     name=name,
     version=version,
@@ -143,7 +138,6 @@
     ],
     platforms="Posix; MacOS X; Windows",
     packages=packages,
-    namespace_packages=namespaces,
     install_requires=dependencies,
     extras_require=extras,
     python_requires=">=3.7",

tests/system/test_pandas.py

@@ -23,9 +23,13 @@
 import warnings
 
 import google.api_core.retry
-import pkg_resources
 import pytest
 
+try:
+    import importlib.metadata as metadata
+except ImportError:
+    import importlib_metadata as metadata
+
 from google.cloud import bigquery
 
 from google.cloud.bigquery import enums
@@ -42,11 +46,9 @@
 )
 
 if pandas is not None:
-    PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version
+    PANDAS_INSTALLED_VERSION = metadata.version("pandas")
 else:
-    PANDAS_INSTALLED_VERSION = pkg_resources.parse_version("0.0.0")
-
-PANDAS_INT64_VERSION = pkg_resources.parse_version("1.0.0")
+    PANDAS_INSTALLED_VERSION = "0.0.0"
 
 
 class MissingDataError(Exception):
@@ -310,10 +312,7 @@ def test_load_table_from_dataframe_w_automatic_schema(bigquery_client, dataset_i
     ]
 
 
-@pytest.mark.skipif(
-    PANDAS_INSTALLED_VERSION < PANDAS_INT64_VERSION,
-    reason="Only `pandas version >=1.0.0` is supported",
-)
+@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
 def test_load_table_from_dataframe_w_nullable_int64_datatype(
     bigquery_client, dataset_id
 ):
@@ -342,7 +341,7 @@ def test_load_table_from_dataframe_w_nullable_int64_datatype(
 
 
 @pytest.mark.skipif(
-    PANDAS_INSTALLED_VERSION < PANDAS_INT64_VERSION,
+    PANDAS_INSTALLED_VERSION[0:2].startswith("0."),
     reason="Only `pandas version >=1.0.0` is supported",
 )
 def test_load_table_from_dataframe_w_nullable_int64_datatype_automatic_schema(
@@ -1043,9 +1042,7 @@ def test_list_rows_max_results_w_bqstorage(bigquery_client):
     assert len(dataframe.index) == 100
 
 
-@pytest.mark.skipif(
-    PANDAS_INSTALLED_VERSION >= pkg_resources.parse_version("2.0.0"), reason=""
-)
+@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="")
 @pytest.mark.parametrize(
     ("max_results",),
     (

tests/unit/job/test_query_pandas.py

@@ -17,7 +17,6 @@
 import json
 
 import mock
-import pkg_resources
 import pytest
 
 
@@ -45,14 +44,19 @@
 except (ImportError, AttributeError):  # pragma: NO COVER
     tqdm = None
 
+try:
+    import importlib.metadata as metadata
+except ImportError:
+    import importlib_metadata as metadata
+
 from ..helpers import make_connection
 from .helpers import _make_client
 from .helpers import _make_job_resource
 
 if pandas is not None:
-    PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version
+    PANDAS_INSTALLED_VERSION = metadata.version("pandas")
 else:
-    PANDAS_INSTALLED_VERSION = pkg_resources.parse_version("0.0.0")
+    PANDAS_INSTALLED_VERSION = "0.0.0"
 
 pandas = pytest.importorskip("pandas")
 
@@ -656,9 +660,7 @@ def test_to_dataframe_bqstorage_no_pyarrow_compression():
     )
 
 
-@pytest.mark.skipif(
-    PANDAS_INSTALLED_VERSION >= pkg_resources.parse_version("2.0.0"), reason=""
-)
+@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="")
 @pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
 def test_to_dataframe_column_dtypes():
     from google.cloud.bigquery.job import QueryJob as target_class

tests/unit/test__pandas_helpers.py

@@ -19,7 +19,11 @@
 import operator
 import queue
 import warnings
-import pkg_resources
+
+try:
+    import importlib.metadata as metadata
+except ImportError:
+    import importlib_metadata as metadata
 
 import mock
 
@@ -57,13 +61,10 @@
 
 bigquery_storage = _versions_helpers.BQ_STORAGE_VERSIONS.try_import()
 
-PANDAS_MINIUM_VERSION = pkg_resources.parse_version("1.0.0")
-
 if pandas is not None:
-    PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version
+    PANDAS_INSTALLED_VERSION = metadata.version("pandas")
 else:
-    # Set to less than MIN version.
-    PANDAS_INSTALLED_VERSION = pkg_resources.parse_version("0.0.0")
+    PANDAS_INSTALLED_VERSION = "0.0.0"
 
 
 skip_if_no_bignumeric = pytest.mark.skipif(
@@ -542,9 +543,7 @@ def test_bq_to_arrow_array_w_nullable_scalars(module_under_test, bq_type, rows):
     ],
 )
 @pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
-@pytest.mark.skipif(
-    PANDAS_INSTALLED_VERSION >= pkg_resources.parse_version("2.0.0"), reason=""
-)
+@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="")
 @pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
 def test_bq_to_arrow_array_w_pandas_timestamp(module_under_test, bq_type, rows):
     rows = [pandas.Timestamp(row) for row in rows]
@@ -806,10 +805,7 @@ def test_list_columns_and_indexes_with_named_index_same_as_column_name(
     assert columns_and_indexes == expected
 
 
-@pytest.mark.skipif(
-    pandas is None or PANDAS_INSTALLED_VERSION < PANDAS_MINIUM_VERSION,
-    reason="Requires `pandas version >= 1.0.0` which introduces pandas.NA",
-)
+@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
 def test_dataframe_to_json_generator(module_under_test):
     utcnow = datetime.datetime.utcnow()
     df_data = collections.OrderedDict(
@@ -837,16 +833,8 @@ def test_dataframe_to_json_generator(module_under_test):
     assert list(rows) == expected
 
 
+@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
 def test_dataframe_to_json_generator_repeated_field(module_under_test):
-    pytest.importorskip(
-        "pandas",
-        minversion=str(PANDAS_MINIUM_VERSION),
-        reason=(
-            f"Requires `pandas version >= {PANDAS_MINIUM_VERSION}` "
-            "which introduces pandas.NA"
-        ),
-    )
-
     df_data = [
         collections.OrderedDict(
             [("repeated_col", [pandas.NA, 2, None, 4]), ("not_repeated_col", "first")]

tests/unit/test_client.py

@@ -30,7 +30,11 @@
 import requests
 import packaging
 import pytest
-import pkg_resources
+
+try:
+    import importlib.metadata as metadata
+except ImportError:
+    import importlib_metadata as metadata
 
 try:
     import pandas
@@ -76,13 +80,10 @@
 from test_utils.imports import maybe_fail_import
 from tests.unit.helpers import make_connection
 
-PANDAS_MINIUM_VERSION = pkg_resources.parse_version("1.0.0")
-
 if pandas is not None:
-    PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version
+    PANDAS_INSTALLED_VERSION = metadata.version("pandas")
 else:
-    # Set to less than MIN version.
-    PANDAS_INSTALLED_VERSION = pkg_resources.parse_version("0.0.0")
+    PANDAS_INSTALLED_VERSION = "0.0.0"
 
 
 def _make_credentials():
@@ -8145,10 +8146,7 @@ def test_load_table_from_dataframe_unknown_table(self):
             timeout=DEFAULT_TIMEOUT,
         )
 
-    @unittest.skipIf(
-        pandas is None or PANDAS_INSTALLED_VERSION < PANDAS_MINIUM_VERSION,
-        "Only `pandas version >=1.0.0` supported",
-    )
+    @unittest.skipIf(pandas is None, "Requires `pandas`")
     @unittest.skipIf(pyarrow is None, "Requires `pyarrow`")
     def test_load_table_from_dataframe_w_nullable_int64_datatype(self):
         from google.cloud.bigquery.client import _DEFAULT_NUM_RETRIES
@@ -8193,10 +8191,7 @@ def test_load_table_from_dataframe_w_nullable_int64_datatype(self):
             SchemaField("x", "INT64", "NULLABLE", None),
         )
 
-    @unittest.skipIf(
-        pandas is None or PANDAS_INSTALLED_VERSION < PANDAS_MINIUM_VERSION,
-        "Only `pandas version >=1.0.0` supported",
-    )
+    @unittest.skipIf(pandas is None, "Requires `pandas`")
     # @unittest.skipIf(pyarrow is None, "Requires `pyarrow`")
     def test_load_table_from_dataframe_w_nullable_int64_datatype_automatic_schema(self):
         from google.cloud.bigquery.client import _DEFAULT_NUM_RETRIES

tests/unit/test_packaging.py

@@ -0,0 +1,37 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import sys
+
+
+def test_namespace_package_compat(tmp_path):
+    # The ``google`` namespace package should not be masked
+    # by the presence of ``google-cloud-bigquery``.
+    google = tmp_path / "google"
+    google.mkdir()
+    google.joinpath("othermod.py").write_text("")
+    env = dict(os.environ, PYTHONPATH=str(tmp_path))
+    cmd = [sys.executable, "-m", "google.othermod"]
+    subprocess.check_call(cmd, env=env)
+
+    # The ``google.cloud`` namespace package should not be masked
+    # by the presence of ``google-cloud-bigquery``.
+    google_cloud = tmp_path / "google" / "cloud"
+    google_cloud.mkdir()
+    google_cloud.joinpath("othermod.py").write_text("")
+    env = dict(os.environ, PYTHONPATH=str(tmp_path))
+    cmd = [sys.executable, "-m", "google.cloud.othermod"]
+    subprocess.check_call(cmd, env=env)