Mark tests as integration tests which skips locally

.github/workflows/python-package.yml

@@ -260,7 +260,7 @@ jobs:
         down-flags: --volumes
     - name: Run IO integration tests
       run: |
-        pytest tests/integration/io --durations=50
+        pytest tests/integration/io -m 'integration' --durations=50
       env:
         DAFT_RUNNER: ${{ matrix.daft-runner }}
     - name: Send Slack notification on failure

pyproject.toml

@@ -74,7 +74,7 @@ module = 'daft.*'
 warn_return_any = false
 
 [tool.pytest.ini_options]
-addopts = "--benchmark-skip -m 'not hypothesis'"
+addopts = "--benchmark-skip -m 'not hypothesis' -m 'not integration'"
 minversion = "6.0"
 testpaths = [
   "tests"

tests/integration/io/test_url_download_http.py

@@ -6,6 +6,7 @@
 import daft
 
 
+@pytest.mark.integration()
 @pytest.mark.parametrize("use_native_downloader", [True, False])
 def test_url_download_http(mock_http_image_urls, image_data, use_native_downloader):
     data = {"urls": mock_http_image_urls}
@@ -14,6 +15,7 @@ def test_url_download_http(mock_http_image_urls, image_data, use_native_download
     assert df.to_pydict() == {**data, "data": [image_data for _ in range(len(mock_http_image_urls))]}
 
 
+@pytest.mark.integration()
 @pytest.mark.parametrize("status_code", [400, 401, 403, 404, 429, 500, 503])
 @pytest.mark.parametrize("use_native_downloader", [True, False])
 def test_url_download_http_error_codes(nginx_config, use_native_downloader, status_code):

tests/integration/io/test_url_download_local.py

@@ -26,13 +26,15 @@ def local_image_data_fixture(tmpdir, image_data) -> YieldFixture[list[str]]:
         child.unlink()
 
 
+@pytest.mark.integration()
 def test_url_download_local(local_image_data_fixture, image_data):
     data = {"urls": local_image_data_fixture}
     df = daft.from_pydict(data)
     df = df.with_column("data", df["urls"].url.download())
     assert df.to_pydict() == {**data, "data": [image_data for _ in range(len(local_image_data_fixture))]}
 
 
+@pytest.mark.integration()
 def test_url_download_local_missing(local_image_data_fixture):
     data = {"urls": local_image_data_fixture + ["/missing/path/x.jpeg"]}
     df = daft.from_pydict(data)
@@ -42,6 +44,7 @@ def test_url_download_local_missing(local_image_data_fixture):
         df.collect()
 
 
+@pytest.mark.integration()
 def test_url_download_local_no_read_permissions(local_image_data_fixture, tmpdir):
     bad_permission_filepath = pathlib.Path(tmpdir) / "bad_file.jpeg"
     bad_permission_filepath.write_bytes(b"foo")

tests/integration/io/test_url_download_public_aws_s3.py

@@ -12,6 +12,7 @@ def small_images_s3_paths() -> list[str]:
     return [f"s3://daft-public-data/test_fixtures/small_images/rickroll{i}.jpg" for i in range(6)]
 
 
+@pytest.mark.integration()
 def test_url_download_aws_s3_public_bucket_custom_s3fs(small_images_s3_paths):
     fs = s3fs.S3FileSystem(anon=True)
     data = {"urls": small_images_s3_paths}
@@ -24,6 +25,7 @@ def test_url_download_aws_s3_public_bucket_custom_s3fs(small_images_s3_paths):
         assert img_bytes is not None
 
 
+@pytest.mark.integration()
 def test_url_download_aws_s3_public_bucket_custom_s3fs_wrong_region(small_images_s3_paths):
     fs = s3fs.S3FileSystem(anon=True)
     data = {"urls": small_images_s3_paths}
@@ -36,6 +38,7 @@ def test_url_download_aws_s3_public_bucket_custom_s3fs_wrong_region(small_images
         assert img_bytes is not None
 
 
+@pytest.mark.integration()
 @pytest.mark.skip(
     reason='[ISSUE #1091] We do not yet support "anonymous-mode" (no credentials) for accessing public buckets with the native downloader'
 )

tests/integration/io/test_url_download_s3_minio.py

@@ -1,10 +1,12 @@
 from __future__ import annotations
 
+import pytest
 import s3fs
 
 import daft
 
 
+@pytest.mark.integration()
 def test_url_download_minio_custom_s3fs(minio_io_config, minio_image_data_fixture, image_data):
     urls = minio_image_data_fixture
     fs = s3fs.S3FileSystem(
@@ -19,6 +21,7 @@ def test_url_download_minio_custom_s3fs(minio_io_config, minio_image_data_fixtur
     assert df.to_pydict() == {**data, "data": [image_data for _ in range(len(urls))]}
 
 
+@pytest.mark.integration()
 def test_url_download_minio_native_downloader(minio_io_config, minio_image_data_fixture, image_data):
     urls = minio_image_data_fixture
     data = {"urls": urls}