diff --git a/docker-compose.yml b/docker-compose.yml index 6f78fa444c..0b9f7ee77c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.9" services: localstack: - image: localstack/localstack:0.14.4 + image: localstack/localstack:1.2 ports: - 4566:4566 - 8080:8080 diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 92d4c9b43f..b5dd65c5cf 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -3,6 +3,7 @@ import subprocess from datetime import date, datetime, timedelta from decimal import Decimal +from time import sleep import pyarrow as pa import pytest @@ -10,6 +11,20 @@ from deltalake import DeltaTable, write_deltalake +def wait_till_host_is_available(host: str, timeout_sec: int = 30): + spacing = 2 + attempts = timeout_sec / spacing + while True: + try: + subprocess.run(["curl", host], timeout=500, check=True) + except: + pass + else: + break + + sleep(spacing) + + @pytest.fixture(scope="session") def s3_localstack_creds(): endpoint_url = "http://localhost:4566" @@ -46,6 +61,8 @@ def s3_localstack_creds(): ], ] + wait_till_host_is_available(endpoint_url) + try: for args in setup_commands: subprocess.run(args, env=env) @@ -109,7 +126,7 @@ def azurite_creds(): f"AccountKey={config['AZURE_STORAGE_ACCOUNT_KEY']};" f"BlobEndpoint={endpoint_url};" ) - + wait_till_host_is_available(endpoint_url) try: subprocess.run( [ diff --git a/python/tests/test_fs.py b/python/tests/test_fs.py index 4113b9a2a6..4fb3ef355c 100644 --- a/python/tests/test_fs.py +++ b/python/tests/test_fs.py @@ -10,7 +10,7 @@ @pytest.mark.s3 @pytest.mark.integration -@pytest.mark.timeout(timeout=5, method="thread") +@pytest.mark.timeout(timeout=15, method="thread") def test_read_files(s3_localstack): table_path = "s3://deltars/simple" handler = DeltaStorageHandler(table_path) @@ -27,7 +27,7 @@ def test_read_files(s3_localstack): @pytest.mark.s3 @pytest.mark.integration -@pytest.mark.timeout(timeout=4, method="thread") +@pytest.mark.timeout(timeout=15, method="thread") def test_s3_authenticated_read_write(s3_localstack_creds): # Create unauthenticated handler storage_handler = DeltaStorageHandler( @@ -52,7 +52,7 @@ def test_s3_authenticated_read_write(s3_localstack_creds): @pytest.mark.s3 @pytest.mark.integration -@pytest.mark.timeout(timeout=5, method="thread") +@pytest.mark.timeout(timeout=15, method="thread") def test_read_simple_table_from_remote(s3_localstack): table_path = "s3://deltars/simple" dt = DeltaTable(table_path) @@ -61,7 +61,7 @@ def test_read_simple_table_from_remote(s3_localstack): @pytest.mark.s3 @pytest.mark.integration -@pytest.mark.timeout(timeout=5, method="thread") +@pytest.mark.timeout(timeout=15, method="thread") def test_roundtrip_s3_env(s3_localstack, sample_data: pa.Table, monkeypatch): table_path = "s3://deltars/roundtrip" @@ -89,7 +89,7 @@ def test_roundtrip_s3_env(s3_localstack, sample_data: pa.Table, monkeypatch): @pytest.mark.s3 @pytest.mark.integration -@pytest.mark.timeout(timeout=5, method="thread") +@pytest.mark.timeout(timeout=15, method="thread") def test_roundtrip_s3_direct(s3_localstack_creds, sample_data: pa.Table): table_path = "s3://deltars/roundtrip2" @@ -144,7 +144,7 @@ def test_roundtrip_s3_direct(s3_localstack_creds, sample_data: pa.Table): @pytest.mark.azure @pytest.mark.integration -@pytest.mark.timeout(timeout=5, method="thread") +@pytest.mark.timeout(timeout=15, method="thread") def test_roundtrip_azure_env(azurite_env_vars, sample_data: pa.Table): table_path = "az://deltars/roundtrip" @@ -166,7 +166,7 @@ def test_roundtrip_azure_env(azurite_env_vars, sample_data: pa.Table): @pytest.mark.azure @pytest.mark.integration -@pytest.mark.timeout(timeout=5, method="thread") +@pytest.mark.timeout(timeout=15, method="thread") def test_roundtrip_azure_direct(azurite_creds, sample_data: pa.Table): table_path = "az://deltars/roundtrip2" @@ -195,7 +195,7 @@ def test_roundtrip_azure_direct(azurite_creds, sample_data: pa.Table): @pytest.mark.azure @pytest.mark.integration -@pytest.mark.timeout(timeout=5, method="thread") +@pytest.mark.timeout(timeout=15, method="thread") def test_roundtrip_azure_sas(azurite_sas_creds, sample_data: pa.Table): table_path = "az://deltars/roundtrip3"