test(python): make sure integration tests wait for services to start

docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3.9"
 services:
   localstack:
-    image: localstack/localstack:0.14.4
+    image: localstack/localstack:1.2
     ports:
       - 4566:4566
       - 8080:8080

python/tests/conftest.py

@@ -3,13 +3,28 @@
 import subprocess
 from datetime import date, datetime, timedelta
 from decimal import Decimal
+from time import sleep
 
 import pyarrow as pa
 import pytest
 
 from deltalake import DeltaTable, write_deltalake
 
 
+def wait_till_host_is_available(host: str, timeout_sec: int = 30):
+    spacing = 2
+    attempts = timeout_sec / spacing
+    while True:
+        try:
+            subprocess.run(["curl", host], timeout=500, check=True)
+        except:
+            pass
+        else:
+            break
+
+        sleep(spacing)
+
+
 @pytest.fixture(scope="session")
 def s3_localstack_creds():
     endpoint_url = "http://localhost:4566"
@@ -46,6 +61,8 @@ def s3_localstack_creds():
         ],
     ]
 
+    wait_till_host_is_available(endpoint_url)
+
     try:
         for args in setup_commands:
             subprocess.run(args, env=env)
@@ -109,7 +126,7 @@ def azurite_creds():
         f"AccountKey={config['AZURE_STORAGE_ACCOUNT_KEY']};"
         f"BlobEndpoint={endpoint_url};"
     )
-
+    wait_till_host_is_available(endpoint_url)
     try:
         subprocess.run(
             [

python/tests/test_fs.py

@@ -10,7 +10,7 @@
 
 @pytest.mark.s3
 @pytest.mark.integration
-@pytest.mark.timeout(timeout=5, method="thread")
+@pytest.mark.timeout(timeout=15, method="thread")
 def test_read_files(s3_localstack):
     table_path = "s3://deltars/simple"
     handler = DeltaStorageHandler(table_path)
@@ -27,7 +27,7 @@ def test_read_files(s3_localstack):
 
 @pytest.mark.s3
 @pytest.mark.integration
-@pytest.mark.timeout(timeout=4, method="thread")
+@pytest.mark.timeout(timeout=15, method="thread")
 def test_s3_authenticated_read_write(s3_localstack_creds):
     # Create unauthenticated handler
     storage_handler = DeltaStorageHandler(
@@ -52,7 +52,7 @@ def test_s3_authenticated_read_write(s3_localstack_creds):
 
 @pytest.mark.s3
 @pytest.mark.integration
-@pytest.mark.timeout(timeout=5, method="thread")
+@pytest.mark.timeout(timeout=15, method="thread")
 def test_read_simple_table_from_remote(s3_localstack):
     table_path = "s3://deltars/simple"
     dt = DeltaTable(table_path)
@@ -61,7 +61,7 @@ def test_read_simple_table_from_remote(s3_localstack):
 
 @pytest.mark.s3
 @pytest.mark.integration
-@pytest.mark.timeout(timeout=5, method="thread")
+@pytest.mark.timeout(timeout=15, method="thread")
 def test_roundtrip_s3_env(s3_localstack, sample_data: pa.Table, monkeypatch):
     table_path = "s3://deltars/roundtrip"
 
@@ -89,7 +89,7 @@ def test_roundtrip_s3_env(s3_localstack, sample_data: pa.Table, monkeypatch):
 
 @pytest.mark.s3
 @pytest.mark.integration
-@pytest.mark.timeout(timeout=5, method="thread")
+@pytest.mark.timeout(timeout=15, method="thread")
 def test_roundtrip_s3_direct(s3_localstack_creds, sample_data: pa.Table):
     table_path = "s3://deltars/roundtrip2"
 
@@ -144,7 +144,7 @@ def test_roundtrip_s3_direct(s3_localstack_creds, sample_data: pa.Table):
 
 @pytest.mark.azure
 @pytest.mark.integration
-@pytest.mark.timeout(timeout=5, method="thread")
+@pytest.mark.timeout(timeout=15, method="thread")
 def test_roundtrip_azure_env(azurite_env_vars, sample_data: pa.Table):
     table_path = "az://deltars/roundtrip"
 
@@ -166,7 +166,7 @@ def test_roundtrip_azure_env(azurite_env_vars, sample_data: pa.Table):
 
 @pytest.mark.azure
 @pytest.mark.integration
-@pytest.mark.timeout(timeout=5, method="thread")
+@pytest.mark.timeout(timeout=15, method="thread")
 def test_roundtrip_azure_direct(azurite_creds, sample_data: pa.Table):
     table_path = "az://deltars/roundtrip2"
 
@@ -195,7 +195,7 @@ def test_roundtrip_azure_direct(azurite_creds, sample_data: pa.Table):
 
 @pytest.mark.azure
 @pytest.mark.integration
-@pytest.mark.timeout(timeout=5, method="thread")
+@pytest.mark.timeout(timeout=15, method="thread")
 def test_roundtrip_azure_sas(azurite_sas_creds, sample_data: pa.Table):
     table_path = "az://deltars/roundtrip3"