Skip to content

Commit

Permalink
Group by cluster
Browse files Browse the repository at this point in the history
  • Loading branch information
hendrikmakait committed Dec 11, 2024
1 parent 9cbf5f6 commit 090f90e
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 15 deletions.
1 change: 1 addition & 0 deletions cluster_kwargs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ small_cluster:
n_workers: 10
worker_vm_types: [m6i.large] # 2CPU, 8GiB


# For tests/benchmarks/test_parquet.py
parquet_cluster:
n_workers: 15
Expand Down
30 changes: 15 additions & 15 deletions tests/benchmarks/test_work_stealing.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,21 @@ def test_trivial_workload_should_not_cause_work_stealing(small_client):
small_client.gather(futs)


@run_up_to_nthreads("small_cluster", 100, reason="fixed dataset")
def test_work_stealing_on_inhomogeneous_workload(small_client):
np.random.seed(42)
delays = np.random.lognormal(1, 1.3, 500)

@delayed
def clog(n):
time.sleep(min(n, 60))
return n

results = [clog(i) for i in delays]
futs = small_client.compute(results)
small_client.gather(futs)


@run_up_to_nthreads("small_cluster", 50, reason="fixed dataset")
@pytest.mark.xfail(
Version(distributed.__version__) < Version("2022.6.1"),
Expand Down Expand Up @@ -70,21 +85,6 @@ def func2(chunk):
_ = future.result()


@run_up_to_nthreads("small_cluster", 100, reason="fixed dataset")
def test_work_stealing_on_inhomogeneous_workload(small_client):
np.random.seed(42)
delays = np.random.lognormal(1, 1.3, 500)

@delayed
def clog(n):
time.sleep(min(n, 60))
return n

results = [clog(i) for i in delays]
futs = small_client.compute(results)
small_client.gather(futs)


@run_up_to_nthreads("small_cluster", 100, reason="fixed dataset")
def test_work_stealing_on_straggling_worker(
test_name_uuid,
Expand Down

0 comments on commit 090f90e

Please sign in to comment.