Skip to content

Commit

Permalink
Revert k8s test taskgroup changes
Browse files Browse the repository at this point in the history
  • Loading branch information
linkous8 committed May 1, 2024
1 parent 67711a1 commit 4b6e400
Showing 1 changed file with 66 additions and 91 deletions.
157 changes: 66 additions & 91 deletions tests/connectors/kubernetes_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -1213,28 +1213,22 @@ async def test_adjust_deployment_insufficient_resources(
setting_name="mem",
value="128Gi",
)
with pytest.raises(ExceptionGroup) as exc_group:
await connector.adjust([adjustment])

rejection_info = unwrap_exception_group(
excg=exc_group, expected_type=AdjustmentRejectedError, expected_count=1
)
assert (
re.search(
with pytest.raises(
AdjustmentRejectedError,
match=(
re.escape(
"Requested adjustment(s) (fiber-http/fiber-http.mem=128Gi) cannot be scheduled due to "
)
+ r"\"\d+/\d+ nodes are available:.* \d+ Insufficient memory.*\"",
str(rejection_info),
)
is not None
)
+ r"\"\d+/\d+ nodes are available:.* \d+ Insufficient memory.*\""
),
) as rejection_info:
await connector.adjust([adjustment])

# Validate the correct error was raised, re-raise if not for additional debugging context
try:
assert rejection_info.reason == "unschedulable"
assert rejection_info.value.reason == "unschedulable"
except AssertionError as e:
raise e from rejection_info
raise e from rejection_info.value

async def test_adjust_deployment_image_pull_backoff(
self,
Expand All @@ -1257,15 +1251,11 @@ async def test_adjust_deployment_image_pull_backoff(
return_value="opsani/bababooey:latest",
)

with pytest.raises(ExceptionGroup) as excg:
with pytest.raises(
AdjustmentFailedError, match="Container image pull failure detected"
):
await connector.adjust([adjustment])

adjustment_failed = unwrap_exception_group(excg, AdjustmentFailedError, 1)
assert (
re.search("Container image pull failure detected", str(adjustment_failed))
is not None
)

async def test_adjust_replicas(self, config):
connector = KubernetesConnector(config=config)
adjustment = Adjustment(
Expand Down Expand Up @@ -1440,28 +1430,25 @@ async def test_adjust_tuning_insufficient_mem(
value="128Gi", # impossible right?
)
try:
with pytest.raises(ExceptionGroup) as excg:
await connector.adjust([adjustment])
rejection_info = unwrap_exception_group(excg, AdjustmentRejectedError, 1)
assert (
re.search(
with pytest.raises(
AdjustmentRejectedError,
match=(
re.escape(
"Requested adjustment(s) (fiber-http/fiber-http-tuning.mem=128Gi) cannot be scheduled due to "
)
+ r"\"\d+/\d+ nodes are available:.* \d+ Insufficient memory.*\"",
str(rejection_info),
)
is not None
)
+ r"\"\d+/\d+ nodes are available:.* \d+ Insufficient memory.*\""
),
) as rejection_info:
await connector.adjust([adjustment])
except AssertionError as ae:
if "does not match '(reason ContainersNotReady)" in str(ae):
pytest.xfail("Unschedulable condition took too long to show up")

# Validate the correct error was raised, re-raise if not for additional debugging context
try:
assert rejection_info.reason == "unschedulable"
assert rejection_info.value.reason == "unschedulable"
except AssertionError as e:
raise e from rejection_info
raise e from rejection_info.value

async def test_adjust_tuning_insufficient_cpu_and_mem(
self, tuning_config: KubernetesConfiguration
Expand All @@ -1488,26 +1475,22 @@ async def test_adjust_tuning_insufficient_cpu_and_mem(
value="100", # impossible right?
),
]
with pytest.raises(ExceptionGroup) as excg:
await connector.adjust(adjustments)

rejection_info = unwrap_exception_group(excg, AdjustmentRejectedError, 1)
assert (
re.search(
with pytest.raises(
AdjustmentRejectedError,
match=(
re.escape(
"Requested adjustment(s) (fiber-http/fiber-http-tuning.mem=128Gi, fiber-http/fiber-http-tuning.cpu=100) cannot be scheduled due to "
)
+ r"\"\d+/\d+ nodes are available:.* \d+ Insufficient cpu.* \d+ Insufficient memory.*\"",
str(rejection_info),
)
is not None
)
+ r"\"\d+/\d+ nodes are available:.* \d+ Insufficient cpu.* \d+ Insufficient memory.*\""
),
) as rejection_info:
await connector.adjust(adjustments)

# Validate the correct error was raised, re-raise if not for additional debugging context
try:
assert rejection_info.reason == "unschedulable"
assert rejection_info.value.reason == "unschedulable"
except AssertionError as e:
raise e from rejection_info
raise e from rejection_info.value

async def test_create_tuning_image_pull_backoff(
self,
Expand Down Expand Up @@ -1563,14 +1546,14 @@ async def test_bad_request_error_handled_gracefully(
messages = []
connector.logger.add(lambda m: messages.append(m.record["message"]), level=10)

with pytest.raises(ExceptionGroup) as error:
with pytest.raises(servo.AdjustmentFailedError) as error:
await connector.adjust([adjustment])

# Check logs
assert "no tuning pod exists, ignoring destroy" in messages[-30:]
# Check error
assert "quantities must match the regular expression" in str(error.value)
assert error.value.exceptions[0].__cause__.status == 400
assert error.value.__cause__.status == 400

async def test_adjust_tuning_cpu_with_settlement(
self, tuning_config, namespace, kube
Expand Down Expand Up @@ -1621,13 +1604,12 @@ async def test_adjust_handle_error_respects_nested_config(
setting_name="mem",
value="128Gi",
)
with pytest.raises(ExceptionGroup) as excg:
with pytest.raises(
AdjustmentRejectedError, match="Insufficient memory."
) as rejection_info:
description = await connector.adjust([adjustment])
debug(description)

rejection_info = unwrap_exception_group(excg, AdjustmentRejectedError, 1)
assert re.search("Insufficient memory.", str(rejection_info)) is not None

deployment = await DeploymentHelper.read("fiber-http", kube.namespace)
# check deployment was not scaled to 0 replicas (i.e., the outer-level 'shutdown' was overridden)
assert deployment.spec.replicas != 0
Expand Down Expand Up @@ -1883,19 +1865,18 @@ async def test_adjust_deployment_never_ready(
value="128Mi",
)

with pytest.raises(ExceptionGroup) as exception_group:
with pytest.raises(AdjustmentRejectedError) as rejection_info:
await connector.adjust([adjustment])

# Validate the correct error was raised, re-raise if not for additional debugging context
rejection_exc = exception_group.value.exceptions[0]
try:
assert (
"(reason ContainersNotReady) containers with unready status: [fiber-http"
in str(rejection_exc)
in str(rejection_info.value)
)
assert rejection_exc.reason == "start-failed"
assert rejection_info.value.reason == "start-failed"
except AssertionError as e:
raise e from exception_group
raise e from rejection_info.value

async def test_adjust_deployment_oom_killed(
self,
Expand All @@ -1912,21 +1893,22 @@ async def test_adjust_deployment_oom_killed(
value="128Mi",
)

with pytest.raises(ExceptionGroup) as exception_group:
with pytest.raises(AdjustmentRejectedError) as rejection_info:
await connector.adjust([adjustment])

# Validate the correct error was raised, re-raise if not for additional debugging context
rejection_exc = exception_group.value.exceptions[0]
try:
assert (
"Deployment fiber-http pod(s) crash restart detected: fiber-http-"
in str(rejection_exc)
in str(rejection_info.value)
)
assert rejection_exc.reason == "unstable"
assert rejection_info.value.reason == "unstable"
except AssertionError as e:
if "Found 1 unready pod(s) for deployment fiber-http" in str(rejection_exc):
if "Found 1 unready pod(s) for deployment fiber-http" in str(
rejection_info.value
):
pytest.xfail("Restart count update took too long")
raise e from exception_group
raise e from rejection_info.value

async def test_adjust_deployment_settlement_failed(
self,
Expand All @@ -1944,22 +1926,21 @@ async def test_adjust_deployment_settlement_failed(
setting_name="mem",
value="128Mi",
)
with pytest.raises(ExceptionGroup) as exception_group:
with pytest.raises(AdjustmentRejectedError) as rejection_info:
await connector.adjust([adjustment])

# Validate the correct error was raised, re-raise if not for additional debugging context
rejection_exc = exception_group.value.exceptions[0]
try:
assert "(reason ContainersNotReady) containers with unready status: [fiber-http]" in str(
rejection_exc
rejection_info.value
) or "Deployment fiber-http pod(s) crash restart detected" in str(
rejection_exc
rejection_info.value
), str(
rejection_exc
rejection_info.value
)
assert rejection_exc.reason == "unstable"
assert rejection_info.value.reason == "unstable"
except AssertionError as e:
raise e from exception_group.value
raise e from rejection_info.value

# Validate deployment scaled down to 0 instances
kubetest_deployment_becomes_unready.refresh()
Expand All @@ -1983,9 +1964,9 @@ async def test_adjust_tuning_never_ready(
)

try:
with pytest.raises(ExceptionGroup) as exception_group:
with pytest.raises(AdjustmentRejectedError) as rejection_info:
await connector.adjust([adjustment])
except* RuntimeError as e:
except RuntimeError as e:
if (
f"Time out after {tuning_config.timeout} waiting for tuning pod shutdown"
in str(e)
Expand All @@ -1995,15 +1976,14 @@ async def test_adjust_tuning_never_ready(
raise

# Validate the correct error was raised, re-raise if not for additional debugging context
rejection_exc = exception_group.value.exceptions[0]
try:
assert (
"(reason ContainersNotReady) containers with unready status: [fiber-http"
in str(rejection_exc)
in str(rejection_info.value)
)
assert rejection_exc.reason == "start-failed"
assert rejection_info.value.reason == "start-failed"
except AssertionError as e:
raise e from exception_group.value
raise e from rejection_info.value

# Validate baseline was restored during handle_error
tuning_pod = kube.get_pods()["fiber-http-tuning"]
Expand Down Expand Up @@ -2032,22 +2012,18 @@ async def test_adjust_tuning_oom_killed(
value="128Mi",
)

# with pytest.raises(AdjustmentRejectedError) as rejection_info:
# await connector.adjust([adjustment])

with pytest.raises(ExceptionGroup) as exception_group:
with pytest.raises(AdjustmentRejectedError) as rejection_info:
await connector.adjust([adjustment])

# Validate the correct error was raised, re-raise if not for additional debugging context
rejection_exc = exception_group.value.exceptions[0]
try:
assert (
"Tuning optimization fiber-http-tuning crash restart detected on container(s): fiber-http"
in str(rejection_exc)
in str(rejection_info.value)
)
assert rejection_exc.reason == "unstable"
assert rejection_info.value.reason == "unstable"
except AssertionError as e:
raise e from exception_group
raise e from rejection_info.value

# Validate baseline was restored during handle_error
tuning_pod = kube.get_pods()["fiber-http-tuning"]
Expand Down Expand Up @@ -2076,20 +2052,19 @@ async def test_adjust_tuning_settlement_failed(
setting_name="mem",
value="128Mi",
)
with pytest.raises(ExceptionGroup) as exception_group:
with pytest.raises(AdjustmentRejectedError) as rejection_info:
await connector.adjust([adjustment])

# Validate the correct error was raised, re-raise if not for additional debugging context
rejection_exc = exception_group.value.exceptions[0]
try:
assert "(reason ContainersNotReady) containers with unready status: [fiber-http]" in str(
rejection_exc
rejection_info.value
) or "Tuning optimization fiber-http-tuning crash restart detected on container(s): fiber-http" in str(
rejection_exc
rejection_info.value
)
rejection_exc.reason == "unstable"
rejection_info.value.reason == "unstable"
except AssertionError as e:
raise e from exception_group
raise e from rejection_info.value

# Validate baseline was restored during handle_error
tuning_pod = kube.get_pods()["fiber-http-tuning"]
Expand Down

0 comments on commit 4b6e400

Please sign in to comment.