From c02eded29a91aa91bd0ae5d2aadae3f36d1b4880 Mon Sep 17 00:00:00 2001 From: Eric Liang Date: Wed, 21 Nov 2018 23:37:36 -0800 Subject: [PATCH 1/3] hb --- python/ray/autoscaler/autoscaler.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/ray/autoscaler/autoscaler.py b/python/ray/autoscaler/autoscaler.py index 80245881cf78..db7887b88641 100644 --- a/python/ray/autoscaler/autoscaler.py +++ b/python/ray/autoscaler/autoscaler.py @@ -491,8 +491,10 @@ def files_up_to_date(self, node_id): def recover_if_needed(self, node_id): if not self.can_update(node_id): return - last_heartbeat_time = self.load_metrics.last_heartbeat_time_by_ip.get( - self.provider.internal_ip(node_id), 0) + key = self.provider.internal_ip(node_id) + if key not in self.load_metrics.last_heartbeat_time_by_ip: + self.load_metrics.last_heartbeat_by_ip = time.time() + last_heartbeat_time = self.load_metrics.last_heartbeat_time_by_ip[key] delta = time.time() - last_heartbeat_time if delta < AUTOSCALER_HEARTBEAT_TIMEOUT_S: return From 40a751186047027ff6c5a1781b1d689b5da77a09 Mon Sep 17 00:00:00 2001 From: Eric Liang Date: Thu, 22 Nov 2018 09:45:51 -0800 Subject: [PATCH 2/3] Update autoscaler.py --- python/ray/autoscaler/autoscaler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/autoscaler/autoscaler.py b/python/ray/autoscaler/autoscaler.py index db7887b88641..e7dbd12e90a7 100644 --- a/python/ray/autoscaler/autoscaler.py +++ b/python/ray/autoscaler/autoscaler.py @@ -493,7 +493,7 @@ def recover_if_needed(self, node_id): return key = self.provider.internal_ip(node_id) if key not in self.load_metrics.last_heartbeat_time_by_ip: - self.load_metrics.last_heartbeat_by_ip = time.time() + self.load_metrics.last_heartbeat_by_ip[key] = time.time() last_heartbeat_time = self.load_metrics.last_heartbeat_time_by_ip[key] delta = time.time() - last_heartbeat_time if delta < AUTOSCALER_HEARTBEAT_TIMEOUT_S: From e370386b8c31a469c4d7833667fbac7069e7e5e5 Mon Sep 17 00:00:00 2001 From: Eric Liang Date: Thu, 22 Nov 2018 12:49:39 -0800 Subject: [PATCH 3/3] Update autoscaler.py --- python/ray/autoscaler/autoscaler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/autoscaler/autoscaler.py b/python/ray/autoscaler/autoscaler.py index e7dbd12e90a7..9c4a452ee268 100644 --- a/python/ray/autoscaler/autoscaler.py +++ b/python/ray/autoscaler/autoscaler.py @@ -493,7 +493,7 @@ def recover_if_needed(self, node_id): return key = self.provider.internal_ip(node_id) if key not in self.load_metrics.last_heartbeat_time_by_ip: - self.load_metrics.last_heartbeat_by_ip[key] = time.time() + self.load_metrics.last_heartbeat_time_by_ip[key] = time.time() last_heartbeat_time = self.load_metrics.last_heartbeat_time_by_ip[key] delta = time.time() - last_heartbeat_time if delta < AUTOSCALER_HEARTBEAT_TIMEOUT_S: