Merge pull request #2033 from HubSpot/lb_sync_updates

Don't allow sync to remove last remaining upstreams
HubSpot · Nov 12, 2019 · 39d65b1 · 39d65b1
2 parents 6232ca6 + 33a13a1
commit 39d65b1
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/...tyService/src/main/java/com/hubspot/singularity/scheduler/SingularityUpstreamChecker.java b/...tyService/src/main/java/com/hubspot/singularity/scheduler/SingularityUpstreamChecker.java
@@ -134,7 +134,7 @@ private Collection<UpstreamInfo> getLoadBalancerUpstreamsForService(String singu
 
   private Optional<SingularityLoadBalancerUpdate> syncUpstreamsForServiceHelper(SingularityRequest singularityRequest, SingularityDeploy deploy, Optional<String> loadBalancerUpstreamGroup) {
     try {
-      LOG.debug("Sending load balancer request to sync upstreams for service {}.", singularityRequest.getId());
+      LOG.debug("Checking load balancer upstreams for service {}.", singularityRequest.getId());
       Collection<UpstreamInfo> upstreamsInLoadBalancerForService = getLoadBalancerUpstreamsForService(singularityRequest.getId(), deploy.getLoadBalancerServiceIdOverride(), loadBalancerUpstreamGroup);
       LOG.debug("Upstreams in load balancer for singularity service {} are {}.", singularityRequest.getId(), upstreamsInLoadBalancerForService);
       Collection<UpstreamInfo> upstreamsInSingularityForService = getUpstreamsFromActiveHealthyAndCleaningTasksForService(singularityRequest.getId(), loadBalancerUpstreamGroup);
@@ -144,6 +144,11 @@ private Optional<SingularityLoadBalancerUpdate> syncUpstreamsForServiceHelper(Si
         LOG.debug("No extra upstreams for service {}. No load balancer request sent.", singularityRequest.getId());
         return Optional.empty();
       }
+
+      if (extraUpstreams.containsAll(upstreamsInLoadBalancerForService) && extraUpstreams.size() == upstreamsInLoadBalancerForService.size()) {
+        throw new IllegalStateException(String.format("Would remove all remaining upstreams for %s in LB, skipping", singularityRequest.getId()));
+      }
+
       final LoadBalancerRequestId loadBalancerRequestId = new LoadBalancerRequestId(String.format("%s-%s-%s", singularityRequest.getId(), deploy.getId(), System.currentTimeMillis()), LoadBalancerRequestType.REMOVE, Optional.empty());
       LOG.info("Syncing upstreams for service {}. Making and sending load balancer request {} to remove {} extra upstreams. The upstreams removed are: {}.", singularityRequest.getId(), loadBalancerRequestId, extraUpstreams.size(), extraUpstreams);
       return Optional.of(lbClient.makeAndSendLoadBalancerRequest(loadBalancerRequestId, Collections.emptyList(), extraUpstreams, deploy, singularityRequest));