diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/DelegatingOortService.java b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/DelegatingOortService.java index 8e188327e0..623361016e 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/DelegatingOortService.java +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/DelegatingOortService.java @@ -17,6 +17,7 @@ package com.netflix.spinnaker.orca.clouddriver; import com.netflix.spinnaker.orca.clouddriver.config.SelectableService; +import com.netflix.spinnaker.orca.clouddriver.model.Manifest; import retrofit.client.Response; import java.util.List; @@ -36,7 +37,7 @@ public Response getCluster(String app, String account, String cluster, String cl } @Override - public Response getManifest(String account, String location, String name) { + public Manifest getManifest(String account, String location, String name) { return getService().getManifest(account, location, name); } diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/OortService.groovy b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/OortService.groovy index f4df383309..037ef3e083 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/OortService.groovy +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/OortService.groovy @@ -17,6 +17,7 @@ package com.netflix.spinnaker.orca.clouddriver +import com.netflix.spinnaker.orca.clouddriver.model.Manifest import retrofit.client.Response import retrofit.http.GET import retrofit.http.Path @@ -42,7 +43,7 @@ interface OortService { @Path("cloudProvider") String cloudProvider) @GET("/manifests/{account}/{location}/{manifest}") - Response getManifest(@Path("account") String account, + Manifest getManifest(@Path("account") String account, @Path("location") String location, @Path("manifest") String manifest) diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/model/Manifest.java b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/model/Manifest.java index 3f95fa05a2..06d2e4e7f5 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/model/Manifest.java +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/model/Manifest.java @@ -25,7 +25,12 @@ public class Manifest { @Data public static class Status { - boolean stable; + Condition stable; + } + + @Data + public static class Condition { + boolean state; String message; } } diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/pipeline/manifest/ResumeRolloutManifestStage.java b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/pipeline/manifest/ResumeRolloutManifestStage.java index ef4873e187..b9202f641c 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/pipeline/manifest/ResumeRolloutManifestStage.java +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/pipeline/manifest/ResumeRolloutManifestStage.java @@ -20,6 +20,7 @@ import com.netflix.spinnaker.orca.clouddriver.tasks.MonitorKatoTask; import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.ResumeRolloutManifestTask; import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.UpdateManifestForceCacheRefreshTask; +import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.UpdateManifestWaitForStableTask; import com.netflix.spinnaker.orca.pipeline.StageDefinitionBuilder; import com.netflix.spinnaker.orca.pipeline.TaskNode; import com.netflix.spinnaker.orca.pipeline.model.Stage; @@ -33,6 +34,7 @@ public class ResumeRolloutManifestStage implements StageDefinitionBuilder { public void taskGraph(Stage stage, TaskNode.Builder builder) { builder.withTask(ResumeRolloutManifestTask.TASK_NAME, ResumeRolloutManifestTask.class) .withTask("monitorResumeRollout", MonitorKatoTask.class) - .withTask(UpdateManifestForceCacheRefreshTask.TASK_NAME, UpdateManifestForceCacheRefreshTask.class); + .withTask(UpdateManifestForceCacheRefreshTask.TASK_NAME, UpdateManifestForceCacheRefreshTask.class) + .withTask(UpdateManifestWaitForStableTask.TASK_NAME, UpdateManifestWaitForStableTask.class); } } diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/pipeline/manifest/ScaleManifestStage.java b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/pipeline/manifest/ScaleManifestStage.java index 56da60a480..70cca91658 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/pipeline/manifest/ScaleManifestStage.java +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/pipeline/manifest/ScaleManifestStage.java @@ -20,6 +20,7 @@ import com.netflix.spinnaker.orca.clouddriver.tasks.MonitorKatoTask; import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.ScaleManifestTask; import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.UpdateManifestForceCacheRefreshTask; +import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.UpdateManifestWaitForStableTask; import com.netflix.spinnaker.orca.pipeline.StageDefinitionBuilder; import com.netflix.spinnaker.orca.pipeline.TaskNode; import com.netflix.spinnaker.orca.pipeline.model.Stage; @@ -33,6 +34,7 @@ public class ScaleManifestStage implements StageDefinitionBuilder { public void taskGraph(Stage stage, TaskNode.Builder builder) { builder.withTask(ScaleManifestTask.TASK_NAME, ScaleManifestTask.class) .withTask("monitorScale", MonitorKatoTask.class) - .withTask(UpdateManifestForceCacheRefreshTask.TASK_NAME, UpdateManifestForceCacheRefreshTask.class); + .withTask(UpdateManifestForceCacheRefreshTask.TASK_NAME, UpdateManifestForceCacheRefreshTask.class) + .withTask(UpdateManifestWaitForStableTask.TASK_NAME, UpdateManifestWaitForStableTask.class); } } diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/pipeline/manifest/UndoRolloutManifestStage.java b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/pipeline/manifest/UndoRolloutManifestStage.java index a30e218df3..426d876ff5 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/pipeline/manifest/UndoRolloutManifestStage.java +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/pipeline/manifest/UndoRolloutManifestStage.java @@ -20,6 +20,7 @@ import com.netflix.spinnaker.orca.clouddriver.tasks.MonitorKatoTask; import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.UndoRolloutManifestTask; import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.UpdateManifestForceCacheRefreshTask; +import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.UpdateManifestWaitForStableTask; import com.netflix.spinnaker.orca.pipeline.StageDefinitionBuilder; import com.netflix.spinnaker.orca.pipeline.TaskNode; import com.netflix.spinnaker.orca.pipeline.model.Stage; @@ -33,6 +34,7 @@ public class UndoRolloutManifestStage implements StageDefinitionBuilder { public void taskGraph(Stage stage, TaskNode.Builder builder) { builder.withTask(UndoRolloutManifestTask.TASK_NAME, UndoRolloutManifestTask.class) .withTask("monitorUndoRollout", MonitorKatoTask.class) - .withTask(UpdateManifestForceCacheRefreshTask.TASK_NAME, UpdateManifestForceCacheRefreshTask.class); + .withTask(UpdateManifestForceCacheRefreshTask.TASK_NAME, UpdateManifestForceCacheRefreshTask.class) + .withTask(UpdateManifestWaitForStableTask.TASK_NAME, UpdateManifestWaitForStableTask.class); } } diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/manifest/UpdateManifestWaitForStableTask.java b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/manifest/UpdateManifestWaitForStableTask.java new file mode 100644 index 0000000000..7e4c644ec6 --- /dev/null +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/manifest/UpdateManifestWaitForStableTask.java @@ -0,0 +1,92 @@ +/* + * Copyright 2017 Google, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.spinnaker.orca.clouddriver.tasks.manifest; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import com.netflix.spinnaker.orca.ExecutionStatus; +import com.netflix.spinnaker.orca.OverridableTimeoutRetryableTask; +import com.netflix.spinnaker.orca.TaskResult; +import com.netflix.spinnaker.orca.clouddriver.OortService; +import com.netflix.spinnaker.orca.clouddriver.model.Manifest; +import com.netflix.spinnaker.orca.clouddriver.utils.CloudProviderAware; +import com.netflix.spinnaker.orca.pipeline.model.Stage; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import retrofit.RetrofitError; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +@Component +@Slf4j +public class UpdateManifestWaitForStableTask implements OverridableTimeoutRetryableTask, CloudProviderAware { + public final static String TASK_NAME = "waitForStable"; + + @Autowired + OortService oortService; + + @Autowired + ObjectMapper objectMapper; + + @Override + public long getBackoffPeriod() { + return TimeUnit.SECONDS.toMillis(5); + } + + @Override + public long getTimeout() { + return TimeUnit.MINUTES.toMillis(30); + } + + + @Override + public TaskResult execute(Stage stage) { + String account = getCredentials(stage); + String name = (String) stage.getContext().get("manifest.name"); + String location = (String) stage.getContext().get("manifest.location"); + String id = String.format("'%s' in '%s' for account %s", name, location, account); + List messages = new ArrayList<>(); + + Manifest manifest; + try { + manifest = oortService.getManifest(account, location, name); + } catch (RetrofitError e) { + log.warn("Unable to read manifest {}", id, e); + return new TaskResult(ExecutionStatus.RUNNING, new HashMap<>(), new HashMap<>()); + } catch (Exception e) { + throw new RuntimeException("Execution '" + stage.getExecution().getId() + "' failed with unexpected reason: " + e.getMessage(), e); + } + + Manifest.Status status = manifest.getStatus(); + if (status.getStable() == null || !status.getStable().isState()) { + messages.add(status.getStable().getMessage()); + Map context = new ImmutableMap.Builder() + .put("stableMessages", messages) + .build(); + + return new TaskResult(ExecutionStatus.RUNNING, context, new HashMap<>()); + } + + return new TaskResult(ExecutionStatus.SUCCEEDED); + } +} diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/manifest/WaitForManifestStableTask.java b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/manifest/WaitForManifestStableTask.java index 17f63fc956..4593461b29 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/manifest/WaitForManifestStableTask.java +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/manifest/WaitForManifestStableTask.java @@ -31,10 +31,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import retrofit.RetrofitError; -import retrofit.client.Response; import javax.annotation.Nonnull; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -54,7 +52,7 @@ public class WaitForManifestStableTask implements OverridableTimeoutRetryableTas @Override public long getBackoffPeriod() { - return TimeUnit.SECONDS.toMillis(1); + return TimeUnit.SECONDS.toMillis(5); } @Override @@ -74,33 +72,20 @@ public TaskResult execute(@Nonnull Stage stage) { String location = entry.getKey(); for (String name : entry.getValue()) { String identifier = readableIdentifier(account, location, name); - Response response; - try { - response = oortService.getManifest(account, location, name); - } catch (RetrofitError e) { - allStable = false; - log.warn("Unable to read manifest " + identifier, e); - continue; - } - - if (response.getStatus() != 200) { - allStable = false; - messages.add(identifier + ": could not retrieve status"); - continue; - } - Manifest manifest; try { - manifest = objectMapper.readValue(response.getBody().in(), Manifest.class); - } catch (IOException e) { - log.error("Failed to read " + identifier, e); - throw new RuntimeException(e); + manifest = oortService.getManifest(account, location, name); + } catch (RetrofitError e) { + log.warn("Unable to read manifest {}", identifier, e); + return new TaskResult(ExecutionStatus.RUNNING, new HashMap<>(), new HashMap<>()); + } catch (Exception e) { + throw new RuntimeException("Execution '" + stage.getExecution().getId() + "' failed with unexpected reason: " + e.getMessage(), e); } Status status = manifest.getStatus(); - if (!status.isStable()) { + if (status.getStable() == null || !status.getStable().isState()) { allStable = false; - messages.add(identifier + ": " + status.getMessage()); + messages.add(identifier + ": " + status.getStable().getMessage()); } } }