Skip to content

Commit

Permalink
feat(manifest): more robust status handling (#1809)
Browse files Browse the repository at this point in the history
  • Loading branch information
lwander authored Nov 17, 2017
1 parent eb20375 commit 1ed2be4
Show file tree
Hide file tree
Showing 8 changed files with 120 additions and 30 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package com.netflix.spinnaker.orca.clouddriver;

import com.netflix.spinnaker.orca.clouddriver.config.SelectableService;
import com.netflix.spinnaker.orca.clouddriver.model.Manifest;
import retrofit.client.Response;

import java.util.List;
Expand All @@ -36,7 +37,7 @@ public Response getCluster(String app, String account, String cluster, String cl
}

@Override
public Response getManifest(String account, String location, String name) {
public Manifest getManifest(String account, String location, String name) {
return getService().getManifest(account, location, name);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package com.netflix.spinnaker.orca.clouddriver

import com.netflix.spinnaker.orca.clouddriver.model.Manifest
import retrofit.client.Response
import retrofit.http.GET
import retrofit.http.Path
Expand All @@ -42,7 +43,7 @@ interface OortService {
@Path("cloudProvider") String cloudProvider)

@GET("/manifests/{account}/{location}/{manifest}")
Response getManifest(@Path("account") String account,
Manifest getManifest(@Path("account") String account,
@Path("location") String location,
@Path("manifest") String manifest)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,12 @@ public class Manifest {

@Data
public static class Status {
boolean stable;
Condition stable;
}

@Data
public static class Condition {
boolean state;
String message;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.netflix.spinnaker.orca.clouddriver.tasks.MonitorKatoTask;
import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.ResumeRolloutManifestTask;
import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.UpdateManifestForceCacheRefreshTask;
import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.UpdateManifestWaitForStableTask;
import com.netflix.spinnaker.orca.pipeline.StageDefinitionBuilder;
import com.netflix.spinnaker.orca.pipeline.TaskNode;
import com.netflix.spinnaker.orca.pipeline.model.Stage;
Expand All @@ -33,6 +34,7 @@ public class ResumeRolloutManifestStage implements StageDefinitionBuilder {
public void taskGraph(Stage stage, TaskNode.Builder builder) {
builder.withTask(ResumeRolloutManifestTask.TASK_NAME, ResumeRolloutManifestTask.class)
.withTask("monitorResumeRollout", MonitorKatoTask.class)
.withTask(UpdateManifestForceCacheRefreshTask.TASK_NAME, UpdateManifestForceCacheRefreshTask.class);
.withTask(UpdateManifestForceCacheRefreshTask.TASK_NAME, UpdateManifestForceCacheRefreshTask.class)
.withTask(UpdateManifestWaitForStableTask.TASK_NAME, UpdateManifestWaitForStableTask.class);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.netflix.spinnaker.orca.clouddriver.tasks.MonitorKatoTask;
import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.ScaleManifestTask;
import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.UpdateManifestForceCacheRefreshTask;
import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.UpdateManifestWaitForStableTask;
import com.netflix.spinnaker.orca.pipeline.StageDefinitionBuilder;
import com.netflix.spinnaker.orca.pipeline.TaskNode;
import com.netflix.spinnaker.orca.pipeline.model.Stage;
Expand All @@ -33,6 +34,7 @@ public class ScaleManifestStage implements StageDefinitionBuilder {
public void taskGraph(Stage stage, TaskNode.Builder builder) {
builder.withTask(ScaleManifestTask.TASK_NAME, ScaleManifestTask.class)
.withTask("monitorScale", MonitorKatoTask.class)
.withTask(UpdateManifestForceCacheRefreshTask.TASK_NAME, UpdateManifestForceCacheRefreshTask.class);
.withTask(UpdateManifestForceCacheRefreshTask.TASK_NAME, UpdateManifestForceCacheRefreshTask.class)
.withTask(UpdateManifestWaitForStableTask.TASK_NAME, UpdateManifestWaitForStableTask.class);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.netflix.spinnaker.orca.clouddriver.tasks.MonitorKatoTask;
import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.UndoRolloutManifestTask;
import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.UpdateManifestForceCacheRefreshTask;
import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.UpdateManifestWaitForStableTask;
import com.netflix.spinnaker.orca.pipeline.StageDefinitionBuilder;
import com.netflix.spinnaker.orca.pipeline.TaskNode;
import com.netflix.spinnaker.orca.pipeline.model.Stage;
Expand All @@ -33,6 +34,7 @@ public class UndoRolloutManifestStage implements StageDefinitionBuilder {
public void taskGraph(Stage stage, TaskNode.Builder builder) {
builder.withTask(UndoRolloutManifestTask.TASK_NAME, UndoRolloutManifestTask.class)
.withTask("monitorUndoRollout", MonitorKatoTask.class)
.withTask(UpdateManifestForceCacheRefreshTask.TASK_NAME, UpdateManifestForceCacheRefreshTask.class);
.withTask(UpdateManifestForceCacheRefreshTask.TASK_NAME, UpdateManifestForceCacheRefreshTask.class)
.withTask(UpdateManifestWaitForStableTask.TASK_NAME, UpdateManifestWaitForStableTask.class);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
/*
* Copyright 2017 Google, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License")
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package com.netflix.spinnaker.orca.clouddriver.tasks.manifest;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableMap;
import com.netflix.spinnaker.orca.ExecutionStatus;
import com.netflix.spinnaker.orca.OverridableTimeoutRetryableTask;
import com.netflix.spinnaker.orca.TaskResult;
import com.netflix.spinnaker.orca.clouddriver.OortService;
import com.netflix.spinnaker.orca.clouddriver.model.Manifest;
import com.netflix.spinnaker.orca.clouddriver.utils.CloudProviderAware;
import com.netflix.spinnaker.orca.pipeline.model.Stage;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import retrofit.RetrofitError;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;

@Component
@Slf4j
public class UpdateManifestWaitForStableTask implements OverridableTimeoutRetryableTask, CloudProviderAware {
public final static String TASK_NAME = "waitForStable";

@Autowired
OortService oortService;

@Autowired
ObjectMapper objectMapper;

@Override
public long getBackoffPeriod() {
return TimeUnit.SECONDS.toMillis(5);
}

@Override
public long getTimeout() {
return TimeUnit.MINUTES.toMillis(30);
}


@Override
public TaskResult execute(Stage stage) {
String account = getCredentials(stage);
String name = (String) stage.getContext().get("manifest.name");
String location = (String) stage.getContext().get("manifest.location");
String id = String.format("'%s' in '%s' for account %s", name, location, account);
List<String> messages = new ArrayList<>();

Manifest manifest;
try {
manifest = oortService.getManifest(account, location, name);
} catch (RetrofitError e) {
log.warn("Unable to read manifest {}", id, e);
return new TaskResult(ExecutionStatus.RUNNING, new HashMap<>(), new HashMap<>());
} catch (Exception e) {
throw new RuntimeException("Execution '" + stage.getExecution().getId() + "' failed with unexpected reason: " + e.getMessage(), e);
}

Manifest.Status status = manifest.getStatus();
if (status.getStable() == null || !status.getStable().isState()) {
messages.add(status.getStable().getMessage());
Map<String, Object> context = new ImmutableMap.Builder<String, Object>()
.put("stableMessages", messages)
.build();

return new TaskResult(ExecutionStatus.RUNNING, context, new HashMap<>());
}

return new TaskResult(ExecutionStatus.SUCCEEDED);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,8 @@
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import retrofit.RetrofitError;
import retrofit.client.Response;

import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
Expand All @@ -54,7 +52,7 @@ public class WaitForManifestStableTask implements OverridableTimeoutRetryableTas

@Override
public long getBackoffPeriod() {
return TimeUnit.SECONDS.toMillis(1);
return TimeUnit.SECONDS.toMillis(5);
}

@Override
Expand All @@ -74,33 +72,20 @@ public TaskResult execute(@Nonnull Stage stage) {
String location = entry.getKey();
for (String name : entry.getValue()) {
String identifier = readableIdentifier(account, location, name);
Response response;
try {
response = oortService.getManifest(account, location, name);
} catch (RetrofitError e) {
allStable = false;
log.warn("Unable to read manifest " + identifier, e);
continue;
}

if (response.getStatus() != 200) {
allStable = false;
messages.add(identifier + ": could not retrieve status");
continue;
}

Manifest manifest;
try {
manifest = objectMapper.readValue(response.getBody().in(), Manifest.class);
} catch (IOException e) {
log.error("Failed to read " + identifier, e);
throw new RuntimeException(e);
manifest = oortService.getManifest(account, location, name);
} catch (RetrofitError e) {
log.warn("Unable to read manifest {}", identifier, e);
return new TaskResult(ExecutionStatus.RUNNING, new HashMap<>(), new HashMap<>());
} catch (Exception e) {
throw new RuntimeException("Execution '" + stage.getExecution().getId() + "' failed with unexpected reason: " + e.getMessage(), e);
}

Status status = manifest.getStatus();
if (!status.isStable()) {
if (status.getStable() == null || !status.getStable().isState()) {
allStable = false;
messages.add(identifier + ": " + status.getMessage());
messages.add(identifier + ": " + status.getStable().getMessage());
}
}
}
Expand Down

0 comments on commit 1ed2be4

Please sign in to comment.