Skip to content

Commit

Permalink
make this betetr
Browse files Browse the repository at this point in the history
  • Loading branch information
wsorenson committed Mar 30, 2017
1 parent 6f2849d commit acad6b5
Show file tree
Hide file tree
Showing 3 changed files with 58 additions and 29 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,9 @@

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.google.common.base.Optional;

@JsonIgnoreProperties(ignoreUnknown = true)
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY)
@JsonSubTypes({
@JsonSubTypes.Type(value = EmbeddedArtifact.class, name = "EmbeddedArtifact"),
@JsonSubTypes.Type(value = S3Artifact.class, name = "S3Artifact"),
@JsonSubTypes.Type(value = S3ArtifactSignature.class, name = "S3ArtifactSignature"),
@JsonSubTypes.Type(value = ExternalArtifact.class, name = "ExternalArtifact")
})
public abstract class Artifact {

private final String name;
Expand Down
43 changes: 43 additions & 0 deletions SingularityBase/src/main/java/com/hubspot/deploy/ArtifactList.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package com.hubspot.deploy;

import java.util.List;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.hubspot.mesos.JavaUtils;

public class ArtifactList {

private final List<EmbeddedArtifact> embeddedArtifacts;
private final List<ExternalArtifact> externalArtifacts;
private final List<S3Artifact> s3Artifacts;
private final List<S3ArtifactSignature> s3ArtifactSignatures;

@JsonCreator
public ArtifactList(@JsonProperty("embeddedArtifacts") List<EmbeddedArtifact> embeddedArtifacts,
@JsonProperty("externalArtifacts") List<ExternalArtifact> externalArtifacts,
@JsonProperty("s3Artifacts") List<S3Artifact> s3Artifacts,
@JsonProperty("s3ArtifactSignatures") List<S3ArtifactSignature> s3ArtifactSignatures) {
this.embeddedArtifacts = JavaUtils.nonNullImmutable(embeddedArtifacts);
this.externalArtifacts = JavaUtils.nonNullImmutable(externalArtifacts);
this.s3Artifacts = JavaUtils.nonNullImmutable(s3Artifacts);
this.s3ArtifactSignatures = JavaUtils.nonNullImmutable(s3ArtifactSignatures);
}

public List<EmbeddedArtifact> getEmbeddedArtifacts() {
return embeddedArtifacts;
}

public List<ExternalArtifact> getExternalArtifacts() {
return externalArtifacts;
}

public List<S3Artifact> getS3Artifacts() {
return s3Artifacts;
}

public List<S3ArtifactSignature> getS3ArtifactSignatures() {
return s3ArtifactSignatures;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,11 @@
import org.apache.mesos.Protos.TaskInfo;
import org.apache.mesos.Protos.TaskState;

import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Optional;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.hubspot.deploy.Artifact;
import com.hubspot.deploy.ArtifactList;
import com.hubspot.deploy.EmbeddedArtifact;
import com.hubspot.deploy.ExecutorData;
import com.hubspot.deploy.ExternalArtifact;
Expand Down Expand Up @@ -92,27 +91,22 @@ public ProcessBuilder call() throws Exception {
executorData.getS3ArtifactSignaturesOrEmpty(), executorData.getExternalArtifacts());
task.getArtifactVerifier().checkSignatures(executorData.getS3ArtifactSignaturesOrEmpty());

List<Artifact> listedArtifacts = new ArrayList<>();
checkArtifactForArtifactList(executorData.getS3Artifacts(), listedArtifacts);
checkArtifactForArtifactList(executorData.getS3ArtifactSignaturesOrEmpty(), listedArtifacts);
checkArtifactForArtifactList(executorData.getExternalArtifacts(), listedArtifacts);
List<ArtifactList> artifactLists = new ArrayList<>();
artifactLists.addAll(checkArtifactsForArtifactLists(executorData.getS3Artifacts()));
artifactLists.addAll(checkArtifactsForArtifactLists(executorData.getS3ArtifactSignaturesOrEmpty()));
artifactLists.addAll(checkArtifactsForArtifactLists(executorData.getExternalArtifacts()));

if (!listedArtifacts.isEmpty()) {
if (!artifactLists.isEmpty()) {
List<EmbeddedArtifact> embeddedArtifacts = new ArrayList<>();
List<S3Artifact> s3Artifacts = new ArrayList<>();
List<S3ArtifactSignature> s3ArtifactSignatures = new ArrayList<>();
List<ExternalArtifact> externalArtifacts = new ArrayList<>();

for (Artifact artifact : listedArtifacts) {
if (artifact instanceof EmbeddedArtifact) {
embeddedArtifacts.add((EmbeddedArtifact) artifact);
} else if (artifact instanceof S3Artifact) {
s3Artifacts.add((S3Artifact) artifact);
} else if (artifact instanceof S3ArtifactSignature) {
s3ArtifactSignatures.add((S3ArtifactSignature) artifact);
} else if (artifact instanceof ExternalArtifact) {
externalArtifacts.add((ExternalArtifact) artifact);
}
for (ArtifactList artifactList : artifactLists) {
embeddedArtifacts.addAll(artifactList.getEmbeddedArtifacts());
s3Artifacts.addAll(artifactList.getS3Artifacts());
s3ArtifactSignatures.addAll(artifactList.getS3ArtifactSignatures());
externalArtifacts.addAll(artifactList.getExternalArtifacts());
}

taskArtifactFetcher.get().fetchFiles(embeddedArtifacts, s3Artifacts, s3ArtifactSignatures, externalArtifacts);
Expand All @@ -126,21 +120,22 @@ public ProcessBuilder call() throws Exception {
return processBuilder;
}

private void checkArtifactForArtifactList(List<? extends RemoteArtifact> remoteArtifacts, List<Artifact> foundArtifacts) {
private List<ArtifactList> checkArtifactsForArtifactLists(List<? extends RemoteArtifact> remoteArtifacts) {
List<ArtifactList> artifactLists = new ArrayList<>();
for (RemoteArtifact remoteArtifact : remoteArtifacts) {
if (remoteArtifact.isArtifactList()) {
Path pathToArtifact = task.getArtifactPath(remoteArtifact, task.getTaskDefinition().getTaskAppDirectoryPath()).resolve(remoteArtifact.getFilename());
if (!Files.exists(pathToArtifact)) {
throw new IllegalStateException(String.format("Couldn't find artifact at %s - %s", pathToArtifact, remoteArtifact));
}
try {
List<Artifact> listOfArtifacts = objectMapper.readValue(pathToArtifact.toFile(), new TypeReference<List<Artifact>>() {});
foundArtifacts.addAll(listOfArtifacts);
artifactLists.add(objectMapper.readValue(pathToArtifact.toFile(), ArtifactList.class));
} catch (IOException e) {
throw new RuntimeException("Couldn't read artifacts from " + pathToArtifact, e);
}
}
}
return artifactLists;
}

public void cancel() {
Expand Down

1 comment on commit acad6b5

@ssalinas
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚢

Please sign in to comment.