Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

CHE-398: Add ability to create local machine snapshots without registry #1351

Merged
merged 1 commit into from
May 27, 2016
Merged

CHE-398: Add ability to create local machine snapshots without registry #1351

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/codenvy/che.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,10 @@ machine.docker.privilege_mode=false
# address and provide this host or IP address here.
machine.docker.local_node_host=NULL

# Allows to use registry for machine snapshots, you should set this property to {true},
# otherwise workspace snapshots would be saved locally.
machine.docker.snapshot_use_registry=false

# URL path to api service.
# Browser clients use this to initiate REST communications with workspace master
api.endpoint=http://localhost:${SERVER_PORT}/wsmaster/api
Expand Down Expand Up @@ -146,7 +150,7 @@ machine.server.terminal.path_to_archive.linux_arm7=${che.home}/lib/linux_arm7/te

# During the stop of the workspace automatically creates a snapshot if the value is {true},
# otherwise just stops the workspace.
workspace.runtime.auto_snapshot=false
workspace.runtime.auto_snapshot=true
# During the start of the workspace automatically restored it from a snapshot if the value is {true},
# otherwise just creates the new workspace.
workspace.runtime.auto_restore=false
workspace.runtime.auto_restore=true
65 changes: 42 additions & 23 deletions ...in-docker-machine/src/main/java/org/eclipse/che/plugin/docker/machine/DockerInstance.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
*******************************************************************************/
package org.eclipse.che.plugin.docker.machine;

import com.google.common.annotations.VisibleForTesting;
import com.google.inject.assistedinject.Assisted;

import org.eclipse.che.api.core.NotFoundException;
Expand All @@ -18,17 +19,20 @@
import org.eclipse.che.api.core.util.LineConsumer;
import org.eclipse.che.api.core.util.ListLineConsumer;
import org.eclipse.che.api.machine.server.exception.MachineException;
import org.eclipse.che.api.machine.server.spi.impl.AbstractInstance;
import org.eclipse.che.api.machine.server.model.impl.MachineRuntimeInfoImpl;
import org.eclipse.che.api.machine.server.spi.Instance;
import org.eclipse.che.api.machine.server.spi.InstanceKey;
import org.eclipse.che.api.machine.server.spi.InstanceProcess;
import org.eclipse.che.api.machine.server.spi.impl.AbstractInstance;
import org.eclipse.che.commons.lang.NameGenerator;
import org.eclipse.che.plugin.docker.client.DockerConnector;
import org.eclipse.che.plugin.docker.client.Exec;
import org.eclipse.che.plugin.docker.client.LogMessage;
import org.eclipse.che.plugin.docker.client.ProgressLineFormatterImpl;
import org.eclipse.che.plugin.docker.client.json.ContainerInfo;
import org.eclipse.che.plugin.docker.client.params.CommitParams;
import org.eclipse.che.plugin.docker.client.params.PushParams;
import org.eclipse.che.plugin.docker.client.params.RemoveImageParams;
import org.eclipse.che.plugin.docker.machine.node.DockerNode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -82,6 +86,7 @@ public class DockerInstance extends AbstractInstance {
private final DockerInstanceStopDetector dockerInstanceStopDetector;
private final DockerInstanceProcessesCleaner processesCleaner;
private final ConcurrentHashMap<Integer, InstanceProcess> machineProcesses;
private final boolean snapshotUseRegistry;

private MachineRuntimeInfoImpl machineRuntime;

Expand All @@ -95,7 +100,8 @@ public DockerInstance(DockerConnector docker,
@Assisted DockerNode node,
@Assisted LineConsumer outputConsumer,
DockerInstanceStopDetector dockerInstanceStopDetector,
DockerInstanceProcessesCleaner processesCleaner) {
DockerInstanceProcessesCleaner processesCleaner,
@Named("machine.docker.snapshot_use_registry") boolean snapshotUseRegistry) {
super(machine);
this.dockerMachineFactory = dockerMachineFactory;
this.container = container;
Expand All @@ -108,6 +114,7 @@ public DockerInstance(DockerConnector docker,
this.processesCleaner = processesCleaner;
this.machineProcesses = new ConcurrentHashMap<>();
processesCleaner.trackProcesses(this);
this.snapshotUseRegistry = snapshotUseRegistry;
}

@Override
Expand Down Expand Up @@ -187,36 +194,48 @@ public InstanceProcess createProcess(Command command, String outputChannel) thro
public InstanceKey saveToSnapshot(String owner) throws MachineException {
try {
final String repository = generateRepository();
String comment = format("Suspended at %1$ta %1$tb %1$td %1$tT %1$tZ %1$tY", System.currentTimeMillis());
if (owner != null) {
comment = comment + " by " + owner;
final String tag = "latest";
if(!snapshotUseRegistry) {
commitContainer(owner, repository, tag);
return new DockerInstanceKey(repository, tag);
}
// !! We SHOULD NOT pause container before commit because all execs will fail
// to push image to private registry it should be tagged with registry in repo name
// https://docs.docker.com/reference/api/docker_remote_api_v1.16/#push-an-image-on-the-registry
docker.commit(container, registry + '/' + repository, "latest", comment, owner);
final String repositoryName = registry + '/' + repository;
commitContainer(owner, repositoryName, tag);
//TODO fix this workaround. Docker image is not visible after commit when using swarm
Thread.sleep(2000);

final ProgressLineFormatterImpl progressLineFormatter = new ProgressLineFormatterImpl();
String digest = docker.push(repository, "latest", registry, currentProgressStatus -> {
try {
outputConsumer.writeLine(progressLineFormatter.format(currentProgressStatus));
} catch (IOException ignored) {
}
});

docker.removeImage(registry + '/' + repository, false);

return new DockerInstanceKey(repository, "latest", registry, digest);
} catch (IOException e) {
throw new MachineException(e);
final ProgressLineFormatterImpl lineFormatter = new ProgressLineFormatterImpl();
final String digest = docker.push(PushParams.create(repository)
.withTag(tag)
.withRegistry(registry),
progressMonitor -> {
try {
outputConsumer.writeLine(lineFormatter.format(progressMonitor));
} catch (IOException ignored) {
}
});
docker.removeImage(RemoveImageParams.create(repositoryName).withForce(false));
return new DockerInstanceKey(repository, tag, registry, digest);
} catch (IOException ioEx) {
throw new MachineException(ioEx);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new MachineException(e.getLocalizedMessage(), e);
}
}

@VisibleForTesting
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why VisibleForTesting and not only protected access instead of package access ?
also there is no test calling this method so I don't see why it should be "VisibleForTesting"

void commitContainer(String owner, String repository, String tag) throws IOException {
String comment = format("Suspended at %1$ta %1$tb %1$td %1$tT %1$tZ %1$tY",
System.currentTimeMillis());
comment = owner == null ? comment : comment + " by " + owner;
// !! We SHOULD NOT pause container before commit because all execs will fail
// to push image to private registry it should be tagged with registry in repo name
// https://docs.docker.com/reference/api/docker_remote_api_v1.16/#push-an-image-on-the-registry
docker.commit(CommitParams.create(container, repository)
.withTag(tag)
.withComment(comment));
}

private String generateRepository() {
return NameGenerator.generate(null, 16);
}
Expand Down
4 changes: 4 additions & 0 deletions ...docker-machine/src/main/java/org/eclipse/che/plugin/docker/machine/DockerInstanceKey.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ public DockerInstanceKey(String repository, String tag, String registry, String
super(ImmutableMap.of(REPOSITORY, repository, TAG, tag, REGISTRY, registry, DIGEST, digest));
}

public DockerInstanceKey(String repository, String tag) {
super(ImmutableMap.of(REPOSITORY, repository, TAG, tag));
}

public String getRepository() {
return getFields().get(REPOSITORY);
}
Expand Down
26 changes: 18 additions & 8 deletions ...r-machine/src/main/java/org/eclipse/che/plugin/docker/machine/DockerInstanceProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import org.eclipse.che.plugin.docker.client.ProgressMonitor;
import org.eclipse.che.plugin.docker.client.json.ContainerConfig;
import org.eclipse.che.plugin.docker.client.json.HostConfig;
import org.eclipse.che.plugin.docker.client.params.RemoveImageParams;
import org.eclipse.che.plugin.docker.machine.node.DockerNode;
import org.eclipse.che.plugin.docker.machine.node.WorkspaceFolderPathProvider;
import org.slf4j.Logger;
Expand Down Expand Up @@ -93,6 +94,7 @@ public class DockerInstanceProvider implements InstanceProvider {
private final Set<String> commonMachineEnvVariables;
private final String[] allMachinesExtraHosts;
private final String projectFolderPath;
private final boolean snapshotUseRegistry;

@Inject
public DockerInstanceProvider(DockerConnector docker,
Expand All @@ -110,9 +112,8 @@ public DockerInstanceProvider(DockerConnector docker,
@Named("machine.docker.pull_image") boolean doForcePullOnBuild,
@Named("machine.docker.privilege_mode") boolean privilegeMode,
@Named("machine.docker.dev_machine.machine_env") Set<String> devMachineEnvVariables,
@Named("machine.docker.machine_env") Set<String> allMachinesEnvVariables)
throws IOException {

@Named("machine.docker.machine_env") Set<String> allMachinesEnvVariables,
@Named("machine.docker.snapshot_use_registry") boolean snapshotUseRegistry) throws IOException {
this.docker = docker;
this.dockerMachineFactory = dockerMachineFactory;
this.dockerInstanceStopDetector = dockerInstanceStopDetector;
Expand All @@ -122,6 +123,7 @@ public DockerInstanceProvider(DockerConnector docker,
this.privilegeMode = privilegeMode;
this.supportedRecipeTypes = Collections.singleton("dockerfile");
this.projectFolderPath = projectFolderPath;
this.snapshotUseRegistry = snapshotUseRegistry;

allMachinesSystemVolumes = removeEmptyAndNullValues(allMachinesSystemVolumes);
devMachineSystemVolumes = removeEmptyAndNullValues(devMachineSystemVolumes);
Expand Down Expand Up @@ -239,9 +241,9 @@ public Instance createInstance(InstanceKey instanceKey,
Machine machine,
LineConsumer creationLogsOutput) throws NotFoundException, MachineException {
final DockerInstanceKey dockerInstanceKey = new DockerInstanceKey(instanceKey);

pullImage(dockerInstanceKey, creationLogsOutput);

if (snapshotUseRegistry) {
pullImage(dockerInstanceKey, creationLogsOutput);
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This code is not covered with tests

}
final String userName = EnvironmentContext.getCurrent().getSubject().getUserName();
final String machineContainerName = containerNameGenerator.generateContainerName(machine.getWorkspaceId(),
machine.getId(),
Expand Down Expand Up @@ -361,8 +363,16 @@ public void removeInstanceSnapshot(InstanceKey instanceKey) throws SnapshotExcep
// use registry API directly because docker doesn't have such API yet
// https://github.com/docker/docker-registry/issues/45
final DockerInstanceKey dockerInstanceKey = new DockerInstanceKey(instanceKey);
String registry = dockerInstanceKey.getRegistry();
String repository = dockerInstanceKey.getRepository();
if (!snapshotUseRegistry) {
try {
docker.removeImage(RemoveImageParams.create(dockerInstanceKey.getFullName()));
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This code is not covered with tests

} catch (IOException ignore) {
}
return;
}

final String registry = dockerInstanceKey.getRegistry();
final String repository = dockerInstanceKey.getRepository();
if (registry == null || repository == null) {
LOG.error("Failed to remove instance snapshot: invalid instance key: {}", instanceKey);
throw new SnapshotException("Snapshot removing failed. Snapshot attributes are not valid");
Expand Down
Loading