Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Che server and workpaces exposed on the same single TCP port #4351

Merged
merged 2 commits into from
Mar 10, 2017
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -209,9 +209,14 @@ che.docker.ip.external=NULL
# - 'default': internal address is address of docker host and ephemeral port are used
# - 'docker-local': internal address is address of container within docker network, and exposed ports
# are used.
# - 'single-port': internal address is set as in docker-local strategy, external address is composed
# using the workspace server name, workspace ID and Che external address
# (e.g terminal.79rfwhqaztq2ru2k.che.local)
# The 'docker-local' strategy may be useful if a firewall prevents communication between che-server and
# workspace containers, but will prevent communication when che-server and workspace containers are not
# on the same Docker network.
# The 'single-port' strategy may be useful when Che and the workspace servers need to be exposed on the
# same single TCP port.
che.docker.server_evaluation_strategy=default

# Provides a Docker network where Che server is running.
2 changes: 2 additions & 0 deletions plugins/plugin-docker/che-plugin-docker-machine/pom.xml
Original file line number Diff line number Diff line change
@@ -161,6 +161,8 @@
<exclude>**/DefaultServerEvaluationStrategyTest.java</exclude>
<exclude>**/LocalDockerServerEvaluationStrategy.java</exclude>
<exclude>**/LocalDockerServerEvaluationStrategyTest.java</exclude>
<exclude>**/LocalDockerSinglePortServerEvaluationStrategy.java</exclude>
<exclude>**/LocalDockerSinglePortServerEvaluationStrategyTest.java</exclude>
<exclude>**/DockerInstanceRuntimeInfo.java</exclude>
<exclude>**/DockerInstanceRuntimeInfoTest.java</exclude>
<!-- End excluded files -->
Original file line number Diff line number Diff line change
@@ -60,7 +60,8 @@ protected Map<String, String> getInternalAddressesAndPorts(ContainerInfo contain
}

@Override
protected Map<String, String> getExternalAddressesAndPorts(ContainerInfo containerInfo, String internalHost) {
protected Map<String, String> getExternalAddressesAndPorts(ContainerInfo containerInfo,
String internalHost) {
String externalAddress = externalAddressProperty != null ?
externalAddressProperty :
internalAddressProperty != null ?
Original file line number Diff line number Diff line change
@@ -84,15 +84,19 @@ protected Map<String, String> getInternalAddressesAndPorts(ContainerInfo contain
}

@Override
protected Map<String, String> getExternalAddressesAndPorts(ContainerInfo containerInfo, String internalHost) {
String externalAddressContainer = containerInfo.getNetworkSettings().getGateway();

String externalAddress = externalAddressProperty != null ?
externalAddressProperty :
!isNullOrEmpty(externalAddressContainer) ?
externalAddressContainer :
internalHost;
protected Map<String, String> getExternalAddressesAndPorts(ContainerInfo containerInfo,
String internalHost) {
String cheExternalAddress = getCheExternalAddress(containerInfo, internalHost);
return getExposedPortsToAddressPorts(cheExternalAddress, containerInfo.getNetworkSettings().getPorts());
}

return getExposedPortsToAddressPorts(externalAddress, containerInfo.getNetworkSettings().getPorts());
protected String getCheExternalAddress(ContainerInfo containerInfo, String internalHost) {
String externalAddressContainer = containerInfo.getNetworkSettings().getGateway();
return externalAddressProperty != null ?
externalAddressProperty :
!isNullOrEmpty(externalAddressContainer) ?
externalAddressContainer :
internalHost;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
/*******************************************************************************
* Copyright (c) 2016-2017 Red Hat Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Red Hat Inc. - initial API and implementation
*******************************************************************************/

package org.eclipse.che.plugin.docker.machine;

import com.google.inject.Inject;
import com.google.inject.name.Named;
import org.eclipse.che.api.machine.server.model.impl.ServerConfImpl;
import org.eclipse.che.api.machine.server.model.impl.ServerImpl;
import org.eclipse.che.commons.annotation.Nullable;
import org.eclipse.che.plugin.docker.client.json.ContainerInfo;
import org.eclipse.che.plugin.docker.client.json.PortBinding;

import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;


/**
* Represents a server evaluation strategy for the configuration where the workspace server and workspace
* containers are running on the same Docker network and are exposed through the same single port.
*
* This server evaluation strategy will return a completed {@link ServerImpl} with internal addresses set
* as {@link LocalDockerServerEvaluationStrategy} does. Contrarily external addresses will have the following format:
*
* serverName.workspaceID.cheExternalAddress (e.g. terminal.79rfwhqaztq2ru2k.che.local)
*
* <p>cheExternalAddress can be set using property {@code che.docker.ip.external}.
* This strategy is useful when Che and the workspace servers need to be exposed on the same single TCP port
*
* @author Mario Loriedo <mloriedo@redhat.com>
* @see ServerEvaluationStrategy
*/
public class LocalDockerSinglePortServerEvaluationStrategy extends LocalDockerServerEvaluationStrategy {

private static final String CHE_WORKSPACE_ID_ENV_VAR = "CHE_WORKSPACE_ID";

@Inject
public LocalDockerSinglePortServerEvaluationStrategy(@Nullable @Named("che.docker.ip") String internalAddress,
@Nullable @Named("che.docker.ip.external") String externalAddress) {
super(internalAddress, externalAddress);
}

@Override
protected Map<String, String> getExternalAddressesAndPorts(ContainerInfo containerInfo,
String internalHost) {
String cheExternalAddress = getCheExternalAddress(containerInfo, internalHost);
String workspaceID = getWorkspaceID(containerInfo.getConfig().getEnv());
Map<String, String> labels= containerInfo.getConfig().getLabels();

Map<String, List<PortBinding>> portBindings = containerInfo.getNetworkSettings().getPorts();
Map<String, String> addressesAndPorts = new HashMap<>();
for (String serverKey : portBindings.keySet()) {
String serverName = getWorkspaceServerName(labels, serverKey);
String serverURL = serverName + "." + workspaceID + "." + cheExternalAddress;
addressesAndPorts.put(serverKey, serverURL);
}

return addressesAndPorts;
}

private String getWorkspaceServerName(Map<String, String> labels, String portKey) {
ServerConfImpl serverConf = getServerConfImpl(portKey, labels, new HashMap<>());
if (serverConf == null) {
return "server-" + portKey.split("/",2)[0];
}
return serverConf.getRef();
}

private String getWorkspaceID(String[] env) {
Stream<String> envStream = Arrays.stream(env);
return envStream.filter(v -> v.startsWith(CHE_WORKSPACE_ID_ENV_VAR) && v.contains("=")).
map(v -> v.split("=",2)[1]).
findFirst().
orElse("unknown-ws").
replaceFirst("workspace","");
}
}
Original file line number Diff line number Diff line change
@@ -156,7 +156,7 @@ public Map<String, ServerImpl> getServers(ContainerInfo containerInfo,
* @return {@code ServerConfImpl}, obtained from {@code serverConfMap} if possible,
* or from {@code labels} if there is no entry in {@code serverConfMap}.
*/
private ServerConfImpl getServerConfImpl(String portProtocol,
protected ServerConfImpl getServerConfImpl(String portProtocol,
Map<String, String> labels,
Map<String, ServerConfImpl> serverConfMap) {
// Label can be specified without protocol -- e.g. 4401 refers to 4401/tcp
Original file line number Diff line number Diff line change
@@ -23,6 +23,7 @@
import org.eclipse.che.plugin.docker.machine.DockerInstance;
import org.eclipse.che.plugin.docker.machine.DockerInstanceRuntimeInfo;
import org.eclipse.che.plugin.docker.machine.DockerProcess;
import org.eclipse.che.plugin.docker.machine.LocalDockerSinglePortServerEvaluationStrategy;
import org.eclipse.che.plugin.docker.machine.ServerEvaluationStrategy;
import org.eclipse.che.plugin.docker.machine.node.DockerNode;
import org.eclipse.che.plugin.openshift.client.OpenShiftConnector;
@@ -55,7 +56,9 @@ protected void configure() {
strategies.addBinding("default")
.to(org.eclipse.che.plugin.docker.machine.DefaultServerEvaluationStrategy.class);
strategies.addBinding("docker-local")
.to(org.eclipse.che.plugin.docker.machine.LocalDockerServerEvaluationStrategy.class);
.to(org.eclipse.che.plugin.docker.machine.LocalDockerServerEvaluationStrategy.class);
strategies.addBinding("single-port")
.to(LocalDockerSinglePortServerEvaluationStrategy.class);

bind(org.eclipse.che.plugin.docker.machine.node.WorkspaceFolderPathProvider.class)
.to(org.eclipse.che.plugin.docker.machine.local.node.provider.LocalWorkspaceFolderPathProvider.class);
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
/*******************************************************************************
* Copyright (c) 2016-2017 Red Hat Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Red Hat Inc. - initial API and implementation
*******************************************************************************/

package org.eclipse.che.plugin.docker.machine;

import org.eclipse.che.api.core.model.machine.MachineConfig;
import org.eclipse.che.api.machine.server.model.impl.ServerConfImpl;
import org.eclipse.che.api.machine.server.model.impl.ServerImpl;
import org.eclipse.che.api.machine.server.model.impl.ServerPropertiesImpl;
import org.eclipse.che.plugin.docker.client.json.ContainerConfig;
import org.eclipse.che.plugin.docker.client.json.ContainerInfo;
import org.eclipse.che.plugin.docker.client.json.NetworkSettings;
import org.eclipse.che.plugin.docker.client.json.PortBinding;
import org.mockito.Mock;
import org.mockito.testng.MockitoTestNGListener;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;

import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;

@Listeners(MockitoTestNGListener.class)
public class LocalDockerSinglePortServerEvaluationStrategyTest {

private static final String CHE_DOCKER_IP_EXTERNAL = "container-host-ext.com";
private static final String ALL_IP_ADDRESS = "0.0.0.0";
private static final String CONTAINERCONFIG_HOSTNAME = "che-ws-y6jwknht0efzczit-4086112300-fm0aj";
private static final String WORKSPACE_ID = "79rfwhqaztq2ru2k";

@Mock
private ContainerInfo containerInfo;
@Mock
private ContainerConfig containerConfig;
@Mock
private NetworkSettings networkSettings;

private ServerEvaluationStrategy strategy;

private Map<String, ServerConfImpl> serverConfs;

private Map<String, List<PortBinding>> ports;

private Map<String, String> labels;

private String[] env;

@BeforeMethod
public void setUp() {

serverConfs = new HashMap<>();
serverConfs.put("4301/tcp", new ServerConfImpl("sysServer1-tcp", "4301/tcp", "http", "/some/path1"));
serverConfs.put("4305/udp", new ServerConfImpl("devSysServer1-udp", "4305/udp", null, "some/path4"));

ports = new HashMap<>();
ports.put("4301/tcp", Collections.singletonList(new PortBinding().withHostIp(ALL_IP_ADDRESS )
.withHostPort("32100")));
ports.put("4305/udp", Collections.singletonList(new PortBinding().withHostIp(ALL_IP_ADDRESS )
.withHostPort("32103")));

labels = new HashMap<>();
labels.put("che:server:4301/tcp:ref", "sysServer1-tcp");
labels.put("che:server:4305/udp:ref", "devSysServer1-udp");

env = new String[]{"CHE_WORKSPACE_ID="+ WORKSPACE_ID};

when(containerInfo.getNetworkSettings()).thenReturn(networkSettings);
when(networkSettings.getIpAddress()).thenReturn(CONTAINERCONFIG_HOSTNAME);
when(networkSettings.getPorts()).thenReturn(ports);
when(containerInfo.getConfig()).thenReturn(containerConfig);
when(containerConfig.getHostname()).thenReturn(CONTAINERCONFIG_HOSTNAME);
when(containerConfig.getEnv()).thenReturn(env);
when(containerConfig.getLabels()).thenReturn(labels);
}

/**
* Test: single port strategy should use .
* @throws Exception
*/
@Test
public void shouldUseServerRefToBuildAddressWhenAvailable() throws Exception {
// given
strategy = new LocalDockerSinglePortServerEvaluationStrategy(null, null);

final Map<String, ServerImpl> expectedServers = getExpectedServers(CHE_DOCKER_IP_EXTERNAL,
CONTAINERCONFIG_HOSTNAME,
true);

// when
final Map<String, ServerImpl> servers = strategy.getServers(containerInfo,
CHE_DOCKER_IP_EXTERNAL,
serverConfs);

// then
assertEquals(servers, expectedServers);
}

private Map<String, ServerImpl> getExpectedServers(String externalAddress,
String internalAddress,
boolean useExposedPorts) {
String port1;
String port2;
if (useExposedPorts) {
port1 = ":4301";
port2 = ":4305";
} else {
port1 = ":32100";
port2 = ":32103";
}
Map<String, ServerImpl> expectedServers = new HashMap<>();
expectedServers.put("4301/tcp", new ServerImpl("sysServer1-tcp",
"http",
"sysServer1-tcp." + WORKSPACE_ID + "." + externalAddress,
"http://" + "sysServer1-tcp." + WORKSPACE_ID + "." + externalAddress + "/some/path1",
new ServerPropertiesImpl("/some/path1",
internalAddress + port1,
"http://" + internalAddress + port1 + "/some/path1")));
expectedServers.put("4305/udp", new ServerImpl("devSysServer1-udp",
null,
"devSysServer1-udp." + WORKSPACE_ID + "." + externalAddress,
null,
new ServerPropertiesImpl("some/path4",
internalAddress + port2,
null)));
return expectedServers;
}

}
Original file line number Diff line number Diff line change
@@ -34,8 +34,10 @@
import javax.inject.Named;
import javax.inject.Singleton;


import io.fabric8.openshift.api.model.Route;
import io.fabric8.openshift.api.model.RouteList;
import org.eclipse.che.commons.annotation.Nullable;
import org.eclipse.che.inject.StringArrayConverter;
import org.eclipse.che.plugin.docker.client.DockerApiVersionPathPrefixProvider;
import org.eclipse.che.plugin.docker.client.DockerConnector;
import org.eclipse.che.plugin.docker.client.DockerConnectorConfiguration;
@@ -147,7 +149,6 @@ public class OpenShiftConnector extends DockerConnector {
private static final int CHE_WORKSPACE_AGENT_PORT = 4401;
private static final int CHE_TERMINAL_AGENT_PORT = 4411;
private static final String DOCKER_PROTOCOL_PORT_DELIMITER = "/";
private static final String OPENSHIFT_SERVICE_TYPE_NODE_PORT = "NodePort";
private static final int OPENSHIFT_WAIT_POD_DELAY = 1000;
private static final int OPENSHIFT_WAIT_POD_TIMEOUT = 240;
private static final int OPENSHIFT_IMAGESTREAM_WAIT_DELAY = 2000;
@@ -171,12 +172,14 @@ public class OpenShiftConnector extends DockerConnector {
private final String workspacesPvcQuantity;
private final String cheWorkspaceStorage;
private final String cheWorkspaceProjectsStorage;
private final String cheServerExternalAddress;

@Inject
public OpenShiftConnector(DockerConnectorConfiguration connectorConfiguration,
DockerConnectionFactory connectionFactory,
DockerRegistryAuthResolver authResolver,
DockerApiVersionPathPrefixProvider dockerApiVersionPathPrefixProvider,
@Named("che.docker.ip.external") String cheServerExternalAddress,
@Named("che.openshift.project") String openShiftCheProjectName,
@Named("che.openshift.serviceaccountname") String openShiftCheServiceAccount,
@Named("che.openshift.liveness.probe.delay") int openShiftLivenessProbeDelay,
@@ -187,6 +190,7 @@ public OpenShiftConnector(DockerConnectorConfiguration connectorConfiguration,
@Named("che.workspace.projects.storage") String cheWorkspaceProjectsStorage) {

super(connectorConfiguration, connectionFactory, authResolver, dockerApiVersionPathPrefixProvider);
this.cheServerExternalAddress = cheServerExternalAddress;
this.openShiftCheProjectName = openShiftCheProjectName;
this.openShiftCheServiceAccount = openShiftCheServiceAccount;
this.openShiftLivenessProbeDelay = openShiftLivenessProbeDelay;
@@ -821,6 +825,23 @@ private Deployment getDeploymentByName(String deploymentName) throws IOException
return deployment;
}

private List<Route> getRoutesByLabel(String labelKey, String labelValue) throws IOException {
RouteList routeList = openShiftClient
.routes()
.inNamespace(this.openShiftCheProjectName)
.withLabel(labelKey, labelValue)
.list();

List<Route> items = routeList.getItems();

if (items.isEmpty()) {
LOG.warn("No Route with label {}={} could be found", labelKey, labelValue);
throw new IOException("No Route with label " + labelKey + "=" + labelValue + " could be found");
}

return items;
}

private Pod getChePodByContainerId(String containerId) throws IOException {
PodList pods = openShiftClient.pods()
.inNamespace(this.openShiftCheProjectName)
@@ -877,25 +898,62 @@ private void createOpenShiftService(String workspaceID,
Set<String> exposedPorts,
Map<String, String> additionalLabels) {

Map<String, String> selector = Collections.singletonMap(OPENSHIFT_DEPLOYMENT_LABEL, CHE_OPENSHIFT_RESOURCES_PREFIX + workspaceID);
String serviceName = CHE_OPENSHIFT_RESOURCES_PREFIX + workspaceID;

Map<String, String> selector = Collections.singletonMap(OPENSHIFT_DEPLOYMENT_LABEL, serviceName);
List<ServicePort> ports = KubernetesService.getServicePortsFrom(exposedPorts);

Service service = openShiftClient
.services()
.inNamespace(this.openShiftCheProjectName)
.createNew()
.withNewMetadata()
.withName(CHE_OPENSHIFT_RESOURCES_PREFIX + workspaceID)
.withAnnotations(KubernetesLabelConverter.labelsToNames(additionalLabels))
.withName(serviceName)
.withAnnotations(KubernetesLabelConverter.labelsToNames(additionalLabels))
.endMetadata()
.withNewSpec()
.withType(OPENSHIFT_SERVICE_TYPE_NODE_PORT)
.withSelector(selector)
.withPorts(ports)
.withSelector(selector)
.withPorts(ports)
.endSpec()
.done();

LOG.info("OpenShift service {} created", service.getMetadata().getName());

for (ServicePort port : ports) {
createOpenShiftRoute(serviceName, port.getName(), workspaceID);
}
}

private void createOpenShiftRoute(String serviceName,
String serverRef,
String workspaceName) {

String routeName = CHE_OPENSHIFT_RESOURCES_PREFIX + workspaceName + "." + serverRef;
String serviceHost = serverRef + "." + workspaceName + "." + this.cheServerExternalAddress;

Route route = openShiftClient
.routes()
.inNamespace(this.openShiftCheProjectName)
.createNew()
.withNewMetadata()
.withName(routeName)
.addToLabels(OPENSHIFT_DEPLOYMENT_LABEL,serviceName)
.endMetadata()
.withNewSpec()
.withHost(serviceHost)
.withNewTo()
.withKind("Service")
.withName(serviceName)
.endTo()
.withNewPort()
.withNewTargetPort()
.withStrVal(serverRef)
.endTargetPort()
.endPort()
.endSpec()
.done();

LOG.info("OpenShift route {} created", route.getMetadata().getName());
}

private String createOpenShiftDeployment(String workspaceID,
@@ -921,7 +979,7 @@ private String createOpenShiftDeployment(String workspaceID,
.withImagePullPolicy(OPENSHIFT_IMAGE_PULL_POLICY_IFNOTPRESENT)
.withNewSecurityContext()
.withRunAsUser(UID)
.withPrivileged(true)
.withPrivileged(false)
.endSecurityContext()
.withLivenessProbe(getLivenessProbeFrom(exposedPorts))
.withVolumeMounts(getVolumeMountsFrom(volumes, workspaceID))
@@ -1085,6 +1143,14 @@ private void cleanUpWorkspaceResources(String deploymentName) throws IOException

Deployment deployment = getDeploymentByName(deploymentName);
Service service = getCheServiceBySelector(OPENSHIFT_DEPLOYMENT_LABEL, deploymentName);
List<Route> routes = getRoutesByLabel(OPENSHIFT_DEPLOYMENT_LABEL, deploymentName);

if (routes != null) {
for (Route route: routes) {
LOG.info("Removing OpenShift Route {}", route.getMetadata().getName());
openShiftClient.resource(route).delete();
}
}

if (service != null) {
LOG.info("Removing OpenShift Service {}", service.getMetadata().getName());
Original file line number Diff line number Diff line change
@@ -39,6 +39,8 @@ public class OpenShiftConnectorTest {
private static final String OPENSHIFT_DEFAULT_WORKSPACE_QUANTITY = "10Gi";
private static final String OPENSHIFT_DEFAULT_WORKSPACE_STORAGE = "/data/workspaces";
private static final String OPENSHIFT_DEFAULT_WORKSPACE_PROJECTS_STORAGE = "/projects";
private static final String CHE_DEFAULT_SERVER_EXTERNAL_ADDRESS = "che.openshift.mini";

@Mock
private DockerConnectorConfiguration dockerConnectorConfiguration;
@Mock
@@ -66,6 +68,7 @@ public void shouldGetWorkspaceIDWhenAValidOneIsProvidedInCreateContainerParams()
dockerConnectionFactory,
authManager,
dockerApiVersionPathPrefixProvider,
CHE_DEFAULT_SERVER_EXTERNAL_ADDRESS,
CHE_DEFAULT_OPENSHIFT_PROJECT_NAME,
CHE_DEFAULT_OPENSHIFT_SERVICEACCOUNT,
OPENSHIFT_LIVENESS_PROBE_DELAY,