Allow to customize ingress controller specific annotations for ingresses

Signed-off-by: Oleksandr Garagatyi <ogaragat@redhat.com>

assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties

@@ -390,6 +390,7 @@ che.infra.kubernetes.pvc.access_mode=ReadWriteOnce
 # then OpenShift infrastructure will reconfigure installer to use first available from this range
 che.infra.kubernetes.installer_server_min_port=10000
 che.infra.kubernetes.installer_server_max_port=20000
+che.infra.kubernetes.ingress.annotations_json=NULL
 
 #
 che.infra.kubernetes.pod.security_context.run_as_user=NULL

dockerfiles/init/modules/kubernetes/Deploy Che.md

@@ -2,9 +2,29 @@
 Tested on minikube with vm provider Virtualbox. Note that Che with workspaces requires quite a lot 
 of RAM. Initial tests were done with 10GB, but it is definitely more than it is needed to start Che 
 and couple of workspaces.
+
 IP of VM is supposed to be `192.168.99.100`. `nip.io` is also used for handling hosts resolution.
 If you have another IP or DNS replace these values in k8s.yml file.
 
+Services are exposed using ingress controller approach. 
+We added ingress annotations to customize ingress controller behavior - 
+not to break websocket connections.
+In particular testing environment was setup with NginX ingress controller 0.9.0-beta.17.
+So we added annotations specific to this implementation and version:
+- ingress.kubernetes.io/rewrite-target: /
+- ingress.kubernetes.io/proxy-read-timeout: "3600"
+- ingress.kubernetes.io/proxy-connect-timeout: "3600"
+
+If you use another ingress controller implementation or version you need to customize 
+Che master ingress and value of environment variable `CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON` stored in ConfigMap. 
+Value of the map should be expressed as a stringified JSON. For example most recent NginX controller uses other annotations:
+- nginx.ingress.kubernetes.io/rewrite-target
+- nginx.ingress.kubernetes.io/proxy-read-timeout
+- nginx.ingress.kubernetes.io/proxy-connect-timeout
+- nginx.ingress.kubernetes.io/ssl-redirect
+
+And environment variable would be: `'{"ingress.kubernetes.io/rewrite-target": "/","ingress.kubernetes.io/ssl-redirect": "false","ingress.kubernetes.io/proxy-connect-timeout": "3600","ingress.kubernetes.io/proxy-read-timeout": "3600"}'`
+
 ###Prerequisites:
 - Ingress controller is running. Note: you can start it on minikube with `minikube addons enable ingress`.
 - Currently Che workspaces work with NginX ingress controller only. Note: it is default ingress controller on minikube.

dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml

@@ -62,17 +62,21 @@ items:
     CHE_LOGS_DIR: /data/logs
     CHE_LOG_LEVEL: "INFO"
     CHE_MULTIUSER: "false"
+    CHE_SINGLE_PORT: "false"
     CHE_OAUTH_GITHUB_CLIENTID: ""
     CHE_OAUTH_GITHUB_CLIENTSECRET: ""
     CHE_PREDEFINED_STACKS_RELOAD__ON__START: "false"
     JAVA_OPTS: "-XX:MaxRAMFraction=2 -XX:+UseParallelGC -XX:MinHeapFreeRatio=10 -XX:MaxHeapFreeRatio=20 -XX:GCTimeRatio=4 -XX:AdaptiveSizePolicyWeight=90 -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -Dsun.zip.disableMemoryMapping=true -Xms20m "
     CHE_WORKSPACE_AUTO_START: "false"
+    CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON: '{"ingress.kubernetes.io/rewrite-target": "/","ingress.kubernetes.io/ssl-redirect": "false","ingress.kubernetes.io/proxy-connect-timeout": "3600","ingress.kubernetes.io/proxy-read-timeout": "3600"}'
 - apiVersion: extensions/v1beta1
   kind: Ingress
   metadata:
     name: che-ingress
     annotations:
       ingress.kubernetes.io/rewrite-target: /
+      ingress.kubernetes.io/proxy-read-timeout: "3600"
+      ingress.kubernetes.io/proxy-connect-timeout: "3600"
   spec:
     rules:
     - host: 192.168.99.100.nip.io
@@ -239,7 +243,17 @@ items:
               configMapKeyRef:
                 key: CHE_WORKSPACE_AUTO_START
                 name: che
-          image: eclipse/che-server:nightly
+          - name: CHE_SINGLE_PORT
+            valueFrom:
+              configMapKeyRef:
+                key: CHE_SINGLE_PORT
+                name: che
+          - name: CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON
+            valueFrom:
+              configMapKeyRef:
+                key: CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON
+                name: che
+          image: garagatyi/che-server2:nightly
           imagePullPolicy: Always
           livenessProbe:
             httpGet:

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java

@@ -14,9 +14,11 @@
 import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.UniqueWorkspacePVCStrategy.UNIQUE_STRATEGY;
 
 import com.google.inject.AbstractModule;
+import com.google.inject.TypeLiteral;
 import com.google.inject.assistedinject.FactoryModuleBuilder;
 import com.google.inject.multibindings.MapBinder;
 import com.google.inject.multibindings.Multibinder;
+import java.util.Map;
 import org.eclipse.che.api.workspace.server.spi.RuntimeInfrastructure;
 import org.eclipse.che.api.workspace.server.spi.environment.InternalEnvironmentFactory;
 import org.eclipse.che.api.workspace.server.spi.provision.env.CheApiEnvVarProvider;
@@ -34,6 +36,7 @@
 import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy;
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.KubernetesCheApiEnvVarProvider;
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.LogsRootEnvVariableProvider;
+import org.eclipse.che.workspace.infrastructure.kubernetes.server.IngressAnnotationsProvider;
 
 /** @author Sergii Leshchenko */
 public class KubernetesInfraModule extends AbstractModule {
@@ -65,5 +68,9 @@ protected void configure() {
     Multibinder<EnvVarProvider> envVarProviders =
         Multibinder.newSetBinder(binder(), EnvVarProvider.class);
     envVarProviders.addBinding().to(LogsRootEnvVariableProvider.class);
+
+    bind(new TypeLiteral<Map<String, String>>() {})
+        .annotatedWith(com.google.inject.name.Names.named("infra.kubernetes.ingress.annotations"))
+        .toProvider(IngressAnnotationsProvider.class);
   }
 }

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/server/ServersConverter.java

@@ -14,6 +14,8 @@
 import io.fabric8.kubernetes.api.model.Pod;
 import io.fabric8.kubernetes.api.model.PodSpec;
 import java.util.Map;
+import javax.inject.Inject;
+import javax.inject.Named;
 import javax.inject.Singleton;
 import org.eclipse.che.api.core.model.workspace.config.ServerConfig;
 import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
@@ -36,6 +38,14 @@
 @Singleton
 public class ServersConverter implements ConfigurationProvisioner {
 
+  private final Map<String, String> ingressAnnotations;
+
+  @Inject
+  public ServersConverter(
+      @Named("infra.kubernetes.ingress.annotations") Map<String, String> ingressAnnotations) {
+    this.ingressAnnotations = ingressAnnotations;
+  }
+
   @Override
   public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity)
       throws InfrastructureException {
@@ -47,7 +57,8 @@ public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity)
         InternalMachineConfig machineConfig = k8sEnv.getMachines().get(machineName);
         if (!machineConfig.getServers().isEmpty()) {
           KubernetesServerExposer kubernetesServerExposer =
-              new KubernetesServerExposer<>(machineName, podConfig, containerConfig, k8sEnv);
+              new KubernetesServerExposer<>(
+                  ingressAnnotations, machineName, podConfig, containerConfig, k8sEnv);
           kubernetesServerExposer.expose(machineConfig.getServers());
         }
       }

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/IngressAnnotationsProvider.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2012-2018 Red Hat, Inc.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *   Red Hat, Inc. - initial API and implementation
+ */
+package org.eclipse.che.workspace.infrastructure.kubernetes.server;
+
+import com.google.common.reflect.TypeToken;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import java.lang.reflect.Type;
+import java.util.Collections;
+import java.util.Map;
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.inject.Provider;
+import javax.inject.Singleton;
+import org.eclipse.che.commons.annotation.Nullable;
+
+/** @author Alexander Garagatyi */
+@Singleton
+public class IngressAnnotationsProvider implements Provider<Map<String, String>> {
+
+  private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create();
+  private static final Type type = new TypeToken<Map<String, String>>() {}.getType();
+
+  private Map<String, String> annotations;
+
+  @Inject
+  public IngressAnnotationsProvider(
+      @Nullable @Named("che.infra.kubernetes.ingress.annotations_json") String annotationsString) {
+
+    if (annotationsString != null) {
+      annotations = GSON.fromJson(annotationsString, type);
+    } else {
+      annotations = Collections.emptyMap();
+    }
+  }
+
+  @Override
+  public Map<String, String> get() {
+    return annotations;
+  }
+}

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposer.java

@@ -45,6 +45,7 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
+import javax.inject.Named;
 import org.eclipse.che.api.core.model.workspace.config.ServerConfig;
 import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
 import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations;
@@ -125,13 +126,19 @@ public class KubernetesServerExposer<T extends KubernetesEnvironment> {
   public static final int SERVER_UNIQUE_PART_SIZE = 8;
   public static final String SERVER_PREFIX = "server";
 
+  private final Map<String, String> ingressAnnotations;
   protected final String machineName;
   protected final Container container;
   protected final Pod pod;
   protected final T kubernetesEnvironment;
 
   public KubernetesServerExposer(
-      String machineName, Pod pod, Container container, T kubernetesEnvironment) {
+      @Named("infra.kubernetes.ingress.annotations") Map<String, String> ingressAnnotations,
+      String machineName,
+      Pod pod,
+      Container container,
+      T kubernetesEnvironment) {
+    this.ingressAnnotations = ingressAnnotations;
     this.machineName = machineName;
     this.pod = pod;
     this.container = container;
@@ -196,6 +203,7 @@ protected void exposeExternalServers(
               .withName(serviceName + '-' + servicePort.getName())
               .withMachineName(machineName)
               .withServiceName(serviceName)
+              .withAnnotations(ingressAnnotations)
               .withServicePort(servicePort.getName())
               .withServers(ingressesServers)
               .build();
@@ -299,6 +307,7 @@ private static class IngressBuilder {
     private IntOrString servicePort;
     private Map<String, ? extends ServerConfig> serversConfigs;
     private String machineName;
+    private Map<String, String> annotations;
 
     private IngressBuilder withName(String name) {
       this.name = name;
@@ -310,6 +319,11 @@ private IngressBuilder withServiceName(String serviceName) {
       return this;
     }
 
+    private IngressBuilder withAnnotations(Map<String, String> annotations) {
+      this.annotations = annotations;
+      return this;
+    }
+
     private IngressBuilder withServicePort(String targetPortName) {
       this.servicePort = new IntOrString(targetPortName);
       return this;
@@ -342,10 +356,7 @@ private Ingress build() {
       IngressRule ingressRule = new IngressRuleBuilder().withHttp(httpIngressRuleValue).build();
       IngressSpec ingressSpec = new IngressSpecBuilder().withRules(ingressRule).build();
 
-      Map<String, String> ingressAnnotations = new HashMap<>();
-      ingressAnnotations.put("ingress.kubernetes.io/rewrite-target", "/");
-      ingressAnnotations.put("ingress.kubernetes.io/ssl-redirect", "false");
-      ingressAnnotations.put("kubernetes.io/ingress.class", "nginx");
+      Map<String, String> ingressAnnotations = new HashMap<>(annotations);
       ingressAnnotations.putAll(
           Annotations.newSerializer()
               .servers(serversConfigs)

infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposerTest.java

@@ -10,6 +10,7 @@
  */
 package org.eclipse.che.workspace.infrastructure.kubernetes.server;
 
+import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
 import static org.eclipse.che.workspace.infrastructure.kubernetes.server.KubernetesServerExposer.SERVER_PREFIX;
@@ -38,14 +39,17 @@
 import org.eclipse.che.api.workspace.server.model.impl.ServerConfigImpl;
 import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations;
 import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
+import org.mockito.testng.MockitoTestNGListener;
 import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Listeners;
 import org.testng.annotations.Test;
 
 /**
  * Test for {@link KubernetesServerExposer}.
  *
  * @author Sergii Leshchenko
  */
+@Listeners(MockitoTestNGListener.class)
 public class KubernetesServerExposerTest {
 
   private static final Map<String, String> ATTRIBUTES_MAP = singletonMap("key", "value");
@@ -76,7 +80,8 @@ public void setUp() throws Exception {
     kubernetesEnvironment =
         KubernetesEnvironment.builder().setPods(ImmutableMap.of("pod", pod)).build();
     this.serverExposer =
-        new KubernetesServerExposer<>(MACHINE_NAME, pod, container, kubernetesEnvironment);
+        new KubernetesServerExposer<>(
+            emptyMap(), MACHINE_NAME, pod, container, kubernetesEnvironment);
   }
 
   @Test

infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/server/OpenShiftServerExposer.java

@@ -19,6 +19,7 @@
 import io.fabric8.kubernetes.api.model.Service;
 import io.fabric8.kubernetes.api.model.ServicePort;
 import io.fabric8.openshift.api.model.Route;
+import java.util.Collections;
 import java.util.Map;
 import org.eclipse.che.api.core.model.workspace.config.ServerConfig;
 import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations;
@@ -94,7 +95,7 @@ public class OpenShiftServerExposer extends KubernetesServerExposer<OpenShiftEnv
 
   public OpenShiftServerExposer(
       String machineName, Pod pod, Container container, OpenShiftEnvironment openShiftEnvironment) {
-    super(machineName, pod, container, openShiftEnvironment);
+    super(Collections.emptyMap(), machineName, pod, container, openShiftEnvironment);
     this.openShiftEnvironment = openShiftEnvironment;
   }