Envoy support (part 1)

cloud-resource-manager/platform/dind/dind-appinst.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/mobiledgex/edge-cloud/cloud-resource-manager/dockermgmt"
 	"github.com/mobiledgex/edge-cloud/cloud-resource-manager/k8smgmt"
-	"github.com/mobiledgex/edge-cloud/cloud-resource-manager/nginx"
+	"github.com/mobiledgex/edge-cloud/cloud-resource-manager/proxy"
 	"github.com/mobiledgex/edge-cloud/cloud-resource-manager/platform/pc"
 	"github.com/mobiledgex/edge-cloud/cloudcommon"
 	"github.com/mobiledgex/edge-cloud/edgeproto"
@@ -36,21 +36,21 @@ func (s *Platform) CreateAppInst(ctx context.Context, clusterInst *edgeproto.Clu
 	}
 	// NOTE: for DIND we don't check whether this is internal
 	if len(appInst.MappedPorts) > 0 {
-		log.SpanLog(ctx, log.DebugLevelMexos, "AddNginxProxy for dind", "ports", appInst.MappedPorts)
+		log.SpanLog(ctx, log.DebugLevelMexos, "Add Proxy for dind", "ports", appInst.MappedPorts)
 		cluster, err := FindCluster(names.ClusterName)
 		if err != nil {
 			return err
 		}
 		masterIP := cluster.MasterAddr
 		network := GetDockerNetworkName(cluster)
-		err = nginx.CreateNginxProxy(client,
+		err = proxy.CreateNginxProxy(ctx, client,
 			names.AppName,
 			masterIP,
 			appInst.MappedPorts,
-			nginx.WithDockerNetwork(network),
-			nginx.WithDockerPublishPorts())
+			proxy.WithDockerNetwork(network),
+			proxy.WithDockerPublishPorts())
 		if err != nil {
-			log.SpanLog(ctx, log.DebugLevelMexos, "cannot add nginx proxy", "appName", names.AppName, "ports", appInst.MappedPorts)
+			log.SpanLog(ctx, log.DebugLevelMexos, "cannot add proxy", "appName", names.AppName, "ports", appInst.MappedPorts)
 			return err
 		}
 	}
@@ -105,8 +105,8 @@ func (s *Platform) DeleteAppInst(ctx context.Context, clusterInst *edgeproto.Clu
 
 	if len(appInst.MappedPorts) > 0 {
 		log.SpanLog(ctx, log.DebugLevelMexos, "DeleteNginxProxy for dind")
-		if err = nginx.DeleteNginxProxy(client, names.AppName); err != nil {
-			log.SpanLog(ctx, log.DebugLevelMexos, "cannot delete nginx proxy", "name", names.AppName)
+		if err = proxy.DeleteNginxProxy(ctx, client, names.AppName); err != nil {
+			log.SpanLog(ctx, log.DebugLevelMexos, "cannot delete proxy", "name", names.AppName)
 			return err
 		}
 	}

cloud-resource-manager/platform/dind/dind-cluster.go

@@ -11,7 +11,7 @@ import (
 
 	sh "github.com/codeskyblue/go-sh"
 	"github.com/mobiledgex/edge-cloud/cloud-resource-manager/k8smgmt"
-	"github.com/mobiledgex/edge-cloud/cloud-resource-manager/nginx"
+	"github.com/mobiledgex/edge-cloud/cloud-resource-manager/proxy"
 	"github.com/mobiledgex/edge-cloud/cloudcommon"
 	"github.com/mobiledgex/edge-cloud/edgeproto"
 	"github.com/mobiledgex/edge-cloud/log"
@@ -114,7 +114,7 @@ func (s *Platform) CreateDINDCluster(ctx context.Context, clusterName, kconfName
 
 	// bridge nginxL7 network to this cluster's network
 	out, err = sh.Command("docker", "network", "connect",
-		GetDockerNetworkName(&cluster), nginx.NginxL7Name).CombinedOutput()
+		GetDockerNetworkName(&cluster), proxy.NginxL7Name).CombinedOutput()
 	if err != nil && strings.Contains(string(out), "already exists") {
 		err = nil
 	}
@@ -143,7 +143,7 @@ func (s *Platform) DeleteDINDCluster(ctx context.Context, clusterInst *edgeproto
 
 	// disconnect nginxL7 network
 	out, err := sh.Command("docker", "network", "disconnect",
-		GetDockerNetworkName(cluster), nginx.NginxL7Name).CombinedOutput()
+		GetDockerNetworkName(cluster), proxy.NginxL7Name).CombinedOutput()
 	if err != nil && strings.Contains(string(out), "is not connected") {
 		err = nil
 	}

cloud-resource-manager/platform/dind/dind.go

@@ -3,7 +3,7 @@ package dind
 import (
 	"context"
 
-	"github.com/mobiledgex/edge-cloud/cloud-resource-manager/nginx"
+	"github.com/mobiledgex/edge-cloud/cloud-resource-manager/proxy"
 	"github.com/mobiledgex/edge-cloud/cloud-resource-manager/platform"
 	"github.com/mobiledgex/edge-cloud/cloud-resource-manager/platform/pc"
 	"github.com/mobiledgex/edge-cloud/edgeproto"
@@ -23,7 +23,7 @@ func (s *Platform) Init(ctx context.Context, platformConfig *platform.PlatformCo
 		return err
 	}
 	updateCallback(edgeproto.UpdateTask, "Setting up Nginx L7 Proxy")
-	err = nginx.InitL7Proxy(client, nginx.WithDockerPublishPorts())
+	err = proxy.InitL7Proxy(ctx, client, proxy.WithDockerPublishPorts())
 	if err != nil {
 		return err
 	}

cloud-resource-manager/proxy/envoy.go

@@ -0,0 +1,209 @@
+package proxy
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"html/template"
+	"strings"
+
+	"github.com/mobiledgex/edge-cloud/cloud-resource-manager/dockermgmt"
+	"github.com/mobiledgex/edge-cloud/cloud-resource-manager/platform/pc"
+	"github.com/mobiledgex/edge-cloud/cloudcommon"
+	dme "github.com/mobiledgex/edge-cloud/d-match-engine/dme-proto"
+	"github.com/mobiledgex/edge-cloud/log"
+)
+
+// Envoy is now handling all of our L4 TCP loadbalancing
+// Eventually L7 and UDP Proxying support will be added so
+// all of our loadbalancing is handled by envoy.
+// UDP proxying is currently blocked by: https://github.com/envoyproxy/envoy/issues/492
+
+var envoyYamlT *template.Template
+
+// this is the default value in envoy, for DOS protection
+const defaultConcurrentConns uint64 = 1024
+
+func init() {
+	envoyYamlT = template.Must(template.New("yaml").Parse(envoyYaml))
+}
+
+func CreateEnvoyProxy(ctx context.Context, client pc.PlatformClient, name, originIP string, ports []dme.AppPort, ops ...Op) error {
+	log.SpanLog(ctx, log.DebugLevelMexos, "create envoy", "name", name, "originip", originIP, "ports", ports)
+	opts := Options{}
+	opts.Apply(ops)
+
+	out, err := client.Output("pwd")
+	if err != nil {
+		return err
+	}
+	pwd := strings.TrimSpace(string(out))
+
+	dir := pwd + "/envoy/" + name
+	log.SpanLog(ctx, log.DebugLevelMexos, "envoy remote dir", "name", name, "dir", dir)
+
+	err = pc.Run(client, "mkdir -p "+dir)
+	if err != nil {
+		return err
+	}
+	accesslogFile := dir + "/access.log"
+	err = pc.Run(client, "touch "+accesslogFile)
+	if err != nil {
+		log.SpanLog(ctx, log.DebugLevelMexos,
+			"envoy %s can't create file %s", name, accesslogFile)
+		return err
+	}
+	eyamlName := dir + "/envoy.yaml"
+	err = createEnvoyYaml(ctx, client, eyamlName, name, originIP, ports)
+	if err != nil {
+		return fmt.Errorf("create envoy.yaml failed, %v", err)
+	}
+
+	// container name is envoy+name for now to avoid conflicts with the nginx containers
+	cmdArgs := []string{"run", "-d", "-l edge-cloud", "--restart=unless-stopped", "--name", "envoy" + name}
+	if opts.DockerPublishPorts {
+		cmdArgs = append(cmdArgs, dockermgmt.GetDockerPortString(ports, dockermgmt.UsePublicPortInContainer)...)
+	}
+	if opts.DockerNetwork != "" {
+		// For dind, we use the network which the dind cluster is on.
+		cmdArgs = append(cmdArgs, "--network", opts.DockerNetwork)
+	}
+	cmdArgs = append(cmdArgs, []string{
+		"-v", accesslogFile + ":/var/log/access.log",
+		"-v", eyamlName + ":/etc/envoy/envoy.yaml",
+		"docker.mobiledgex.net/mobiledgex/mobiledgex_public/envoy-with-curl"}...)
+	cmd := "docker " + strings.Join(cmdArgs, " ")
+	log.SpanLog(ctx, log.DebugLevelMexos, "envoy docker command", "name", "envoy"+name,
+		"cmd", cmd)
+	out, err = client.Output(cmd)
+	if err != nil {
+		return fmt.Errorf("can't create envoy container %s, %s, %v", "envoy"+name, out, err)
+	}
+	log.SpanLog(ctx, log.DebugLevelMexos, "created envoy container", "name", name)
+	return nil
+}
+
+func createEnvoyYaml(ctx context.Context, client pc.PlatformClient, yamlname, name, originIP string, ports []dme.AppPort) error {
+	spec := ProxySpec{
+		Name:       name,
+		MetricPort: cloudcommon.ProxyMetricsPort,
+	}
+	for _, p := range ports {
+		switch p.Proto {
+		// only support tcp for now
+		case dme.LProto_L_PROTO_TCP:
+			tcpPort := TCPSpecDetail{
+				Port:       p.PublicPort,
+				Origin:     originIP,
+				OriginPort: p.InternalPort,
+			}
+			tcpconns, err := getTCPConcurrentConnections()
+			if err != nil {
+				return err
+			}
+			tcpPort.ConcurrentConns = tcpconns
+			spec.TCPSpec = append(spec.TCPSpec, &tcpPort)
+			spec.L4 = true
+		}
+	}
+
+	log.SpanLog(ctx, log.DebugLevelMexos, "create envoy yaml", "name", name)
+	buf := bytes.Buffer{}
+	err := envoyYamlT.Execute(&buf, &spec)
+	if err != nil {
+		return err
+	}
+	err = pc.WriteFile(client, yamlname, buf.String(), "envoy.yaml", pc.NoSudo)
+	if err != nil {
+		log.SpanLog(ctx, log.DebugLevelMexos, "write envoy.yaml failed",
+			"name", name, "err", err)
+		return err
+	}
+	return nil
+}
+
+// TODO: Probably should eventually find a better way to uniquely name clusters other than just by the port theyre getting proxied from
+var envoyYaml = `
+{{if .L4 -}}
+static_resources:
+  listeners:
+  {{- range .TCPSpec}}
+  - address:
+      socket_address:
+        address: 0.0.0.0
+        port_value: {{.Port}}
+    filter_chains:
+    - filters:
+      - name: envoy.tcp_proxy
+        config:
+          stat_prefix: ingress_tcp
+          cluster: backend{{.Port}}
+          access_log:
+            - name: envoy.file_access_log
+              config:
+                path: /var/log/access.log
+                json_format: {
+                  "start_time": "%START_TIME%",
+                  "duration": "%DURATION%",
+                  "bytes_sent": "%BYTES_SENT%",
+                  "bytes_received": "%BYTES_RECEIVED%",
+                  "client_address": "%DOWNSTREAM_REMOTE_ADDRESS%",
+                  "upstream_cluster": "%UPSTREAM_CLUSTER%"
+				}
+  {{- end}}
+  clusters:
+  {{- range .TCPSpec}}
+  - name: backend{{.Port}}
+    connect_timeout: 0.25s
+    type: strict_dns
+    circuit_breakers:
+        thresholds:
+            max_connections: {{.ConcurrentConns}}
+    lb_policy: round_robin
+    hosts:
+    - socket_address:
+        address: {{.Origin}}
+        port_value: {{.OriginPort}}
+{{- end}}
+admin:
+  access_log_path: "/var/log/admin.log"
+  address:
+    socket_address:
+      address: 0.0.0.0
+      port_value: {{.MetricPort}}
+{{- end}}
+`
+
+func DeleteEnvoyProxy(ctx context.Context, client pc.PlatformClient, name string) error {
+	log.SpanLog(ctx, log.DebugLevelMexos, "delete envoy", "name", "envoy"+name)
+	out, err := client.Output("docker kill " + "envoy" + name)
+	deleteContainer := false
+	if err == nil {
+		deleteContainer = true
+	} else {
+		if strings.Contains(string(out), "No such container") {
+			log.SpanLog(ctx, log.DebugLevelMexos,
+				"envoy LB container already gone", "name", "envoy"+name)
+		} else {
+			return fmt.Errorf("can't delete envoy container %s, %s, %v", name, out, err)
+		}
+	}
+	envoyDir := "envoy/" + name
+	out, err = client.Output("rm -rf " + envoyDir)
+	if err != nil {
+		log.SpanLog(ctx, log.DebugLevelMexos, "delete envoy dir", "name", name, "dir", envoyDir, "out", out, "err", err)
+	}
+	if deleteContainer {
+		out, err = client.Output("docker rm " + "envoy" + name)
+		if err != nil && !strings.Contains(string(out), "No such container") {
+			return fmt.Errorf("can't remove envoy container %s, %s, %v", "envoy"+name, out, err)
+		}
+	}
+
+	log.SpanLog(ctx, log.DebugLevelMexos, "deleted envoy", "name", name)
+	return nil
+}
+
+func GetEnvoyContainerName(name string) string {
+	return "envoy" + name
+}