Merge pull request #40 from salehsedghpour/feature/grpc-support

Feature/grpc support

generator/input/new_description.json

@@ -8,11 +8,11 @@
   ],
   "services": [
     {
-      "name": "service-1",
+      "name": "service1",
       "clusters": [
         {
           "cluster": "cluster-1",
-          "namespace": "ns-1",
+          "namespace": "default",
           "node": "node-1"
         }
       ],
@@ -29,17 +29,17 @@
       "processes": 2,
       "endpoints": [
         {
-          "name": "/end-1",
+          "name": "end1",
           "protocol": "http",
           "cpu_consumption": 0.003,
           "network_consumption": 0.002,
           "memory_consumption": 0.003,
           "forward_requests": "asynchronous",
           "called_services": [
             {
-              "service": "service-2",
+              "service": "service2",
               "port": "80",
-              "endpoint": "/end-2",
+              "endpoint": "end2",
               "protocol": "http",
               "traffic_forward_ratio": 1
             }

generator/src/pkg/generate/generate.go

@@ -18,12 +18,14 @@ package generate
 import (
 	"application-generator/src/pkg/model"
 	s "application-generator/src/pkg/service"
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"gopkg.in/yaml.v3"
 	"io/ioutil"
 	"os"
 	"strings"
+	"text/template"
 )
 
 const (
@@ -33,8 +35,6 @@ const (
 	imageName = "app"
 	imageURL  = "app-demo:latest"
 
-	protocol = "http"
-
 	defaultExtPort = 80
 	defaultPort    = 5000
 
@@ -59,18 +59,18 @@ var (
 
 type CalledServices struct {
 	Service             string  `json:"service"`
-	Port           			string	`json:"port"`
+	Port                string  `json:"port"`
 	Endpoint            string  `json:"endpoint"`
 	Protocol            string  `json:"protocol"`
 	TrafficForwardRatio float32 `json:"traffic_forward_ratio"`
 }
 type Endpoints struct {
 	Name               string           `json:"name"`
-	Protocol           string						`json:"protocol"`
+	Protocol           string           `json:"protocol"`
 	CpuConsumption     float64          `json:"cpu_consumption"`
 	NetworkConsumption float64          `json:"network_consumption"`
 	MemoryConsumption  float64          `json:"memory_consumption"`
-	ForwardRequests    string  					`json:"forward_requests"`
+	ForwardRequests    string           `json:"forward_requests"`
 	CalledServices     []CalledServices `json:"called_services"`
 }
 type ResourceLimits struct {
@@ -89,7 +89,7 @@ type Services struct {
 	Name      string      `json:"name"`
 	Clusters  []Clusters  `json:"clusters"`
 	Resources Resources   `json:"resources"`
-	Processes	int					`json:"processes"`
+	Processes int         `json:"processes"`
 	Endpoints []Endpoints `json:"endpoints"`
 }
 
@@ -111,8 +111,8 @@ type Config struct {
 }
 
 type ConfigMap struct {
-	Processes	int					`json:"processes"`
-	Endpoints []Endpoints	`json:"endpoints"`
+	Processes int         `json:"processes"`
+	Endpoints []Endpoints `json:"endpoints"`
 }
 
 // the slices to store services, cluster and endpoints for counting and printing
@@ -166,16 +166,23 @@ func Parse(configFilename string) (Config, []string) {
 
 func Create(config Config, readinessProbe int, clusters []string) {
 	path, _ := os.Getwd()
+	proto_temp, _ := template.ParseFiles(path + "/template/service.tmpl")
 	path = path + "/k8s"
 
 	for i := 0; i < len(clusters); i++ {
 		directory := fmt.Sprintf(path+"/%s", clusters[i])
 		os.Mkdir(directory, 0777)
 	}
-
+	var proto_temp_filled_byte bytes.Buffer
+	err := proto_temp.Execute(&proto_temp_filled_byte, config.Services)
+	if err != nil {
+		panic(err)
+	}
+	proto_temp_filled := proto_temp_filled_byte.String()
 	for i := 0; i < len(config.Services); i++ {
 		serv := config.Services[i].Name
 		resources := Resources(config.Services[i].Resources)
+		protocol := config.Services[i].Endpoints[0].Protocol
 
 		if resources.Limits.Cpu == "" {
 			resources.Limits.Cpu = limitsCPUDefault
@@ -216,18 +223,19 @@ func Create(config Config, readinessProbe int, clusters []string) {
 				manifests = append(manifests, string(yamlDoc))
 				return nil
 			}
-			configmap = s.CreateConfig("config-"+serv, "config-"+serv, c_id, namespace, string(serv_json))
+			configmap = s.CreateConfig("config-"+serv, "config-"+serv, c_id, namespace, string(serv_json), proto_temp_filled)
 			appendManifest(configmap)
 			if nodeAffinity == "" {
 				deployment := s.CreateDeployment(serv, serv, c_id, replicaNumber, serv, c_id, namespace,
 					defaultPort, imageName, imageURL, volumePath, volumeName, "config-"+serv, readinessProbe,
-					resources.Requests.Cpu, resources.Requests.Memory, resources.Limits.Cpu, resources.Limits.Memory)
+					resources.Requests.Cpu, resources.Requests.Memory, resources.Limits.Cpu, resources.Limits.Memory, protocol)
 
 				appendManifest(deployment)
 			} else {
 				deployment := s.CreateDeploymentWithAffinity(serv, serv, c_id, replicaNumber, serv, c_id, namespace,
 					defaultPort, imageName, imageURL, volumePath, volumeName, "config-"+serv, readinessProbe,
-					resources.Requests.Cpu, resources.Requests.Memory, resources.Limits.Cpu, resources.Limits.Memory, nodeAffinity)
+					resources.Requests.Cpu, resources.Requests.Memory, resources.Limits.Cpu, resources.Limits.Memory,
+					nodeAffinity, protocol)
 				appendManifest(deployment)
 			}
 

generator/src/pkg/model/config.go

@@ -21,13 +21,14 @@ type ConfigMapInstance struct {
 	Metadata   struct {
 		Name   string `yaml:"name"`
 		Labels struct {
-			Name string `yaml:"name"`
-			Cluster string  `yaml:"version,omitempty"`
+			Name    string `yaml:"name"`
+			Cluster string `yaml:"version,omitempty"`
 		} `yaml:"labels"`
 		Namespace string `yaml:"namespace"`
 	} `yaml:"metadata"`
 	Data struct {
-		Config string `yaml:"conf.json"`
+		Config  string `yaml:"conf.json"`
+		Service string `yaml:"service.proto"`
 	} `yaml:"data"`
 }
 

generator/src/pkg/model/deployment.go

@@ -100,6 +100,7 @@ type ContainerInstance struct {
 	Name            string                    `yaml:"name"`
 	Image           string                    `yaml:"image"`
 	ImagePullPolicy string                    `yaml:"imagePullPolicy"`
+	Env             []EnvInstance             `yaml:"env"`
 	Ports           []ContainerPortInstance   `yaml:"ports"`
 	Volumes         []ContainerVolumeInstance `yaml:"volumeMounts"`
 	ReadinessProbe  ReadinessProbeInstance    `yaml:"readinessProbe,omitempty"`
@@ -110,16 +111,24 @@ type ContainerPortInstance struct {
 	ContainerPort int `yaml:"containerPort"`
 }
 
+type EnvInstance struct {
+	Name  string `yaml:"name"`
+	Value string `yaml:"value"`
+}
+
 type ContainerVolumeInstance struct {
 	MountPath string `yaml:"mountPath,omitempty"`
 	MountName string `yaml:"name,omitempty"`
 }
 
 type ReadinessProbeInstance struct {
 	HttpGet struct {
-		Path string `yaml:"path"`
-		Port int    `yaml:"port"`
-	} `yaml:"httpGet"`
+		Path string `yaml:"path,omitempty"`
+		Port int    `yaml:"port,omitempty"`
+	} `yaml:"httpGet,omitempty"`
+	Exec struct {
+		Command []string `yaml:"command,flow,omitempty"`
+	} `yaml:"exec,omitempty"`
 	InitialDelaySeconds int `yaml:"initialDelaySeconds"`
 	PeriodSeconds       int `yaml:"periodSeconds"`
 }

generator/src/pkg/service/util.go

@@ -23,15 +23,18 @@ import (
 func CreateDeployment(metadataName, selectorAppName, selectorClusterName string, numberOfReplicas int,
 	templateAppLabel, templateClusterLabel, namespace string, containerPort int, containerName, containerImage,
 	mountPath string, volumeName, configMapName string, readinessProbe int, requestCPU, requestMemory, limitCPU,
-	limitMemory string) (deploymentInstance model.DeploymentInstance) {
+	limitMemory, protocol string) (deploymentInstance model.DeploymentInstance) {
 
 	var deployment model.DeploymentInstance
 	var containerInstance model.ContainerInstance
+	var envInstance model.EnvInstance
 
 	var containerPortInstance model.ContainerPortInstance
 	var containerVolume model.ContainerVolumeInstance
 	var volumeInstance model.VolumeInstance
 
+	envInstance.Name = "SERVICE_NAME"
+	envInstance.Value = metadataName
 	containerPortInstance.ContainerPort = containerPort
 	volumeInstance.Name = volumeName
 	volumeInstance.ConfigMap.Name = configMapName
@@ -44,8 +47,14 @@ func CreateDeployment(metadataName, selectorAppName, selectorClusterName string,
 	containerInstance.Name = containerName
 	containerInstance.Image = containerImage
 	containerInstance.ImagePullPolicy = "Never"
-	containerInstance.ReadinessProbe.HttpGet.Path = "/"
-	containerInstance.ReadinessProbe.HttpGet.Port = containerPort
+	if protocol == "http" {
+		containerInstance.ReadinessProbe.HttpGet.Path = "/"
+		containerInstance.ReadinessProbe.HttpGet.Port = containerPort
+	}
+	if protocol == "grpc" {
+		containerInstance.ReadinessProbe.Exec.Command = append(containerInstance.ReadinessProbe.Exec.Command, string("/bin/grpc_health_probe"))
+		containerInstance.ReadinessProbe.Exec.Command = append(containerInstance.ReadinessProbe.Exec.Command, "-addr=:5000")
+	}
 	containerInstance.ReadinessProbe.InitialDelaySeconds = readinessProbe
 	containerInstance.ReadinessProbe.PeriodSeconds = 1
 	containerInstance.Resources.ResourceRequests.Cpu = requestCPU
@@ -73,14 +82,17 @@ func CreateDeployment(metadataName, selectorAppName, selectorClusterName string,
 func CreateDeploymentWithAffinity(metadataName, selectorAppName, selectorClusterName string, numberOfReplicas int,
 	templateAppLabel, templateClusterLabel, namespace string, containerPort int, containerName, containerImage,
 	mountPath string, volumeName, configMapName string, readinessProbe int, requestCPU, requestMemory, limitCPU,
-	limitMemory, nodeAffinity string) (deploymentInstance model.DeploymentInstanceWithAffinity) {
+	limitMemory, nodeAffinity, protocol string) (deploymentInstance model.DeploymentInstanceWithAffinity) {
 
 	var deployment model.DeploymentInstanceWithAffinity
 	var containerInstance model.ContainerInstance
+	var envInstance model.EnvInstance
 	var containerPortInstance model.ContainerPortInstance
 	var containerVolume model.ContainerVolumeInstance
 	var volumeInstance model.VolumeInstance
 
+	envInstance.Name = "SERVICE_NAME"
+	envInstance.Value = metadataName
 	containerPortInstance.ContainerPort = containerPort
 	volumeInstance.Name = volumeName
 	volumeInstance.ConfigMap.Name = configMapName
@@ -93,8 +105,16 @@ func CreateDeploymentWithAffinity(metadataName, selectorAppName, selectorCluster
 	containerInstance.Name = containerName
 	containerInstance.Image = containerImage
 	containerInstance.ImagePullPolicy = "Never"
-	containerInstance.ReadinessProbe.HttpGet.Path = "/"
-	containerInstance.ReadinessProbe.HttpGet.Port = containerPort
+	containerInstance.Env = append(containerInstance.Env, envInstance)
+	if protocol == "http" {
+		containerInstance.ReadinessProbe.HttpGet.Path = "/"
+		containerInstance.ReadinessProbe.HttpGet.Port = containerPort
+	}
+	if protocol == "grpc" {
+		containerInstance.ReadinessProbe.Exec.Command = append(containerInstance.ReadinessProbe.Exec.Command, ("/bin/grpc_health_probe"), "-addr=:"+string(containerPort))
+
+	}
+
 	containerInstance.ReadinessProbe.InitialDelaySeconds = readinessProbe
 	containerInstance.ReadinessProbe.PeriodSeconds = 1
 	containerInstance.Resources.ResourceRequests.Cpu = requestCPU
@@ -194,7 +214,7 @@ func CreateServiceAccount(metadataName, accountName string) (serviceAccountInsta
 	return serviceAccount
 }
 
-func CreateConfig(metadataName, metadataLabelName, metadataLabelCluster, namespace, config string) (configMapInstance model.ConfigMapInstance) {
+func CreateConfig(metadataName, metadataLabelName, metadataLabelCluster, namespace, config, proto string) (configMapInstance model.ConfigMapInstance) {
 
 	const apiVersion = "v1"
 
@@ -209,6 +229,7 @@ func CreateConfig(metadataName, metadataLabelName, metadataLabelCluster, namespa
 	configMap.Metadata.Labels.Name = metadataLabelName
 	configMap.Metadata.Namespace = namespace
 	configMap.Data.Config = config
+	configMap.Data.Service = proto
 
 	return configMap
 }

generator/template/service.tmpl

@@ -0,0 +1,17 @@
+syntax = "proto3";
+
+{{ range . }}
+service {{ .Name }} {
+    {{ range .Endpoints }}
+    rpc {{ .Name }} (Request) returns (Response) {}
+    {{ end }}
+}
+{{ end }}
+
+message Request {
+    string data = 1;
+}
+
+message Response {
+    string data = 1;
+}

model/Dockerfile

@@ -14,13 +14,13 @@
 # limitations under the License.
 #
 
-FROM python:3.8.0-alpine
+FROM python:3.8.0-slim
 
 RUN mkdir -p /usr/src/app
-
-RUN apk update \
- && apk add jq \
- && rm -rf /var/cache/apk/*
+RUN apt update
+RUN apt install -y jq \
+    wget \
+    2to3
 
 WORKDIR /usr/src/app
 
@@ -29,8 +29,14 @@ ADD ./requirements.txt /usr/src/app/requirements.txt
 RUN pip install --upgrade pip
 RUN pip install -r requirements.txt
 
+# download the grpc health probe
+RUN GRPC_HEALTH_PROBE_VERSION=v0.4.8 && \
+    wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
+    chmod +x /bin/grpc_health_probe
+
 ENV CONF="/usr/src/app/config/conf.json"
+
 COPY . /usr/src/app
 
-#EXPOSE 5000
+EXPOSE 5000
 ENTRYPOINT ["/usr/src/app/run.sh"]

model/__init__.py

@@ -0,0 +1,15 @@
+"""
+Copyright 2021 Ericsson AB
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""

model/common/__init__.py

@@ -0,0 +1,15 @@
+"""
+Copyright 2021 Ericsson AB
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""