diff --git a/generator/input/new_description.json b/generator/input/new_description.json index c1dd6802..80927268 100644 --- a/generator/input/new_description.json +++ b/generator/input/new_description.json @@ -8,11 +8,11 @@ ], "services": [ { - "name": "service-1", + "name": "service1", "clusters": [ { "cluster": "cluster-1", - "namespace": "ns-1", + "namespace": "default", "node": "node-1" } ], @@ -29,7 +29,7 @@ "processes": 2, "endpoints": [ { - "name": "/end-1", + "name": "end1", "protocol": "http", "cpu_consumption": 0.003, "network_consumption": 0.002, @@ -37,9 +37,9 @@ "forward_requests": "asynchronous", "called_services": [ { - "service": "service-2", + "service": "service2", "port": "80", - "endpoint": "/end-2", + "endpoint": "end2", "protocol": "http", "traffic_forward_ratio": 1 } diff --git a/generator/src/pkg/generate/generate.go b/generator/src/pkg/generate/generate.go index f69e8876..3a1fa263 100644 --- a/generator/src/pkg/generate/generate.go +++ b/generator/src/pkg/generate/generate.go @@ -18,12 +18,14 @@ package generate import ( "application-generator/src/pkg/model" s "application-generator/src/pkg/service" + "bytes" "encoding/json" "fmt" "gopkg.in/yaml.v3" "io/ioutil" "os" "strings" + "text/template" ) const ( @@ -33,8 +35,6 @@ const ( imageName = "app" imageURL = "app-demo:latest" - protocol = "http" - defaultExtPort = 80 defaultPort = 5000 @@ -59,18 +59,18 @@ var ( type CalledServices struct { Service string `json:"service"` - Port string `json:"port"` + Port string `json:"port"` Endpoint string `json:"endpoint"` Protocol string `json:"protocol"` TrafficForwardRatio float32 `json:"traffic_forward_ratio"` } type Endpoints struct { Name string `json:"name"` - Protocol string `json:"protocol"` + Protocol string `json:"protocol"` CpuConsumption float64 `json:"cpu_consumption"` NetworkConsumption float64 `json:"network_consumption"` MemoryConsumption float64 `json:"memory_consumption"` - ForwardRequests string `json:"forward_requests"` + ForwardRequests string `json:"forward_requests"` CalledServices []CalledServices `json:"called_services"` } type ResourceLimits struct { @@ -89,7 +89,7 @@ type Services struct { Name string `json:"name"` Clusters []Clusters `json:"clusters"` Resources Resources `json:"resources"` - Processes int `json:"processes"` + Processes int `json:"processes"` Endpoints []Endpoints `json:"endpoints"` } @@ -111,8 +111,8 @@ type Config struct { } type ConfigMap struct { - Processes int `json:"processes"` - Endpoints []Endpoints `json:"endpoints"` + Processes int `json:"processes"` + Endpoints []Endpoints `json:"endpoints"` } // the slices to store services, cluster and endpoints for counting and printing @@ -166,16 +166,23 @@ func Parse(configFilename string) (Config, []string) { func Create(config Config, readinessProbe int, clusters []string) { path, _ := os.Getwd() + proto_temp, _ := template.ParseFiles(path + "/template/service.tmpl") path = path + "/k8s" for i := 0; i < len(clusters); i++ { directory := fmt.Sprintf(path+"/%s", clusters[i]) os.Mkdir(directory, 0777) } - + var proto_temp_filled_byte bytes.Buffer + err := proto_temp.Execute(&proto_temp_filled_byte, config.Services) + if err != nil { + panic(err) + } + proto_temp_filled := proto_temp_filled_byte.String() for i := 0; i < len(config.Services); i++ { serv := config.Services[i].Name resources := Resources(config.Services[i].Resources) + protocol := config.Services[i].Endpoints[0].Protocol if resources.Limits.Cpu == "" { resources.Limits.Cpu = limitsCPUDefault @@ -216,18 +223,19 @@ func Create(config Config, readinessProbe int, clusters []string) { manifests = append(manifests, string(yamlDoc)) return nil } - configmap = s.CreateConfig("config-"+serv, "config-"+serv, c_id, namespace, string(serv_json)) + configmap = s.CreateConfig("config-"+serv, "config-"+serv, c_id, namespace, string(serv_json), proto_temp_filled) appendManifest(configmap) if nodeAffinity == "" { deployment := s.CreateDeployment(serv, serv, c_id, replicaNumber, serv, c_id, namespace, defaultPort, imageName, imageURL, volumePath, volumeName, "config-"+serv, readinessProbe, - resources.Requests.Cpu, resources.Requests.Memory, resources.Limits.Cpu, resources.Limits.Memory) + resources.Requests.Cpu, resources.Requests.Memory, resources.Limits.Cpu, resources.Limits.Memory, protocol) appendManifest(deployment) } else { deployment := s.CreateDeploymentWithAffinity(serv, serv, c_id, replicaNumber, serv, c_id, namespace, defaultPort, imageName, imageURL, volumePath, volumeName, "config-"+serv, readinessProbe, - resources.Requests.Cpu, resources.Requests.Memory, resources.Limits.Cpu, resources.Limits.Memory, nodeAffinity) + resources.Requests.Cpu, resources.Requests.Memory, resources.Limits.Cpu, resources.Limits.Memory, + nodeAffinity, protocol) appendManifest(deployment) } diff --git a/generator/src/pkg/model/config.go b/generator/src/pkg/model/config.go index bf821070..f3c2aaec 100644 --- a/generator/src/pkg/model/config.go +++ b/generator/src/pkg/model/config.go @@ -21,13 +21,14 @@ type ConfigMapInstance struct { Metadata struct { Name string `yaml:"name"` Labels struct { - Name string `yaml:"name"` - Cluster string `yaml:"version,omitempty"` + Name string `yaml:"name"` + Cluster string `yaml:"version,omitempty"` } `yaml:"labels"` Namespace string `yaml:"namespace"` } `yaml:"metadata"` Data struct { - Config string `yaml:"conf.json"` + Config string `yaml:"conf.json"` + Service string `yaml:"service.proto"` } `yaml:"data"` } diff --git a/generator/src/pkg/model/deployment.go b/generator/src/pkg/model/deployment.go index 7415f070..30ff727c 100644 --- a/generator/src/pkg/model/deployment.go +++ b/generator/src/pkg/model/deployment.go @@ -100,6 +100,7 @@ type ContainerInstance struct { Name string `yaml:"name"` Image string `yaml:"image"` ImagePullPolicy string `yaml:"imagePullPolicy"` + Env []EnvInstance `yaml:"env"` Ports []ContainerPortInstance `yaml:"ports"` Volumes []ContainerVolumeInstance `yaml:"volumeMounts"` ReadinessProbe ReadinessProbeInstance `yaml:"readinessProbe,omitempty"` @@ -110,6 +111,11 @@ type ContainerPortInstance struct { ContainerPort int `yaml:"containerPort"` } +type EnvInstance struct { + Name string `yaml:"name"` + Value string `yaml:"value"` +} + type ContainerVolumeInstance struct { MountPath string `yaml:"mountPath,omitempty"` MountName string `yaml:"name,omitempty"` @@ -117,9 +123,12 @@ type ContainerVolumeInstance struct { type ReadinessProbeInstance struct { HttpGet struct { - Path string `yaml:"path"` - Port int `yaml:"port"` - } `yaml:"httpGet"` + Path string `yaml:"path,omitempty"` + Port int `yaml:"port,omitempty"` + } `yaml:"httpGet,omitempty"` + Exec struct { + Command []string `yaml:"command,flow,omitempty"` + } `yaml:"exec,omitempty"` InitialDelaySeconds int `yaml:"initialDelaySeconds"` PeriodSeconds int `yaml:"periodSeconds"` } diff --git a/generator/src/pkg/service/util.go b/generator/src/pkg/service/util.go index edb993f6..99a809fd 100644 --- a/generator/src/pkg/service/util.go +++ b/generator/src/pkg/service/util.go @@ -23,15 +23,18 @@ import ( func CreateDeployment(metadataName, selectorAppName, selectorClusterName string, numberOfReplicas int, templateAppLabel, templateClusterLabel, namespace string, containerPort int, containerName, containerImage, mountPath string, volumeName, configMapName string, readinessProbe int, requestCPU, requestMemory, limitCPU, - limitMemory string) (deploymentInstance model.DeploymentInstance) { + limitMemory, protocol string) (deploymentInstance model.DeploymentInstance) { var deployment model.DeploymentInstance var containerInstance model.ContainerInstance + var envInstance model.EnvInstance var containerPortInstance model.ContainerPortInstance var containerVolume model.ContainerVolumeInstance var volumeInstance model.VolumeInstance + envInstance.Name = "SERVICE_NAME" + envInstance.Value = metadataName containerPortInstance.ContainerPort = containerPort volumeInstance.Name = volumeName volumeInstance.ConfigMap.Name = configMapName @@ -44,8 +47,14 @@ func CreateDeployment(metadataName, selectorAppName, selectorClusterName string, containerInstance.Name = containerName containerInstance.Image = containerImage containerInstance.ImagePullPolicy = "Never" - containerInstance.ReadinessProbe.HttpGet.Path = "/" - containerInstance.ReadinessProbe.HttpGet.Port = containerPort + if protocol == "http" { + containerInstance.ReadinessProbe.HttpGet.Path = "/" + containerInstance.ReadinessProbe.HttpGet.Port = containerPort + } + if protocol == "grpc" { + containerInstance.ReadinessProbe.Exec.Command = append(containerInstance.ReadinessProbe.Exec.Command, string("/bin/grpc_health_probe")) + containerInstance.ReadinessProbe.Exec.Command = append(containerInstance.ReadinessProbe.Exec.Command, "-addr=:5000") + } containerInstance.ReadinessProbe.InitialDelaySeconds = readinessProbe containerInstance.ReadinessProbe.PeriodSeconds = 1 containerInstance.Resources.ResourceRequests.Cpu = requestCPU @@ -73,14 +82,17 @@ func CreateDeployment(metadataName, selectorAppName, selectorClusterName string, func CreateDeploymentWithAffinity(metadataName, selectorAppName, selectorClusterName string, numberOfReplicas int, templateAppLabel, templateClusterLabel, namespace string, containerPort int, containerName, containerImage, mountPath string, volumeName, configMapName string, readinessProbe int, requestCPU, requestMemory, limitCPU, - limitMemory, nodeAffinity string) (deploymentInstance model.DeploymentInstanceWithAffinity) { + limitMemory, nodeAffinity, protocol string) (deploymentInstance model.DeploymentInstanceWithAffinity) { var deployment model.DeploymentInstanceWithAffinity var containerInstance model.ContainerInstance + var envInstance model.EnvInstance var containerPortInstance model.ContainerPortInstance var containerVolume model.ContainerVolumeInstance var volumeInstance model.VolumeInstance + envInstance.Name = "SERVICE_NAME" + envInstance.Value = metadataName containerPortInstance.ContainerPort = containerPort volumeInstance.Name = volumeName volumeInstance.ConfigMap.Name = configMapName @@ -93,8 +105,16 @@ func CreateDeploymentWithAffinity(metadataName, selectorAppName, selectorCluster containerInstance.Name = containerName containerInstance.Image = containerImage containerInstance.ImagePullPolicy = "Never" - containerInstance.ReadinessProbe.HttpGet.Path = "/" - containerInstance.ReadinessProbe.HttpGet.Port = containerPort + containerInstance.Env = append(containerInstance.Env, envInstance) + if protocol == "http" { + containerInstance.ReadinessProbe.HttpGet.Path = "/" + containerInstance.ReadinessProbe.HttpGet.Port = containerPort + } + if protocol == "grpc" { + containerInstance.ReadinessProbe.Exec.Command = append(containerInstance.ReadinessProbe.Exec.Command, ("/bin/grpc_health_probe"), "-addr=:"+string(containerPort)) + + } + containerInstance.ReadinessProbe.InitialDelaySeconds = readinessProbe containerInstance.ReadinessProbe.PeriodSeconds = 1 containerInstance.Resources.ResourceRequests.Cpu = requestCPU @@ -194,7 +214,7 @@ func CreateServiceAccount(metadataName, accountName string) (serviceAccountInsta return serviceAccount } -func CreateConfig(metadataName, metadataLabelName, metadataLabelCluster, namespace, config string) (configMapInstance model.ConfigMapInstance) { +func CreateConfig(metadataName, metadataLabelName, metadataLabelCluster, namespace, config, proto string) (configMapInstance model.ConfigMapInstance) { const apiVersion = "v1" @@ -209,6 +229,7 @@ func CreateConfig(metadataName, metadataLabelName, metadataLabelCluster, namespa configMap.Metadata.Labels.Name = metadataLabelName configMap.Metadata.Namespace = namespace configMap.Data.Config = config + configMap.Data.Service = proto return configMap } diff --git a/generator/template/service.tmpl b/generator/template/service.tmpl new file mode 100644 index 00000000..c13643c4 --- /dev/null +++ b/generator/template/service.tmpl @@ -0,0 +1,17 @@ +syntax = "proto3"; + +{{ range . }} +service {{ .Name }} { + {{ range .Endpoints }} + rpc {{ .Name }} (Request) returns (Response) {} + {{ end }} +} +{{ end }} + +message Request { + string data = 1; +} + +message Response { + string data = 1; +} \ No newline at end of file diff --git a/model/Dockerfile b/model/Dockerfile index 6e5a5b99..55a00925 100644 --- a/model/Dockerfile +++ b/model/Dockerfile @@ -14,13 +14,13 @@ # limitations under the License. # -FROM python:3.8.0-alpine +FROM python:3.8.0-slim RUN mkdir -p /usr/src/app - -RUN apk update \ - && apk add jq \ - && rm -rf /var/cache/apk/* +RUN apt update +RUN apt install -y jq \ + wget \ + 2to3 WORKDIR /usr/src/app @@ -29,8 +29,14 @@ ADD ./requirements.txt /usr/src/app/requirements.txt RUN pip install --upgrade pip RUN pip install -r requirements.txt +# download the grpc health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.4.8 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + ENV CONF="/usr/src/app/config/conf.json" + COPY . /usr/src/app -#EXPOSE 5000 +EXPOSE 5000 ENTRYPOINT ["/usr/src/app/run.sh"] \ No newline at end of file diff --git a/model/__init__.py b/model/__init__.py new file mode 100644 index 00000000..5cb01e45 --- /dev/null +++ b/model/__init__.py @@ -0,0 +1,15 @@ +""" +Copyright 2021 Ericsson AB + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" \ No newline at end of file diff --git a/model/common/__init__.py b/model/common/__init__.py new file mode 100644 index 00000000..5cb01e45 --- /dev/null +++ b/model/common/__init__.py @@ -0,0 +1,15 @@ +""" +Copyright 2021 Ericsson AB + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" \ No newline at end of file diff --git a/model/common/pre_grpc_client.py b/model/common/pre_grpc_client.py new file mode 100644 index 00000000..edcde271 --- /dev/null +++ b/model/common/pre_grpc_client.py @@ -0,0 +1,40 @@ +""" +Copyright 2021 Ericsson AB + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from jinja2 import Template +import json + +with open("../config/conf.json", "r") as f: + conf = json.load(f) + +with open("template/grpc_client.jinja", "r") as j: + temp = j.read() + +called_svc = [] + +for endpoint in conf['endpoints']: + for svc in endpoint['calledServices']: + if svc['protocol'] == "grpc": + called_svc.append(svc) + +grpc_temp = Template(temp) +filled_temp = grpc_temp.render({"called_svc": called_svc}) + +with open("grpc_client.py", "w") as output: + output.write(filled_temp) +output.close() + + diff --git a/model/common/template/grpc_client.jinja b/model/common/template/grpc_client.jinja new file mode 100644 index 00000000..79225a7b --- /dev/null +++ b/model/common/template/grpc_client.jinja @@ -0,0 +1,14 @@ +from pathlib import Path +import sys +path = str(Path(Path(__file__).parent.absolute()).parent.absolute()) +sys.path.insert(0, path) +import grpc +from common import service_pb2 +from common import service_pb2_grpc + +{% for svc in called_svc %} +def {{ svc.service }}_{{ svc.endpoint }}_client(): + with grpc.insecure_channel('{{ svc.service }}:{{ svc.port }}') as channel: + stub = service_pb2_grpc.{{ svc.service }}Stub(channel) + response = stub.{{ svc.endpoint }}(service_pb2.Request(data='you')) +{% endfor %} diff --git a/model/grpc/pre_app.py b/model/grpc/pre_app.py new file mode 100644 index 00000000..4b5754b8 --- /dev/null +++ b/model/grpc/pre_app.py @@ -0,0 +1,34 @@ +""" +Copyright 2021 Ericsson AB + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from jinja2 import Template +import json +import os + +with open("../config/conf.json", "r") as f: + conf = json.load(f) + +with open("template/app.jinja", "r") as j: + temp = j.read() + + +proto_temp = Template(temp) +filled_temp = proto_temp.render({"endpoints": conf['endpoints'], "service_name": os.environ['SERVICE_NAME']}) + +with open("app.py", "w") as output: + output.write(filled_temp) +output.close() + diff --git a/model/grpc/template/app.jinja b/model/grpc/template/app.jinja new file mode 100644 index 00000000..aafdfbf0 --- /dev/null +++ b/model/grpc/template/app.jinja @@ -0,0 +1,45 @@ +from pathlib import Path +import sys +path = str(Path(Path(__file__).parent.absolute()).parent.absolute()) +sys.path.insert(0, path) +from concurrent import futures +import grpc +import time +import os +from common import service_pb2 +from common import service_pb2_grpc +from grpc_reflection.v1alpha import reflection +from grpc_health.v1 import health_pb2 +from grpc_health.v1 import health_pb2_grpc + + +class Req_Res(service_pb2_grpc.{{ service_name }}Servicer): + {% for endpoint in endpoints %} + def {{ endpoint['name'] }}(self, request, context): + return service_pb2.Response(data='Hi, {{ endpoint['name'] }} %s!' % request.data) + {% endfor %} + def Check(self, request, context): + return health_pb2.HealthCheckResponse(status=health_pb2.HealthCheckResponse.SERVING) + + def Watch(self, request, context): + return health_pb2.HealthCheckResponse(status=health_pb2.HealthCheckResponse.UNIMPLEMENTED) + + + +server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) +service_pb2_grpc.add_{{ service_name }}Servicer_to_server(Req_Res(), server) +health_pb2_grpc.add_HealthServicer_to_server(Req_Res(), server) +server.add_insecure_port('[::]:'+'5000') +server.start() +SERVICE_NAMES = ( + service_pb2.DESCRIPTOR.services_by_name['{{ service_name }}'].full_name, + reflection.SERVICE_NAME, +) +reflection.enable_server_reflection(SERVICE_NAMES, server) +try: + while True: + time.sleep(10000) +except KeyboardInterrupt: + server.stop(0) + + diff --git a/model/requirements.txt b/model/requirements.txt index 314c29a5..478d1ed7 100644 --- a/model/requirements.txt +++ b/model/requirements.txt @@ -8,4 +8,8 @@ requests==2.23 flatten_json six aiohttp -gunicorn==20.1.0 \ No newline at end of file +gunicorn==20.1.0 +grpcio==1.45.0 +grpcio-tools==1.45.0 +grpcio-reflection==1.45.0 +grpcio-health-checking==1.45.0 \ No newline at end of file diff --git a/model/run.sh b/model/run.sh index 1fe27fba..96e03ece 100755 --- a/model/run.sh +++ b/model/run.sh @@ -14,12 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. # +PROTOCOL=$(jq '.endpoints[0].protocol' config/conf.json -r) +PROCESSES=$(jq '.processes' config/conf.json -r) - -PROTOCOL="$(jq '.endpoints[0].protocol' conf.json -r)" -PROCESSES=$(jq '.processes' conf.json -r) - -if [ $PROTOCOL == "http" ]; -then +if [ $PROTOCOL = "http" ]; then $(gunicorn --chdir restful -w $PROCESSES app:app -b 0.0.0.0:5000 ); +elif [ $PROTOCOL = "grpc" ]; then + $(cat config/service.proto > service.proto) + $(python -m grpc_tools.protoc -I. --python_out=./common --grpc_python_out=./common service.proto); + $(cd grpc && python pre_app.py) + 2to3 common/ -w -n + # Uninstall the extra apps + apt remove -y 2to3 wget + $(cd grpc && python app.py) fi \ No newline at end of file