From 4bc86f3ebd4f02f262b92d639dba5c484814dc8a Mon Sep 17 00:00:00 2001 From: Aleksandra Obeso Duque Date: Wed, 13 Apr 2022 12:41:09 +0000 Subject: [PATCH 01/18] adding post to endpoint --- model/restful/resources/endpoint.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/model/restful/resources/endpoint.py b/model/restful/resources/endpoint.py index 48fa94e8..b264d63e 100644 --- a/model/restful/resources/endpoint.py +++ b/model/restful/resources/endpoint.py @@ -25,6 +25,12 @@ def not_found(endpoint): class Endpoint(Resource): def get(self, endpoint=None): + if endpoint is None: + message = {"status": "ok"} + return message + not_found(endpoint) + + def post(self, endpoint=None): if endpoint is None: message = {"status": "ok"} return message From 2941e9bf5bf3132b84a7501eff7754fdc01c7019 Mon Sep 17 00:00:00 2001 From: Aleksandra Obeso Duque Date: Wed, 13 Apr 2022 12:52:48 +0000 Subject: [PATCH 02/18] updating simple example --- .../simple/application_description.json | 44 +++++++++-- .../{cluster-1 => cluster1}/service1.yaml | 20 ++--- .../examples/simple/cluster2/service2.yaml | 78 +++++++++++++++++++ 3 files changed, 127 insertions(+), 15 deletions(-) rename generator/examples/simple/{cluster-1 => cluster1}/service1.yaml (81%) create mode 100644 generator/examples/simple/cluster2/service2.yaml diff --git a/generator/examples/simple/application_description.json b/generator/examples/simple/application_description.json index 2a6db2fd..cd7cac0d 100644 --- a/generator/examples/simple/application_description.json +++ b/generator/examples/simple/application_description.json @@ -1,8 +1,8 @@ { "cluster_latencies": [ { - "src" : "cluster-1", - "dest": "cluster-2", + "src" : "cluster1", + "dest": "cluster2", "latency": 0.002 } ], @@ -11,9 +11,9 @@ "name": "service1", "clusters": [ { - "cluster": "cluster-1", - "namespace": "default", - "node": "node-1" + "cluster": "cluster1", + "namespace": "edge-namespace", + "node": "cluster1-control-plane" } ], "resources": { @@ -48,6 +48,40 @@ ] } ] + }, + { + "name": "service2", + "clusters": [ + { + "cluster": "cluster2", + "namespace": "edge-namespace", + "node": "cluster2-control-plane" + } + ], + "resources": { + "limits": { + "memory": "1024M", + "cpu": "1000m" + }, + "requests": { + "memory": "512M", + "cpu": "500m" + } + }, + "processes": 2, + "threads": 2, + "readiness_probe": 5, + "endpoints": [ + { + "name": "end2", + "protocol": "http", + "cpu_consumption": 0.003, + "network_consumption": 0.002, + "memory_consumption": 0.003, + "forward_requests": "asynchronous", + "called_services": [] + } + ] } ] } \ No newline at end of file diff --git a/generator/examples/simple/cluster-1/service1.yaml b/generator/examples/simple/cluster1/service1.yaml similarity index 81% rename from generator/examples/simple/cluster-1/service1.yaml rename to generator/examples/simple/cluster1/service1.yaml index 9b2c8dc3..a6b1c61d 100644 --- a/generator/examples/simple/cluster-1/service1.yaml +++ b/generator/examples/simple/cluster1/service1.yaml @@ -4,32 +4,32 @@ metadata: name: config-service1 labels: name: config-service1 - version: cluster-1 - namespace: default + version: cluster1 + namespace: edge-namespace data: conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end1","protocol":"http","cpu_consumption":0.003,"network_consumption":0.002,"memory_consumption":0.003,"forward_requests":"asynchronous","called_services":[{"service":"service2","port":"80","endpoint":"end2","protocol":"http","traffic_forward_ratio":1}]}]}' - service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" + service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc end2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" --- apiVersion: apps/v1 kind: Deployment metadata: name: service1 - namespace: default + namespace: edge-namespace labels: - version: cluster-1 + version: cluster1 spec: selector: matchLabels: app: service1 - version: cluster-1 + version: cluster1 replicas: 1 template: metadata: labels: app: service1 - version: cluster-1 + version: cluster1 spec: - nodeName: node-1 + nodeName: cluster1-control-plane containers: - name: app image: app-demo:latest @@ -64,9 +64,9 @@ apiVersion: v1 kind: Service metadata: name: service1 - namespace: default + namespace: edge-namespace labels: - version: cluster-1 + version: cluster1 annotations: http: / spec: diff --git a/generator/examples/simple/cluster2/service2.yaml b/generator/examples/simple/cluster2/service2.yaml new file mode 100644 index 00000000..9348b5ee --- /dev/null +++ b/generator/examples/simple/cluster2/service2.yaml @@ -0,0 +1,78 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-service2 + labels: + name: config-service2 + version: cluster2 + namespace: edge-namespace +data: + conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end2","protocol":"http","cpu_consumption":0.003,"network_consumption":0.002,"memory_consumption":0.003,"forward_requests":"asynchronous","called_services":[]}]}' + service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc end2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: service2 + namespace: edge-namespace + labels: + version: cluster2 +spec: + selector: + matchLabels: + app: service2 + version: cluster2 + replicas: 1 + template: + metadata: + labels: + app: service2 + version: cluster2 + spec: + nodeName: cluster1-control-plane + containers: + - name: app + image: app-demo:latest + imagePullPolicy: Never + env: + - name: SERVICE_NAME + value: service2 + ports: + - containerPort: 5000 + volumeMounts: + - mountPath: /usr/src/app/config + name: config-data-volume + readinessProbe: + httpGet: + path: / + port: 5000 + initialDelaySeconds: 5 + periodSeconds: 1 + resources: + limits: + cpu: 1000m + memory: 1024M + requests: + cpu: 500m + memory: 512M + volumes: + - name: config-data-volume + configMap: + name: config-service2 +--- +apiVersion: v1 +kind: Service +metadata: + name: service2 + namespace: edge-namespace + labels: + version: cluster2 + annotations: + http: / +spec: + selector: + app: service2 + ports: + - name: http + port: 80 + targetPort: 5000 From ded65416ea2e8573c550a365df97b0bfd091417b Mon Sep 17 00:00:00 2001 From: Aleksandra Obeso Duque Date: Tue, 19 Apr 2022 07:35:37 +0000 Subject: [PATCH 03/18] wip: add random generation of payload --- model/restful/resources/endpoint.py | 11 +--- model/restful/utils/task.py | 89 ++++++++++++++++++----------- 2 files changed, 56 insertions(+), 44 deletions(-) diff --git a/model/restful/resources/endpoint.py b/model/restful/resources/endpoint.py index 3fb90f9c..38cb1507 100644 --- a/model/restful/resources/endpoint.py +++ b/model/restful/resources/endpoint.py @@ -25,12 +25,6 @@ def not_found(endpoint): class Endpoint(Resource): def get(self, endpoint=None): - if endpoint is None: - message = {"status": "ok"} - return message - not_found(endpoint) - - def post(self, endpoint=None): if endpoint is None: message = {"status": "ok"} return message @@ -50,7 +44,4 @@ def post(self, endpoint=None): if ep['name'] == endpoint: res = task.run_task(service_endpoint=ep) return res - not_found(endpoint) - - - + not_found(endpoint) \ No newline at end of file diff --git a/model/restful/utils/task.py b/model/restful/utils/task.py index 56937b0b..11f56bc0 100644 --- a/model/restful/utils/task.py +++ b/model/restful/utils/task.py @@ -15,14 +15,14 @@ """ import logging +from wsgiref import headers from flask import Blueprint, jsonify, request import path from aiohttp import ClientSession import asyncio import uuid +import subprocess -# TODO: So far, we only support a hard-coded namespace. For more flexible support of namespaces we will need to pass that info as part of the config map -# TODO: So far, we only support http client FORMATTED_REMOTE_URL = "http://{0}:{1}/{2}" @@ -46,36 +46,36 @@ def getForwardHeaders(request): def run_task(service_endpoint): headers = getForwardHeaders(request) + if request.get_data() == bytes("", "utf-8"): + json_data = {} + else: + json_data = request.json + if service_endpoint["forward_requests"] == "asynchronous": asyncio.set_event_loop(asyncio.new_event_loop()) loop = asyncio.get_event_loop() - response = loop.run_until_complete(async_tasks(service_endpoint, headers)) + response = loop.run_until_complete(async_tasks(service_endpoint, headers, json_data)) return response else: # "synchronous" asyncio.set_event_loop(asyncio.new_event_loop()) loop = asyncio.get_event_loop() - response = loop.run_until_complete(sync_tasks(service_endpoint, headers)) + response = loop.run_until_complete(sync_tasks(service_endpoint, headers, json_data)) return response -async def async_tasks(service_endpoint, headers): +async def async_tasks(service_endpoint, headers, json_data): async with ClientSession() as session: - # TODO: CPU-bounded tasks not supported yet io_tasks = [] - if request.get_data() == bytes("", "utf-8"): - json_data = {} - else: - json_data = request.json if len(service_endpoint["called_services"]) > 0: for svc in service_endpoint["called_services"]: io_task = asyncio.create_task(execute_io_bounded_task(session=session, target_service=svc, - json_data=json_data, forward_headers=headers)) + json_data=json_data, sync=False, forward_headers=headers)) io_tasks.append(io_task) services = await asyncio.gather(*io_tasks) - # Concatenate json responses + # Concatenate json responses response = {} response["services"] = [] response["statuses"] = [] @@ -87,19 +87,15 @@ async def async_tasks(service_endpoint, headers): return response -async def sync_tasks(service_endpoint, headers): +async def sync_tasks(service_endpoint, headers, json_data): async with ClientSession() as session: - # TODO: CPU-bounded tasks not supported yet response = {} response["services"] = [] response["statuses"] = [] - if request.get_data() == bytes("", "utf-8"): - json_data = {} - else: - json_data = request.json + if len(service_endpoint["called_services"]) > 0: for svc in service_endpoint["called_services"]: - res = await execute_io_bounded_task(session=session, target_service=svc, json_data=json_data, forward_headers=headers) + res = await execute_io_bounded_task(session=session, target_service=svc, json_data=json_data, sync=True, forward_headers=headers) response["services"] += res["services"] response["statuses"] += res["statuses"] @@ -114,6 +110,7 @@ def execute_cpu_bounded_task(origin_service_name, target_service, headers): task_config["network_consumption"] = target_service["network_consumption"] task_config["memory_consumption"] = target_service["memory_consumption"] + # TODO: CPU-bounded tasks not supported yet # TODO: Implement resource stress emulation... response_object = { @@ -126,22 +123,46 @@ def execute_cpu_bounded_task(origin_service_name, target_service, headers): return response_object -async def execute_io_bounded_task(session, target_service, json_data, forward_headers={}): - # TODO: Request forwarding to a service on a particular cluster is not supported yet - # TODO: Requests for other protocols than html are not supported yet +async def execute_io_bounded_task(session, target_service, json_data, sync, forward_headers={}): dst = FORMATTED_REMOTE_URL.format(target_service["service"], target_service["port"], target_service["endpoint"]) forward_headers.update({'Content-type' : 'application/json'}) - # TODO: traffic_forward_ratio not supported yet - traffic_forward_ratio = target_service["traffic_forward_ratio"] - - res = await session.post(dst, data=json_data, headers=forward_headers) - res_payload = await res.json() - - response = {} - response["services"] = res_payload["services"] - response['services'].append(str(res.url)) - response["statuses"] = res_payload["statuses"] - response['statuses'].append(res.status) - return response + forward_ratio = target_service["traffic_forward_ratio"] + forward_payload_size = target_service["traffic_forward_payload"] + + if forward_payload_size: + with subprocess.run(['cat /dev/urandom | tr -dc "[:alnum:]" | head -c${1:-%s}' % forward_payload_size], shell=True) as json_data: + responses = {} + responses["services"] = [] + responses["statuses"] = [] + + if forward_ratio > 0: + if not sync: + async with ClientSession() as session: + io_tasks = [] + + for i in range(forward_ratio): + io_task = asyncio.create_task(session.post(dst, data=json_data, headers=forward_headers)) + io_tasks.append(io_task) + calls = await asyncio.gather(*io_tasks) + + # Concatenate json responses + for res_payload in calls: + responses["services"] = res_payload["services"] + responses['services'].append(str(res.url)) + responses["statuses"] = res_payload["statuses"] + responses['statuses'].append(res.status) + + else: # "synchronous" + async with ClientSession() as session: + for i in range(forward_ratio): + res = await session.post(dst, data=json_data, headers=forward_headers) + res_payload = await res.json() + + responses["services"] = res_payload["services"] + responses['services'].append(str(res.url)) + responses["statuses"] = res_payload["statuses"] + responses['statuses'].append(res.status) + + return responses \ No newline at end of file From 026e3b7a4672440c31d6183f95301863145670f1 Mon Sep 17 00:00:00 2001 From: Aleksandra Obeso Duque Date: Tue, 19 Apr 2022 19:11:52 +0000 Subject: [PATCH 04/18] adding network complexity configuratiion --- model/restful/utils/task.py | 102 +++++++++++++++++++----------------- 1 file changed, 55 insertions(+), 47 deletions(-) diff --git a/model/restful/utils/task.py b/model/restful/utils/task.py index 11f56bc0..2748a15c 100644 --- a/model/restful/utils/task.py +++ b/model/restful/utils/task.py @@ -22,6 +22,7 @@ import asyncio import uuid import subprocess +import sys FORMATTED_REMOTE_URL = "http://{0}:{1}/{2}" @@ -46,32 +47,30 @@ def getForwardHeaders(request): def run_task(service_endpoint): headers = getForwardHeaders(request) - if request.get_data() == bytes("", "utf-8"): - json_data = {} - else: - json_data = request.json + response_payload_size = service_endpoint["response_payload_size"] + response_payload = subprocess.run(['cat /dev/urandom | tr -dc "[:alnum:]" | head -c${1:-%s}' % response_payload_size], capture_output=True, shell=True) + res_payload = response_payload.stdout.decode("utf-8") if service_endpoint["forward_requests"] == "asynchronous": asyncio.set_event_loop(asyncio.new_event_loop()) loop = asyncio.get_event_loop() - response = loop.run_until_complete(async_tasks(service_endpoint, headers, json_data)) + response = loop.run_until_complete(async_tasks(service_endpoint, headers, res_payload)) return response else: # "synchronous" asyncio.set_event_loop(asyncio.new_event_loop()) loop = asyncio.get_event_loop() - response = loop.run_until_complete(sync_tasks(service_endpoint, headers, json_data)) + response = loop.run_until_complete(sync_tasks(service_endpoint, headers, res_payload)) return response -async def async_tasks(service_endpoint, headers, json_data): +async def async_tasks(service_endpoint, headers, res_payload): async with ClientSession() as session: io_tasks = [] if len(service_endpoint["called_services"]) > 0: for svc in service_endpoint["called_services"]: - io_task = asyncio.create_task(execute_io_bounded_task(session=session, target_service=svc, - json_data=json_data, sync=False, forward_headers=headers)) + io_task = asyncio.create_task(execute_io_bounded_task(session=session, target_service=svc, sync=False, forward_headers=headers)) io_tasks.append(io_task) services = await asyncio.gather(*io_tasks) @@ -79,6 +78,7 @@ async def async_tasks(service_endpoint, headers, json_data): response = {} response["services"] = [] response["statuses"] = [] + response["payload"] = res_payload if len(service_endpoint["called_services"]) > 0: for svc in services: @@ -87,15 +87,16 @@ async def async_tasks(service_endpoint, headers, json_data): return response -async def sync_tasks(service_endpoint, headers, json_data): +async def sync_tasks(service_endpoint, headers, res_payload): async with ClientSession() as session: response = {} response["services"] = [] response["statuses"] = [] + response["payload"] = res_payload if len(service_endpoint["called_services"]) > 0: for svc in service_endpoint["called_services"]: - res = await execute_io_bounded_task(session=session, target_service=svc, json_data=json_data, sync=True, forward_headers=headers) + res = await execute_io_bounded_task(session=session, target_service=svc, sync=True, forward_headers=headers) response["services"] += res["services"] response["statuses"] += res["statuses"] @@ -123,46 +124,53 @@ def execute_cpu_bounded_task(origin_service_name, target_service, headers): return response_object -async def execute_io_bounded_task(session, target_service, json_data, sync, forward_headers={}): +async def execute_io_bounded_task(session, target_service, sync, forward_headers={}): dst = FORMATTED_REMOTE_URL.format(target_service["service"], target_service["port"], target_service["endpoint"]) forward_headers.update({'Content-type' : 'application/json'}) + json_data = {} + json_data["payload"] = "" + forward_ratio = target_service["traffic_forward_ratio"] - forward_payload_size = target_service["traffic_forward_payload"] - - if forward_payload_size: - with subprocess.run(['cat /dev/urandom | tr -dc "[:alnum:]" | head -c${1:-%s}' % forward_payload_size], shell=True) as json_data: - responses = {} - responses["services"] = [] - responses["statuses"] = [] - - if forward_ratio > 0: - if not sync: - async with ClientSession() as session: - io_tasks = [] - - for i in range(forward_ratio): - io_task = asyncio.create_task(session.post(dst, data=json_data, headers=forward_headers)) - io_tasks.append(io_task) - calls = await asyncio.gather(*io_tasks) - - # Concatenate json responses - for res_payload in calls: - responses["services"] = res_payload["services"] + request_payload_size = target_service["request_payload_size"] + + responses = {} + responses["services"] = [] + responses["statuses"] = [] + + if request_payload_size: + request_payload = subprocess.run(['cat /dev/urandom | tr -dc "[:alnum:]" | head -c${1:-%s}' % request_payload_size], capture_output=True, shell=True) + json_data["payload"] = request_payload.stdout.decode("utf-8") + + if forward_ratio > 0: + if not sync: + async with ClientSession() as session: + io_tasks = [] + + for i in range(forward_ratio): + io_task = asyncio.create_task(session.post(dst, data=json_data, headers=forward_headers)) + io_tasks.append(io_task) + calls = await asyncio.gather(*io_tasks) + + # Concatenate json responses + for res in calls: + res_payload = await res.json() + + responses["services"] += res_payload["services"] + responses['services'].append(str(res.url)) + responses["statuses"] += res_payload["statuses"] + responses['statuses'].append(res.status) + + else: # "synchronous" + async with ClientSession() as session: + for i in range(forward_ratio): + res = await session.post(dst, data=json_data, headers=forward_headers) + res_payload = await res.json() + + responses["services"] += res_payload["services"] responses['services'].append(str(res.url)) - responses["statuses"] = res_payload["statuses"] + responses["statuses"] += res_payload["statuses"] responses['statuses'].append(res.status) - - else: # "synchronous" - async with ClientSession() as session: - for i in range(forward_ratio): - res = await session.post(dst, data=json_data, headers=forward_headers) - res_payload = await res.json() - - responses["services"] = res_payload["services"] - responses['services'].append(str(res.url)) - responses["statuses"] = res_payload["statuses"] - responses['statuses'].append(res.status) - - return responses \ No newline at end of file + + return responses \ No newline at end of file From 755b2c5e2fb478cc0f4529e4934fadb537b1af52 Mon Sep 17 00:00:00 2001 From: Aleksandra Obeso Duque Date: Tue, 19 Apr 2022 21:24:04 +0000 Subject: [PATCH 05/18] adding network complexity support in generator --- .../simple/application_description.json | 41 +++++++++++-------- .../examples/simple/cluster1/service1.yaml | 2 +- .../examples/simple/cluster2/service2.yaml | 4 +- generator/input/new_description.json | 29 +++++++------ generator/src/pkg/generate/generate.go | 2 +- generator/src/pkg/model/input.go | 29 +++++++------ generator/src/pkg/service/util.go | 20 +++++---- 7 files changed, 71 insertions(+), 56 deletions(-) diff --git a/generator/examples/simple/application_description.json b/generator/examples/simple/application_description.json index cd7cac0d..0e5c2434 100644 --- a/generator/examples/simple/application_description.json +++ b/generator/examples/simple/application_description.json @@ -33,19 +33,22 @@ { "name": "end1", "protocol": "http", - "cpu_consumption": 0.003, - "network_consumption": 0.002, - "memory_consumption": 0.003, - "forward_requests": "asynchronous", - "called_services": [ - { - "service": "service2", - "port": "80", - "endpoint": "end2", - "protocol": "http", - "traffic_forward_ratio": 1 - } - ] + "cpu_complexity": 0.003, + "memory_complexity": 0.003, + "network_complexity": { + "forward_requests": "asynchronous", + "response_payload_size": 512, + "called_services": [ + { + "service": "service2", + "port": "80", + "endpoint": "end2", + "protocol": "http", + "traffic_forward_ratio": 1, + "request_payload_size": 256 + } + ] + } } ] }, @@ -75,11 +78,13 @@ { "name": "end2", "protocol": "http", - "cpu_consumption": 0.003, - "network_consumption": 0.002, - "memory_consumption": 0.003, - "forward_requests": "asynchronous", - "called_services": [] + "cpu_complexity": 0.003, + "memory_complexity": 0.003, + "network_complexity": { + "forward_requests": "asynchronous", + "response_payload_size": 512, + "called_services": [] + } } ] } diff --git a/generator/examples/simple/cluster1/service1.yaml b/generator/examples/simple/cluster1/service1.yaml index a6b1c61d..38e1e746 100644 --- a/generator/examples/simple/cluster1/service1.yaml +++ b/generator/examples/simple/cluster1/service1.yaml @@ -7,7 +7,7 @@ metadata: version: cluster1 namespace: edge-namespace data: - conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end1","protocol":"http","cpu_consumption":0.003,"network_consumption":0.002,"memory_consumption":0.003,"forward_requests":"asynchronous","called_services":[{"service":"service2","port":"80","endpoint":"end2","protocol":"http","traffic_forward_ratio":1}]}]}' + conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end1","protocol":"http","cpu_complexity":0.003,"memory_complexity":0.003,"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[{"service":"service2","port":"80","endpoint":"end2","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256}]}}]}' service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc end2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" --- apiVersion: apps/v1 diff --git a/generator/examples/simple/cluster2/service2.yaml b/generator/examples/simple/cluster2/service2.yaml index 9348b5ee..ab73187f 100644 --- a/generator/examples/simple/cluster2/service2.yaml +++ b/generator/examples/simple/cluster2/service2.yaml @@ -7,7 +7,7 @@ metadata: version: cluster2 namespace: edge-namespace data: - conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end2","protocol":"http","cpu_consumption":0.003,"network_consumption":0.002,"memory_consumption":0.003,"forward_requests":"asynchronous","called_services":[]}]}' + conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end2","protocol":"http","cpu_complexity":0.003,"memory_complexity":0.003,"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc end2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" --- apiVersion: apps/v1 @@ -29,7 +29,7 @@ spec: app: service2 version: cluster2 spec: - nodeName: cluster1-control-plane + nodeName: cluster2-control-plane containers: - name: app image: app-demo:latest diff --git a/generator/input/new_description.json b/generator/input/new_description.json index 2a6db2fd..67935254 100644 --- a/generator/input/new_description.json +++ b/generator/input/new_description.json @@ -33,19 +33,22 @@ { "name": "end1", "protocol": "http", - "cpu_consumption": 0.003, - "network_consumption": 0.002, - "memory_consumption": 0.003, - "forward_requests": "asynchronous", - "called_services": [ - { - "service": "service2", - "port": "80", - "endpoint": "end2", - "protocol": "http", - "traffic_forward_ratio": 1 - } - ] + "cpu_complexity": 0.003, + "memory_complexity": 0.003, + "network_complexity": { + "forward_requests": "asynchronous", + "response_payload_size": 256, + "called_services": [ + { + "service": "service2", + "port": "80", + "endpoint": "end2", + "protocol": "http", + "traffic_forward_ratio": 1, + "request_payload_size": 512 + } + ] + } } ] } diff --git a/generator/src/pkg/generate/generate.go b/generator/src/pkg/generate/generate.go index 5e5dc5a4..86326496 100644 --- a/generator/src/pkg/generate/generate.go +++ b/generator/src/pkg/generate/generate.go @@ -234,7 +234,7 @@ func CreateJsonInput(userConfig model.UserConfig) string { // NOTE: Always calling the first endpoint of the called service calledService.Endpoint = s.EpNamePrefix + "1" - endpoint.CalledServices = append(endpoint.CalledServices, calledService) + endpoint.NetworkComplexity.CalledServices = append(endpoint.NetworkComplexity.CalledServices, calledService) } } diff --git a/generator/src/pkg/model/input.go b/generator/src/pkg/model/input.go index 15a31384..d38ec7c2 100644 --- a/generator/src/pkg/model/input.go +++ b/generator/src/pkg/model/input.go @@ -17,21 +17,26 @@ limitations under the License. package model type CalledService struct { - Service string `json:"service"` - Port string `json:"port"` - Endpoint string `json:"endpoint"` - Protocol string `json:"protocol"` - TrafficForwardRatio float32 `json:"traffic_forward_ratio"` + Service string `json:"service"` + Port string `json:"port"` + Endpoint string `json:"endpoint"` + Protocol string `json:"protocol"` + TrafficForwardRatio int `json:"traffic_forward_ratio"` + RequestPayloadSize int `json:"request_payload_size"` +} + +type NetworkComplexity struct { + ForwardRequests string `json:"forward_requests"` + ResponsePayloadSize int `json:"response_payload_size"` + CalledServices []CalledService `json:"called_services"` } type Endpoint struct { - Name string `json:"name"` - Protocol string `json:"protocol"` - CpuConsumption float64 `json:"cpu_consumption"` - NetworkConsumption float64 `json:"network_consumption"` - MemoryConsumption float64 `json:"memory_consumption"` - ForwardRequests string `json:"forward_requests"` - CalledServices []CalledService `json:"called_services"` + Name string `json:"name"` + Protocol string `json:"protocol"` + CpuComplexity float64 `json:"cpu_complexity"` + MemoryComplexity float64 `json:"memory_complexity"` + NetworkComplexity NetworkComplexity `json:"network_complexity"` } type ResourceLimits struct { diff --git a/generator/src/pkg/service/util.go b/generator/src/pkg/service/util.go index 10c8b439..34caab0b 100644 --- a/generator/src/pkg/service/util.go +++ b/generator/src/pkg/service/util.go @@ -46,13 +46,14 @@ const ( SvcThreadsDefault = 2 SvcReadinessProbeDefault = 5 - EpNamePrefix = "/end" - EpCPUConsumptionDefault = 0.003 - EpNetworkConsumptionDefault = 0.002 - EpMemoryConsumptionDefault = 0.003 - EpForwardRequests = "asynchronous" + EpNamePrefix = "/end" + EpCPUConsumptionDefault = 0.003 + EpResponseSizeDefault = 512 + EpMemoryConsumptionDefault = 0.003 + EpForwardRequests = "asynchronous" CsTrafficForwardRatio = 1 + CsRequestSizeDefault = 256 ) func CreateDeployment(metadataName, selectorAppName, selectorClusterName string, numberOfReplicas int, @@ -312,10 +313,10 @@ func CreateInputEndpoint() model.Endpoint { var ep model.Endpoint ep.Protocol = defaultProtocol - ep.CpuConsumption = EpCPUConsumptionDefault - ep.NetworkConsumption = EpNetworkConsumptionDefault - ep.MemoryConsumption = EpMemoryConsumptionDefault - ep.ForwardRequests = EpForwardRequests + ep.CpuComplexity = EpCPUConsumptionDefault + ep.MemoryComplexity = EpMemoryConsumptionDefault + ep.NetworkComplexity.ForwardRequests = EpForwardRequests + ep.NetworkComplexity.ResponsePayloadSize = EpResponseSizeDefault return ep } @@ -327,6 +328,7 @@ func CreateInputCalledSvc() model.CalledService { calledSvc.Port = strconv.Itoa(DefaultExtPort) calledSvc.Protocol = defaultProtocol calledSvc.TrafficForwardRatio = CsTrafficForwardRatio + calledSvc.RequestPayloadSize = CsRequestSizeDefault return calledSvc } From c8be96360128005207ca2dcb8da8244b44e66941 Mon Sep 17 00:00:00 2001 From: Aleksandra Obeso Duque Date: Wed, 20 Apr 2022 09:27:53 +0000 Subject: [PATCH 06/18] adding cpu stress to generator --- .../simple/application_description.json | 28 ++++++- generator/input/new_description.json | 14 +++- generator/k8s/cluster1/service1.yaml | 78 +++++++++++++++++++ generator/k8s/cluster2/service2.yaml | 78 +++++++++++++++++++ generator/src/pkg/model/input.go | 11 ++- generator/src/pkg/service/util.go | 2 - 6 files changed, 205 insertions(+), 6 deletions(-) create mode 100644 generator/k8s/cluster1/service1.yaml create mode 100644 generator/k8s/cluster2/service2.yaml diff --git a/generator/examples/simple/application_description.json b/generator/examples/simple/application_description.json index 0e5c2434..48326bf5 100644 --- a/generator/examples/simple/application_description.json +++ b/generator/examples/simple/application_description.json @@ -33,7 +33,19 @@ { "name": "end1", "protocol": "http", - "cpu_complexity": 0.003, + "cpu_complexity": { + "execution_time": 10, + "methods": [ + "fibonacci" + ], + "workers": 2, + "execution_mode": "parallel", + "cpu_affinity": [ + 0, + 2 + ], + "cpu_load": "80%" + }, "memory_complexity": 0.003, "network_complexity": { "forward_requests": "asynchronous", @@ -78,7 +90,19 @@ { "name": "end2", "protocol": "http", - "cpu_complexity": 0.003, + "cpu_complexity": { + "execution_time": 10, + "methods": [ + "fibonacci" + ], + "workers": 2, + "execution_mode": "parallel", + "cpu_affinity": [ + 0, + 2 + ], + "cpu_load": "80%" + }, "memory_complexity": 0.003, "network_complexity": { "forward_requests": "asynchronous", diff --git a/generator/input/new_description.json b/generator/input/new_description.json index 67935254..5f9d6b97 100644 --- a/generator/input/new_description.json +++ b/generator/input/new_description.json @@ -33,7 +33,19 @@ { "name": "end1", "protocol": "http", - "cpu_complexity": 0.003, + "cpu_complexity": { + "execution_time": 10, + "methods": [ + "fibonacci" + ], + "workers": 2, + "execution_mode": "parallel", + "cpu_affinity": [ + 0, + 2 + ], + "cpu_load": "80%" + }, "memory_complexity": 0.003, "network_complexity": { "forward_requests": "asynchronous", diff --git a/generator/k8s/cluster1/service1.yaml b/generator/k8s/cluster1/service1.yaml new file mode 100644 index 00000000..d04d1ccb --- /dev/null +++ b/generator/k8s/cluster1/service1.yaml @@ -0,0 +1,78 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-service1 + labels: + name: config-service1 + version: cluster1 + namespace: edge-namespace +data: + conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end1","protocol":"http","cpu_complexity":{"execution_time":10,"methods":["fibonacci"],"workers":2,"execution_mode":"parallel","cpu_affinity":[0,2],"cpu_load":"80%"},"memory_complexity":0.003,"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[{"service":"service2","port":"80","endpoint":"end2","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256}]}}]}' + service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc end2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: service1 + namespace: edge-namespace + labels: + version: cluster1 +spec: + selector: + matchLabels: + app: service1 + version: cluster1 + replicas: 1 + template: + metadata: + labels: + app: service1 + version: cluster1 + spec: + nodeName: cluster1-control-plane + containers: + - name: app + image: app-demo:latest + imagePullPolicy: Never + env: + - name: SERVICE_NAME + value: service1 + ports: + - containerPort: 5000 + volumeMounts: + - mountPath: /usr/src/app/config + name: config-data-volume + readinessProbe: + httpGet: + path: / + port: 5000 + initialDelaySeconds: 5 + periodSeconds: 1 + resources: + limits: + cpu: 1000m + memory: 1024M + requests: + cpu: 500m + memory: 512M + volumes: + - name: config-data-volume + configMap: + name: config-service1 +--- +apiVersion: v1 +kind: Service +metadata: + name: service1 + namespace: edge-namespace + labels: + version: cluster1 + annotations: + http: / +spec: + selector: + app: service1 + ports: + - name: http + port: 80 + targetPort: 5000 diff --git a/generator/k8s/cluster2/service2.yaml b/generator/k8s/cluster2/service2.yaml new file mode 100644 index 00000000..07d222a8 --- /dev/null +++ b/generator/k8s/cluster2/service2.yaml @@ -0,0 +1,78 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-service2 + labels: + name: config-service2 + version: cluster2 + namespace: edge-namespace +data: + conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end2","protocol":"http","cpu_complexity":{"execution_time":10,"methods":["fibonacci"],"workers":2,"execution_mode":"parallel","cpu_affinity":[0,2],"cpu_load":"80%"},"memory_complexity":0.003,"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' + service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc end2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: service2 + namespace: edge-namespace + labels: + version: cluster2 +spec: + selector: + matchLabels: + app: service2 + version: cluster2 + replicas: 1 + template: + metadata: + labels: + app: service2 + version: cluster2 + spec: + nodeName: cluster2-control-plane + containers: + - name: app + image: app-demo:latest + imagePullPolicy: Never + env: + - name: SERVICE_NAME + value: service2 + ports: + - containerPort: 5000 + volumeMounts: + - mountPath: /usr/src/app/config + name: config-data-volume + readinessProbe: + httpGet: + path: / + port: 5000 + initialDelaySeconds: 5 + periodSeconds: 1 + resources: + limits: + cpu: 1000m + memory: 1024M + requests: + cpu: 500m + memory: 512M + volumes: + - name: config-data-volume + configMap: + name: config-service2 +--- +apiVersion: v1 +kind: Service +metadata: + name: service2 + namespace: edge-namespace + labels: + version: cluster2 + annotations: + http: / +spec: + selector: + app: service2 + ports: + - name: http + port: 80 + targetPort: 5000 diff --git a/generator/src/pkg/model/input.go b/generator/src/pkg/model/input.go index d38ec7c2..ef86159b 100644 --- a/generator/src/pkg/model/input.go +++ b/generator/src/pkg/model/input.go @@ -25,6 +25,15 @@ type CalledService struct { RequestPayloadSize int `json:"request_payload_size"` } +type CpuComplexity struct { + ExecutionTime float32 `json:"execution_time"` + Methods []string `json:"methods"` + Workers int `json:"workers"` + ExecutionMode string `json:"execution_mode"` + CpuAffinity []int `json:"cpu_affinity"` + CpuLoad string `json:"cpu_load"` +} + type NetworkComplexity struct { ForwardRequests string `json:"forward_requests"` ResponsePayloadSize int `json:"response_payload_size"` @@ -34,7 +43,7 @@ type NetworkComplexity struct { type Endpoint struct { Name string `json:"name"` Protocol string `json:"protocol"` - CpuComplexity float64 `json:"cpu_complexity"` + CpuComplexity CpuComplexity `json:"cpu_complexity"` MemoryComplexity float64 `json:"memory_complexity"` NetworkComplexity NetworkComplexity `json:"network_complexity"` } diff --git a/generator/src/pkg/service/util.go b/generator/src/pkg/service/util.go index 34caab0b..cb55d69d 100644 --- a/generator/src/pkg/service/util.go +++ b/generator/src/pkg/service/util.go @@ -47,7 +47,6 @@ const ( SvcReadinessProbeDefault = 5 EpNamePrefix = "/end" - EpCPUConsumptionDefault = 0.003 EpResponseSizeDefault = 512 EpMemoryConsumptionDefault = 0.003 EpForwardRequests = "asynchronous" @@ -313,7 +312,6 @@ func CreateInputEndpoint() model.Endpoint { var ep model.Endpoint ep.Protocol = defaultProtocol - ep.CpuComplexity = EpCPUConsumptionDefault ep.MemoryComplexity = EpMemoryConsumptionDefault ep.NetworkComplexity.ForwardRequests = EpForwardRequests ep.NetworkComplexity.ResponsePayloadSize = EpResponseSizeDefault From 123468534206ae64684d2503bae5535a9523c377 Mon Sep 17 00:00:00 2001 From: Aleksandra Obeso Duque Date: Thu, 21 Apr 2022 07:23:58 +0000 Subject: [PATCH 07/18] updating model based on new service description format --- model/restful/utils/task.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/model/restful/utils/task.py b/model/restful/utils/task.py index 2748a15c..6b6f6f5f 100644 --- a/model/restful/utils/task.py +++ b/model/restful/utils/task.py @@ -47,11 +47,11 @@ def getForwardHeaders(request): def run_task(service_endpoint): headers = getForwardHeaders(request) - response_payload_size = service_endpoint["response_payload_size"] + response_payload_size = service_endpoint["network_complexity"]["response_payload_size"] response_payload = subprocess.run(['cat /dev/urandom | tr -dc "[:alnum:]" | head -c${1:-%s}' % response_payload_size], capture_output=True, shell=True) res_payload = response_payload.stdout.decode("utf-8") - if service_endpoint["forward_requests"] == "asynchronous": + if service_endpoint["network_complexity"]["forward_requests"] == "asynchronous": asyncio.set_event_loop(asyncio.new_event_loop()) loop = asyncio.get_event_loop() response = loop.run_until_complete(async_tasks(service_endpoint, headers, res_payload)) @@ -68,8 +68,8 @@ async def async_tasks(service_endpoint, headers, res_payload): async with ClientSession() as session: io_tasks = [] - if len(service_endpoint["called_services"]) > 0: - for svc in service_endpoint["called_services"]: + if len(service_endpoint["network_complexity"]["called_services"]) > 0: + for svc in service_endpoint["network_complexity"]["called_services"]: io_task = asyncio.create_task(execute_io_bounded_task(session=session, target_service=svc, sync=False, forward_headers=headers)) io_tasks.append(io_task) services = await asyncio.gather(*io_tasks) @@ -80,7 +80,7 @@ async def async_tasks(service_endpoint, headers, res_payload): response["statuses"] = [] response["payload"] = res_payload - if len(service_endpoint["called_services"]) > 0: + if len(service_endpoint["network_complexity"]["called_services"]) > 0: for svc in services: response["services"] += svc["services"] response["statuses"] += svc["statuses"] @@ -94,8 +94,8 @@ async def sync_tasks(service_endpoint, headers, res_payload): response["statuses"] = [] response["payload"] = res_payload - if len(service_endpoint["called_services"]) > 0: - for svc in service_endpoint["called_services"]: + if len(service_endpoint["network_complexity"]["called_services"]) > 0: + for svc in service_endpoint["network_complexity"]["called_services"]: res = await execute_io_bounded_task(session=session, target_service=svc, sync=True, forward_headers=headers) response["services"] += res["services"] response["statuses"] += res["statuses"] From 81e5574e6fc24f5222c0897b70259ef814bd2f67 Mon Sep 17 00:00:00 2001 From: Aleksandra Obeso Duque Date: Thu, 21 Apr 2022 15:47:38 +0000 Subject: [PATCH 08/18] adding support for cpu task --- .../simple/application_description.json | 32 ++-- .../examples/simple/cluster1/service1.yaml | 8 +- .../examples/simple/cluster2/service2.yaml | 8 +- generator/input/new_description.json | 5 +- generator/src/pkg/model/input.go | 11 +- model/Dockerfile | 3 + model/path.py | 3 +- model/restful/resources/endpoint.py | 6 +- model/restful/utils/task.py | 143 ++++++++++++------ model/run.sh | 4 +- 10 files changed, 130 insertions(+), 93 deletions(-) diff --git a/generator/examples/simple/application_description.json b/generator/examples/simple/application_description.json index 48326bf5..42e37605 100644 --- a/generator/examples/simple/application_description.json +++ b/generator/examples/simple/application_description.json @@ -19,32 +19,29 @@ "resources": { "limits": { "memory": "1024M", - "cpu": "1000m" + "cpu": "2000m" }, "requests": { "memory": "512M", - "cpu": "500m" + "cpu": "2000m" } }, "processes": 2, "threads": 2, - "readiness_probe": 5, + "readiness_probe": 1, "endpoints": [ { "name": "end1", "protocol": "http", "cpu_complexity": { "execution_time": 10, - "methods": [ - "fibonacci" - ], + "method": "fibonacci", "workers": 2, - "execution_mode": "parallel", "cpu_affinity": [ 0, - 2 + 1 ], - "cpu_load": "80%" + "cpu_load": "100%" }, "memory_complexity": 0.003, "network_complexity": { @@ -76,32 +73,29 @@ "resources": { "limits": { "memory": "1024M", - "cpu": "1000m" + "cpu": "2000m" }, "requests": { "memory": "512M", - "cpu": "500m" + "cpu": "2000m" } }, "processes": 2, "threads": 2, - "readiness_probe": 5, + "readiness_probe": 1, "endpoints": [ { "name": "end2", "protocol": "http", "cpu_complexity": { "execution_time": 10, - "methods": [ - "fibonacci" - ], + "method": "fibonacci", "workers": 2, - "execution_mode": "parallel", "cpu_affinity": [ - 0, - 2 + 22, + 23 ], - "cpu_load": "80%" + "cpu_load": "100%" }, "memory_complexity": 0.003, "network_complexity": { diff --git a/generator/examples/simple/cluster1/service1.yaml b/generator/examples/simple/cluster1/service1.yaml index 38e1e746..e72cfe37 100644 --- a/generator/examples/simple/cluster1/service1.yaml +++ b/generator/examples/simple/cluster1/service1.yaml @@ -7,7 +7,7 @@ metadata: version: cluster1 namespace: edge-namespace data: - conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end1","protocol":"http","cpu_complexity":0.003,"memory_complexity":0.003,"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[{"service":"service2","port":"80","endpoint":"end2","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256}]}}]}' + conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end1","protocol":"http","cpu_complexity":{"execution_time":10,"method":"fibonacci","workers":2,"cpu_affinity":[0,1],"cpu_load":"100%"},"memory_complexity":0.003,"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[{"service":"service2","port":"80","endpoint":"end2","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256}]}}]}' service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc end2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" --- apiVersion: apps/v1 @@ -46,14 +46,14 @@ spec: httpGet: path: / port: 5000 - initialDelaySeconds: 5 + initialDelaySeconds: 1 periodSeconds: 1 resources: limits: - cpu: 1000m + cpu: 2000m memory: 1024M requests: - cpu: 500m + cpu: 2000m memory: 512M volumes: - name: config-data-volume diff --git a/generator/examples/simple/cluster2/service2.yaml b/generator/examples/simple/cluster2/service2.yaml index ab73187f..e52d15a5 100644 --- a/generator/examples/simple/cluster2/service2.yaml +++ b/generator/examples/simple/cluster2/service2.yaml @@ -7,7 +7,7 @@ metadata: version: cluster2 namespace: edge-namespace data: - conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end2","protocol":"http","cpu_complexity":0.003,"memory_complexity":0.003,"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' + conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end2","protocol":"http","cpu_complexity":{"execution_time":10,"method":"fibonacci","workers":2,"cpu_affinity":[22,23],"cpu_load":"100%"},"memory_complexity":0.003,"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc end2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" --- apiVersion: apps/v1 @@ -46,14 +46,14 @@ spec: httpGet: path: / port: 5000 - initialDelaySeconds: 5 + initialDelaySeconds: 1 periodSeconds: 1 resources: limits: - cpu: 1000m + cpu: 2000m memory: 1024M requests: - cpu: 500m + cpu: 2000m memory: 512M volumes: - name: config-data-volume diff --git a/generator/input/new_description.json b/generator/input/new_description.json index 5f9d6b97..db776a8f 100644 --- a/generator/input/new_description.json +++ b/generator/input/new_description.json @@ -35,11 +35,8 @@ "protocol": "http", "cpu_complexity": { "execution_time": 10, - "methods": [ - "fibonacci" - ], + "method": "fibonacci", "workers": 2, - "execution_mode": "parallel", "cpu_affinity": [ 0, 2 diff --git a/generator/src/pkg/model/input.go b/generator/src/pkg/model/input.go index ef86159b..0f9837a1 100644 --- a/generator/src/pkg/model/input.go +++ b/generator/src/pkg/model/input.go @@ -26,12 +26,11 @@ type CalledService struct { } type CpuComplexity struct { - ExecutionTime float32 `json:"execution_time"` - Methods []string `json:"methods"` - Workers int `json:"workers"` - ExecutionMode string `json:"execution_mode"` - CpuAffinity []int `json:"cpu_affinity"` - CpuLoad string `json:"cpu_load"` + ExecutionTime float32 `json:"execution_time"` + Method string `json:"method"` + Workers int `json:"workers"` + CpuAffinity []int `json:"cpu_affinity"` + CpuLoad string `json:"cpu_load"` } type NetworkComplexity struct { diff --git a/model/Dockerfile b/model/Dockerfile index 55a00925..502bdf61 100644 --- a/model/Dockerfile +++ b/model/Dockerfile @@ -20,6 +20,9 @@ RUN mkdir -p /usr/src/app RUN apt update RUN apt install -y jq \ wget \ + curl \ + vim \ + stress-ng \ 2to3 WORKDIR /usr/src/app diff --git a/model/path.py b/model/path.py index f78f01e7..4e7985db 100644 --- a/model/path.py +++ b/model/path.py @@ -29,4 +29,5 @@ def process_configfile(): config_data = process_configfile() -SERVICE_CONFIG = config_data \ No newline at end of file +SERVICE_CONFIG = config_data +SERVICE_NAME = os.environ['SERVICE_NAME'] \ No newline at end of file diff --git a/model/restful/resources/endpoint.py b/model/restful/resources/endpoint.py index 38cb1507..0206bb32 100644 --- a/model/restful/resources/endpoint.py +++ b/model/restful/resources/endpoint.py @@ -15,7 +15,7 @@ """ from flask_restful import Resource, abort -from path import SERVICE_CONFIG +from path import SERVICE_CONFIG, SERVICE_NAME from restful.utils import task @@ -31,7 +31,7 @@ def get(self, endpoint=None): else: for ep in SERVICE_CONFIG['endpoints']: if ep['name'] == endpoint: - res = task.run_task(service_endpoint=ep) + res = task.run_task(service_name=SERVICE_NAME, service_endpoint=ep) return res not_found(endpoint) @@ -42,6 +42,6 @@ def post(self, endpoint=None): else: for ep in SERVICE_CONFIG['endpoints']: if ep['name'] == endpoint: - res = task.run_task(service_endpoint=ep) + res = task.run_task(service_name=SERVICE_NAME, service_endpoint=ep) return res not_found(endpoint) \ No newline at end of file diff --git a/model/restful/utils/task.py b/model/restful/utils/task.py index 6b6f6f5f..acf1b439 100644 --- a/model/restful/utils/task.py +++ b/model/restful/utils/task.py @@ -44,87 +44,113 @@ def getForwardHeaders(request): return headers -def run_task(service_endpoint): +def run_task(service_name, service_endpoint): headers = getForwardHeaders(request) response_payload_size = service_endpoint["network_complexity"]["response_payload_size"] response_payload = subprocess.run(['cat /dev/urandom | tr -dc "[:alnum:]" | head -c${1:-%s}' % response_payload_size], capture_output=True, shell=True) res_payload = response_payload.stdout.decode("utf-8") + source_svc = {} + source_svc["service"] = service_name + source_svc["endpoint"] = service_endpoint["name"] + + # CPU task + if service_endpoint["cpu_complexity"]: + cpu_response = execute_cpu_bounded_task(conf=service_endpoint["cpu_complexity"]) + + # TODO: Memory task + mem_response = {} + + # Network task if service_endpoint["network_complexity"]["forward_requests"] == "asynchronous": asyncio.set_event_loop(asyncio.new_event_loop()) loop = asyncio.get_event_loop() - response = loop.run_until_complete(async_tasks(service_endpoint, headers, res_payload)) - return response + nw_response = loop.run_until_complete(async_network_task(source_svc, service_endpoint, headers, res_payload)) else: # "synchronous" asyncio.set_event_loop(asyncio.new_event_loop()) loop = asyncio.get_event_loop() - response = loop.run_until_complete(sync_tasks(service_endpoint, headers, res_payload)) + nw_response = loop.run_until_complete(sync_network_task(source_svc, service_endpoint, headers, res_payload)) - return response + nw_response["cpu_task"]["statuses"].append(cpu_response["status"]) + + return nw_response -async def async_tasks(service_endpoint, headers, res_payload): +async def async_network_task(source_svc, service_endpoint, headers, res_payload): async with ClientSession() as session: io_tasks = [] if len(service_endpoint["network_complexity"]["called_services"]) > 0: - for svc in service_endpoint["network_complexity"]["called_services"]: - io_task = asyncio.create_task(execute_io_bounded_task(session=session, target_service=svc, sync=False, forward_headers=headers)) + for target_svc in service_endpoint["network_complexity"]["called_services"]: + io_task = asyncio.create_task(execute_io_bounded_task(session=session, source_service=source_svc, target_service=target_svc, sync=False, forward_headers=headers)) io_tasks.append(io_task) services = await asyncio.gather(*io_tasks) # Concatenate json responses response = {} - response["services"] = [] - response["statuses"] = [] - response["payload"] = res_payload + response["cpu_task"] = {} + response["cpu_task"]["services"] = [] + response["cpu_task"]["statuses"] = [] + + response["memory_task"] = {} + + response["network_task"] = {} + response["network_task"]["services"] = [] + response["network_task"]["statuses"] = [] + response["network_task"]["payload"] = res_payload if len(service_endpoint["network_complexity"]["called_services"]) > 0: for svc in services: - response["services"] += svc["services"] - response["statuses"] += svc["statuses"] + response["cpu_task"]["services"] += svc["cpu_task"]["services"] + response["cpu_task"]["statuses"] += svc["cpu_task"]["statuses"] + + response["network_task"]["services"] += svc["network_task"]["services"] + response["network_task"]["statuses"] += svc["network_task"]["statuses"] + return response -async def sync_tasks(service_endpoint, headers, res_payload): +async def sync_network_task(source_svc, service_endpoint, headers, res_payload): async with ClientSession() as session: response = {} - response["services"] = [] - response["statuses"] = [] - response["payload"] = res_payload + response["cpu_task"] = {} + response["cpu_task"]["services"] = [] + response["cpu_task"]["statuses"] = [] + + response["memory_task"] = {} + + response["network_task"] = {} + response["network_task"]["services"] = [] + response["network_task"]["statuses"] = [] + response["network_task"]["payload"] = res_payload if len(service_endpoint["network_complexity"]["called_services"]) > 0: - for svc in service_endpoint["network_complexity"]["called_services"]: - res = await execute_io_bounded_task(session=session, target_service=svc, sync=True, forward_headers=headers) - response["services"] += res["services"] - response["statuses"] += res["statuses"] + for target_svc in service_endpoint["network_complexity"]["called_services"]: + res = await execute_io_bounded_task(session=session, source_service=source_svc, target_service=target_svc, sync=True, forward_headers=headers) + + # Concatenate json responses + response["cpu_task"]["services"] += res["cpu_task"]["services"] + response["cpu_task"]["statuses"] += res["cpu_task"]["statuses"] - return response + response["network_task"]["services"] += res["network_task"]["services"] + response["network_task"]["statuses"] += res["network_task"]["statuses"] + return response -def execute_cpu_bounded_task(origin_service_name, target_service, headers): - task_id = str(uuid.uuid4()) - task_config = {} - task_config["task_id"] = task_id - task_config["cpu_consumption"] = target_service["cpu_consumption"] - task_config["network_consumption"] = target_service["network_consumption"] - task_config["memory_consumption"] = target_service["memory_consumption"] - # TODO: CPU-bounded tasks not supported yet +def execute_cpu_bounded_task(conf): # TODO: Implement resource stress emulation... + + res = subprocess.run(['stress-ng --class cpu --cpu %s --cpu-method %s --taskset %s --cpu-load %s --timeout %f --metrics-brief' % (conf["workers"], conf["method"], ",".join(str(cpu_id) for cpu_id in conf["cpu_affinity"]), conf["cpu_load"], conf["execution_time"])], capture_output=True, shell=True) - response_object = { - "status": "CPU-bounded task executed", - "data": { - "svc_name": origin_service_name, - "task_id": task_config["task_id"] - } + response = { + "status": res.stderr.decode("utf-8") } - return response_object + return response -async def execute_io_bounded_task(session, target_service, sync, forward_headers={}): +async def execute_io_bounded_task(session, source_service, target_service, sync, forward_headers={}): dst = FORMATTED_REMOTE_URL.format(target_service["service"], target_service["port"], target_service["endpoint"]) forward_headers.update({'Content-type' : 'application/json'}) @@ -135,9 +161,17 @@ async def execute_io_bounded_task(session, target_service, sync, forward_headers forward_ratio = target_service["traffic_forward_ratio"] request_payload_size = target_service["request_payload_size"] - responses = {} - responses["services"] = [] - responses["statuses"] = [] + response = {} + response["cpu_task"] = {} + response["cpu_task"]["services"] = [] + response["cpu_task"]["statuses"] = [] + + response["memory_task"] = {} + + response["network_task"] = {} + response["network_task"]["services"] = [] + response["network_task"]["statuses"] = [] + if request_payload_size: request_payload = subprocess.run(['cat /dev/urandom | tr -dc "[:alnum:]" | head -c${1:-%s}' % request_payload_size], capture_output=True, shell=True) @@ -157,10 +191,14 @@ async def execute_io_bounded_task(session, target_service, sync, forward_headers for res in calls: res_payload = await res.json() - responses["services"] += res_payload["services"] - responses['services'].append(str(res.url)) - responses["statuses"] += res_payload["statuses"] - responses['statuses'].append(res.status) + response["cpu_task"]["services"] += res_payload["cpu_task"]["services"] + response["cpu_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") + response["cpu_task"]["statuses"] += res_payload["cpu_task"]["statuses"] + + response["network_task"]["services"] += res_payload["network_task"]["services"] + response["network_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") + response["network_task"]["statuses"] += res_payload["network_task"]["statuses"] + response["network_task"]['statuses'].append(res.status) else: # "synchronous" async with ClientSession() as session: @@ -168,9 +206,14 @@ async def execute_io_bounded_task(session, target_service, sync, forward_headers res = await session.post(dst, data=json_data, headers=forward_headers) res_payload = await res.json() - responses["services"] += res_payload["services"] - responses['services'].append(str(res.url)) - responses["statuses"] += res_payload["statuses"] - responses['statuses'].append(res.status) + # Concatenate json responses + response["cpu_task"]["services"] += res_payload["cpu_task"]["services"] + response["cpu_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") + response["cpu_task"]["statuses"] += res_payload["cpu_task"]["statuses"] + + response["network_task"]["services"] += res_payload["network_task"]["services"] + response["network_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") + response["network_task"]["statuses"] += res_payload["network_task"]["statuses"] + response["network_task"]['statuses'].append(res.status) - return responses \ No newline at end of file + return response \ No newline at end of file diff --git a/model/run.sh b/model/run.sh index 96e03ece..c2162156 100755 --- a/model/run.sh +++ b/model/run.sh @@ -18,7 +18,7 @@ PROTOCOL=$(jq '.endpoints[0].protocol' config/conf.json -r) PROCESSES=$(jq '.processes' config/conf.json -r) if [ $PROTOCOL = "http" ]; then - $(gunicorn --chdir restful -w $PROCESSES app:app -b 0.0.0.0:5000 ); + $(gunicorn --chdir restful -w $PROCESSES app:app -b 0.0.0.0:5000 --capture-output --log-level debug); elif [ $PROTOCOL = "grpc" ]; then $(cat config/service.proto > service.proto) $(python -m grpc_tools.protoc -I. --python_out=./common --grpc_python_out=./common service.proto); @@ -27,4 +27,4 @@ elif [ $PROTOCOL = "grpc" ]; then # Uninstall the extra apps apt remove -y 2to3 wget $(cd grpc && python app.py) -fi \ No newline at end of file +fi From 07c331de4b996436ff26c88147e8edc768abda6e Mon Sep 17 00:00:00 2001 From: Aleksandra Obeso Duque Date: Fri, 22 Apr 2022 07:28:56 +0000 Subject: [PATCH 09/18] adding memory complexity to model --- .../simple/application_description.json | 18 +++++++++--- .../examples/simple/cluster1/service1.yaml | 2 +- .../examples/simple/cluster2/service2.yaml | 2 +- generator/input/new_description.json | 9 ++++-- generator/src/pkg/model/input.go | 19 ++++++++---- generator/src/pkg/service/util.go | 29 ++++++++++++++----- 6 files changed, 58 insertions(+), 21 deletions(-) diff --git a/generator/examples/simple/application_description.json b/generator/examples/simple/application_description.json index 42e37605..255fc40d 100644 --- a/generator/examples/simple/application_description.json +++ b/generator/examples/simple/application_description.json @@ -34,7 +34,7 @@ "name": "end1", "protocol": "http", "cpu_complexity": { - "execution_time": 10, + "execution_time": "10s", "method": "fibonacci", "workers": 2, "cpu_affinity": [ @@ -43,7 +43,12 @@ ], "cpu_load": "100%" }, - "memory_complexity": 0.003, + "memory_complexity": { + "execution_time": "10s", + "method": "swap", + "workers": 2, + "bytes_load": "80%" + }, "network_complexity": { "forward_requests": "asynchronous", "response_payload_size": 512, @@ -88,7 +93,7 @@ "name": "end2", "protocol": "http", "cpu_complexity": { - "execution_time": 10, + "execution_time": "10s", "method": "fibonacci", "workers": 2, "cpu_affinity": [ @@ -97,7 +102,12 @@ ], "cpu_load": "100%" }, - "memory_complexity": 0.003, + "memory_complexity": { + "execution_time": "10s", + "method": "swap", + "workers": 2, + "bytes_load": "80%" + }, "network_complexity": { "forward_requests": "asynchronous", "response_payload_size": 512, diff --git a/generator/examples/simple/cluster1/service1.yaml b/generator/examples/simple/cluster1/service1.yaml index e72cfe37..ab630637 100644 --- a/generator/examples/simple/cluster1/service1.yaml +++ b/generator/examples/simple/cluster1/service1.yaml @@ -7,7 +7,7 @@ metadata: version: cluster1 namespace: edge-namespace data: - conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end1","protocol":"http","cpu_complexity":{"execution_time":10,"method":"fibonacci","workers":2,"cpu_affinity":[0,1],"cpu_load":"100%"},"memory_complexity":0.003,"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[{"service":"service2","port":"80","endpoint":"end2","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256}]}}]}' + conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end1","protocol":"http","cpu_complexity":{"execution_time":"10s","method":"fibonacci","workers":2,"cpu_affinity":[0,1],"cpu_load":"100%"},"memory_complexity":{"execution_time":"10s","method":"swap","workers":2,"bytes_load":"80%"},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[{"service":"service2","port":"80","endpoint":"end2","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256}]}}]}' service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc end2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" --- apiVersion: apps/v1 diff --git a/generator/examples/simple/cluster2/service2.yaml b/generator/examples/simple/cluster2/service2.yaml index e52d15a5..657f6981 100644 --- a/generator/examples/simple/cluster2/service2.yaml +++ b/generator/examples/simple/cluster2/service2.yaml @@ -7,7 +7,7 @@ metadata: version: cluster2 namespace: edge-namespace data: - conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end2","protocol":"http","cpu_complexity":{"execution_time":10,"method":"fibonacci","workers":2,"cpu_affinity":[22,23],"cpu_load":"100%"},"memory_complexity":0.003,"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' + conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end2","protocol":"http","cpu_complexity":{"execution_time":"10s","method":"fibonacci","workers":2,"cpu_affinity":[22,23],"cpu_load":"100%"},"memory_complexity":{"execution_time":"10s","method":"swap","workers":2,"bytes_load":"80%"},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc end2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" --- apiVersion: apps/v1 diff --git a/generator/input/new_description.json b/generator/input/new_description.json index db776a8f..35d9c9b9 100644 --- a/generator/input/new_description.json +++ b/generator/input/new_description.json @@ -34,7 +34,7 @@ "name": "end1", "protocol": "http", "cpu_complexity": { - "execution_time": 10, + "execution_time": "10s", "method": "fibonacci", "workers": 2, "cpu_affinity": [ @@ -43,7 +43,12 @@ ], "cpu_load": "80%" }, - "memory_complexity": 0.003, + "memory_complexity": { + "execution_time": "10s", + "method": "swap", + "workers": 2, + "bytes_load": "80%" + }, "network_complexity": { "forward_requests": "asynchronous", "response_payload_size": 256, diff --git a/generator/src/pkg/model/input.go b/generator/src/pkg/model/input.go index 0f9837a1..37dc9af9 100644 --- a/generator/src/pkg/model/input.go +++ b/generator/src/pkg/model/input.go @@ -26,11 +26,18 @@ type CalledService struct { } type CpuComplexity struct { - ExecutionTime float32 `json:"execution_time"` - Method string `json:"method"` - Workers int `json:"workers"` - CpuAffinity []int `json:"cpu_affinity"` - CpuLoad string `json:"cpu_load"` + ExecutionTime string `json:"execution_time"` + Method string `json:"method"` + Workers int `json:"workers"` + CpuAffinity []int `json:"cpu_affinity"` + CpuLoad string `json:"cpu_load"` +} + +type MemoryComplexity struct { + ExecutionTime string `json:"execution_time"` + Method string `json:"method"` + Workers int `json:"workers"` + BytesLoad string `json:"bytes_load"` } type NetworkComplexity struct { @@ -43,7 +50,7 @@ type Endpoint struct { Name string `json:"name"` Protocol string `json:"protocol"` CpuComplexity CpuComplexity `json:"cpu_complexity"` - MemoryComplexity float64 `json:"memory_complexity"` + MemoryComplexity MemoryComplexity `json:"memory_complexity"` NetworkComplexity NetworkComplexity `json:"network_complexity"` } diff --git a/generator/src/pkg/service/util.go b/generator/src/pkg/service/util.go index cb55d69d..5363f097 100644 --- a/generator/src/pkg/service/util.go +++ b/generator/src/pkg/service/util.go @@ -46,10 +46,15 @@ const ( SvcThreadsDefault = 2 SvcReadinessProbeDefault = 5 - EpNamePrefix = "/end" - EpResponseSizeDefault = 512 - EpMemoryConsumptionDefault = 0.003 - EpForwardRequests = "asynchronous" + EpNamePrefix = "/end" + EpNwResponseSizeDefault = 512 + + EpExecTimeDefault = "10s" + EpMethodDefault = "all" + EpWorkersDefault = 1 + EpLoadDefault = "100%" + + EpNwForwardRequests = "asynchronous" CsTrafficForwardRatio = 1 CsRequestSizeDefault = 256 @@ -312,9 +317,19 @@ func CreateInputEndpoint() model.Endpoint { var ep model.Endpoint ep.Protocol = defaultProtocol - ep.MemoryComplexity = EpMemoryConsumptionDefault - ep.NetworkComplexity.ForwardRequests = EpForwardRequests - ep.NetworkComplexity.ResponsePayloadSize = EpResponseSizeDefault + + ep.CpuComplexity.ExecutionTime = EpExecTimeDefault + ep.CpuComplexity.Method = EpMethodDefault + ep.CpuComplexity.Workers = EpWorkersDefault + ep.CpuComplexity.CpuLoad = EpLoadDefault + + ep.MemoryComplexity.ExecutionTime = EpExecTimeDefault + ep.MemoryComplexity.Method = EpMethodDefault + ep.MemoryComplexity.Workers = EpWorkersDefault + ep.MemoryComplexity.BytesLoad = EpLoadDefault + + ep.NetworkComplexity.ForwardRequests = EpNwForwardRequests + ep.NetworkComplexity.ResponsePayloadSize = EpNwResponseSizeDefault return ep } From d52d5dffd3086ae02452c84d490749faeb636369 Mon Sep 17 00:00:00 2001 From: Aleksandra Obeso Duque Date: Fri, 22 Apr 2022 08:14:43 +0000 Subject: [PATCH 10/18] adding memory tasks to model --- .../simple/application_description.json | 20 +++++----- .../examples/simple/cluster1/service1.yaml | 4 +- .../examples/simple/cluster2/service2.yaml | 4 +- model/restful/utils/task.py | 39 ++++++++++++++++--- 4 files changed, 48 insertions(+), 19 deletions(-) diff --git a/generator/examples/simple/application_description.json b/generator/examples/simple/application_description.json index 255fc40d..03929bce 100644 --- a/generator/examples/simple/application_description.json +++ b/generator/examples/simple/application_description.json @@ -22,7 +22,7 @@ "cpu": "2000m" }, "requests": { - "memory": "512M", + "memory": "1024M", "cpu": "2000m" } }, @@ -34,7 +34,7 @@ "name": "end1", "protocol": "http", "cpu_complexity": { - "execution_time": "10s", + "execution_time": "5s", "method": "fibonacci", "workers": 2, "cpu_affinity": [ @@ -44,10 +44,10 @@ "cpu_load": "100%" }, "memory_complexity": { - "execution_time": "10s", + "execution_time": "5s", "method": "swap", - "workers": 2, - "bytes_load": "80%" + "workers": 24, + "bytes_load": "100%" }, "network_complexity": { "forward_requests": "asynchronous", @@ -81,7 +81,7 @@ "cpu": "2000m" }, "requests": { - "memory": "512M", + "memory": "1024M", "cpu": "2000m" } }, @@ -93,7 +93,7 @@ "name": "end2", "protocol": "http", "cpu_complexity": { - "execution_time": "10s", + "execution_time": "5s", "method": "fibonacci", "workers": 2, "cpu_affinity": [ @@ -103,10 +103,10 @@ "cpu_load": "100%" }, "memory_complexity": { - "execution_time": "10s", + "execution_time": "5s", "method": "swap", - "workers": 2, - "bytes_load": "80%" + "workers": 24, + "bytes_load": "100%" }, "network_complexity": { "forward_requests": "asynchronous", diff --git a/generator/examples/simple/cluster1/service1.yaml b/generator/examples/simple/cluster1/service1.yaml index ab630637..4286608b 100644 --- a/generator/examples/simple/cluster1/service1.yaml +++ b/generator/examples/simple/cluster1/service1.yaml @@ -7,7 +7,7 @@ metadata: version: cluster1 namespace: edge-namespace data: - conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end1","protocol":"http","cpu_complexity":{"execution_time":"10s","method":"fibonacci","workers":2,"cpu_affinity":[0,1],"cpu_load":"100%"},"memory_complexity":{"execution_time":"10s","method":"swap","workers":2,"bytes_load":"80%"},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[{"service":"service2","port":"80","endpoint":"end2","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256}]}}]}' + conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end1","protocol":"http","cpu_complexity":{"execution_time":"5s","method":"fibonacci","workers":2,"cpu_affinity":[0,1],"cpu_load":"100%"},"memory_complexity":{"execution_time":"5s","method":"swap","workers":24,"bytes_load":"100%"},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[{"service":"service2","port":"80","endpoint":"end2","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256}]}}]}' service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc end2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" --- apiVersion: apps/v1 @@ -54,7 +54,7 @@ spec: memory: 1024M requests: cpu: 2000m - memory: 512M + memory: 1024M volumes: - name: config-data-volume configMap: diff --git a/generator/examples/simple/cluster2/service2.yaml b/generator/examples/simple/cluster2/service2.yaml index 657f6981..5f326740 100644 --- a/generator/examples/simple/cluster2/service2.yaml +++ b/generator/examples/simple/cluster2/service2.yaml @@ -7,7 +7,7 @@ metadata: version: cluster2 namespace: edge-namespace data: - conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end2","protocol":"http","cpu_complexity":{"execution_time":"10s","method":"fibonacci","workers":2,"cpu_affinity":[22,23],"cpu_load":"100%"},"memory_complexity":{"execution_time":"10s","method":"swap","workers":2,"bytes_load":"80%"},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' + conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end2","protocol":"http","cpu_complexity":{"execution_time":"5s","method":"fibonacci","workers":2,"cpu_affinity":[22,23],"cpu_load":"100%"},"memory_complexity":{"execution_time":"5s","method":"swap","workers":24,"bytes_load":"100%"},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc end2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" --- apiVersion: apps/v1 @@ -54,7 +54,7 @@ spec: memory: 1024M requests: cpu: 2000m - memory: 512M + memory: 1024M volumes: - name: config-data-volume configMap: diff --git a/model/restful/utils/task.py b/model/restful/utils/task.py index acf1b439..96dfe264 100644 --- a/model/restful/utils/task.py +++ b/model/restful/utils/task.py @@ -59,8 +59,9 @@ def run_task(service_name, service_endpoint): if service_endpoint["cpu_complexity"]: cpu_response = execute_cpu_bounded_task(conf=service_endpoint["cpu_complexity"]) - # TODO: Memory task - mem_response = {} + # Memory task + if service_endpoint["memory_complexity"]: + mem_response = execute_memory_bounded_task(conf=service_endpoint["memory_complexity"]) # Network task if service_endpoint["network_complexity"]["forward_requests"] == "asynchronous": @@ -73,6 +74,7 @@ def run_task(service_name, service_endpoint): nw_response = loop.run_until_complete(sync_network_task(source_svc, service_endpoint, headers, res_payload)) nw_response["cpu_task"]["statuses"].append(cpu_response["status"]) + nw_response["memory_task"]["statuses"].append(mem_response["status"]) return nw_response @@ -94,6 +96,8 @@ async def async_network_task(source_svc, service_endpoint, headers, res_payload) response["cpu_task"]["statuses"] = [] response["memory_task"] = {} + response["memory_task"]["services"] = [] + response["memory_task"]["statuses"] = [] response["network_task"] = {} response["network_task"]["services"] = [] @@ -105,6 +109,9 @@ async def async_network_task(source_svc, service_endpoint, headers, res_payload) response["cpu_task"]["services"] += svc["cpu_task"]["services"] response["cpu_task"]["statuses"] += svc["cpu_task"]["statuses"] + response["memory_task"]["services"] += svc["memory_task"]["services"] + response["memory_task"]["statuses"] += svc["memory_task"]["statuses"] + response["network_task"]["services"] += svc["network_task"]["services"] response["network_task"]["statuses"] += svc["network_task"]["statuses"] @@ -119,6 +126,8 @@ async def sync_network_task(source_svc, service_endpoint, headers, res_payload): response["cpu_task"]["statuses"] = [] response["memory_task"] = {} + response["memory_task"]["services"] = [] + response["memory_task"]["statuses"] = [] response["network_task"] = {} response["network_task"]["services"] = [] @@ -133,6 +142,9 @@ async def sync_network_task(source_svc, service_endpoint, headers, res_payload): response["cpu_task"]["services"] += res["cpu_task"]["services"] response["cpu_task"]["statuses"] += res["cpu_task"]["statuses"] + response["memory_task"]["services"] += svc["memory_task"]["services"] + response["memory_task"]["statuses"] += svc["memory_task"]["statuses"] + response["network_task"]["services"] += res["network_task"]["services"] response["network_task"]["statuses"] += res["network_task"]["statuses"] @@ -140,9 +152,16 @@ async def sync_network_task(source_svc, service_endpoint, headers, res_payload): def execute_cpu_bounded_task(conf): - # TODO: Implement resource stress emulation... - - res = subprocess.run(['stress-ng --class cpu --cpu %s --cpu-method %s --taskset %s --cpu-load %s --timeout %f --metrics-brief' % (conf["workers"], conf["method"], ",".join(str(cpu_id) for cpu_id in conf["cpu_affinity"]), conf["cpu_load"], conf["execution_time"])], capture_output=True, shell=True) + res = subprocess.run(['stress-ng --class cpu --cpu %s --cpu-method %s --taskset %s --cpu-load %s --timeout %s --metrics-brief' % (conf["workers"], conf["method"], ",".join(str(cpu_id) for cpu_id in conf["cpu_affinity"]), conf["cpu_load"], conf["execution_time"])], capture_output=True, shell=True) + + response = { + "status": res.stderr.decode("utf-8") + } + return response + + +def execute_memory_bounded_task(conf): + res = subprocess.run(['stress-ng --class memory --vm %s --vm-method %s --vm-bytes %s --timeout %s --metrics-brief' % (conf["workers"], conf["method"], conf["bytes_load"], conf["execution_time"])], capture_output=True, shell=True) response = { "status": res.stderr.decode("utf-8") @@ -167,6 +186,8 @@ async def execute_io_bounded_task(session, source_service, target_service, sync, response["cpu_task"]["statuses"] = [] response["memory_task"] = {} + response["memory_task"]["services"] = [] + response["memory_task"]["statuses"] = [] response["network_task"] = {} response["network_task"]["services"] = [] @@ -194,6 +215,10 @@ async def execute_io_bounded_task(session, source_service, target_service, sync, response["cpu_task"]["services"] += res_payload["cpu_task"]["services"] response["cpu_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") response["cpu_task"]["statuses"] += res_payload["cpu_task"]["statuses"] + + response["memory_task"]["services"] += res_payload["memory_task"]["services"] + response["memory_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") + response["memory_task"]["statuses"] += res_payload["memory_task"]["statuses"] response["network_task"]["services"] += res_payload["network_task"]["services"] response["network_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") @@ -211,6 +236,10 @@ async def execute_io_bounded_task(session, source_service, target_service, sync, response["cpu_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") response["cpu_task"]["statuses"] += res_payload["cpu_task"]["statuses"] + response["memory_task"]["services"] += res_payload["memory_task"]["services"] + response["memory_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") + response["memory_task"]["statuses"] += res_payload["memory_task"]["statuses"] + response["network_task"]["services"] += res_payload["network_task"]["services"] response["network_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") response["network_task"]["statuses"] += res_payload["network_task"]["statuses"] From 597be4a9d9af5f2cf7d894b3ff9da2b2d6c7a6a8 Mon Sep 17 00:00:00 2001 From: Aleksandra Obeso Duque Date: Mon, 25 Apr 2022 21:59:10 +0000 Subject: [PATCH 11/18] adding parallel execution of tasks --- .../simple/application_description.json | 2 + .../examples/simple/cluster1/service1.yaml | 2 +- .../examples/simple/cluster2/service2.yaml | 2 +- generator/input/new_description.json | 1 + generator/k8s/cluster1/service1.yaml | 78 ------------ generator/k8s/cluster2/service2.yaml | 78 ------------ generator/src/pkg/model/input.go | 1 + generator/src/pkg/service/util.go | 3 + model/restful/utils/task.py | 111 +++++++++++++----- 9 files changed, 89 insertions(+), 189 deletions(-) delete mode 100644 generator/k8s/cluster1/service1.yaml delete mode 100644 generator/k8s/cluster2/service2.yaml diff --git a/generator/examples/simple/application_description.json b/generator/examples/simple/application_description.json index 03929bce..2069929e 100644 --- a/generator/examples/simple/application_description.json +++ b/generator/examples/simple/application_description.json @@ -33,6 +33,7 @@ { "name": "end1", "protocol": "http", + "execution_mode": "parallel", "cpu_complexity": { "execution_time": "5s", "method": "fibonacci", @@ -92,6 +93,7 @@ { "name": "end2", "protocol": "http", + "execution_mode": "sequential", "cpu_complexity": { "execution_time": "5s", "method": "fibonacci", diff --git a/generator/examples/simple/cluster1/service1.yaml b/generator/examples/simple/cluster1/service1.yaml index 4286608b..6edd9f6b 100644 --- a/generator/examples/simple/cluster1/service1.yaml +++ b/generator/examples/simple/cluster1/service1.yaml @@ -7,7 +7,7 @@ metadata: version: cluster1 namespace: edge-namespace data: - conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end1","protocol":"http","cpu_complexity":{"execution_time":"5s","method":"fibonacci","workers":2,"cpu_affinity":[0,1],"cpu_load":"100%"},"memory_complexity":{"execution_time":"5s","method":"swap","workers":24,"bytes_load":"100%"},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[{"service":"service2","port":"80","endpoint":"end2","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256}]}}]}' + conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end1","protocol":"http","execution_mode":"parallel","cpu_complexity":{"execution_time":"5s","method":"fibonacci","workers":2,"cpu_affinity":[0,1],"cpu_load":"100%"},"memory_complexity":{"execution_time":"5s","method":"swap","workers":24,"bytes_load":"100%"},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[{"service":"service2","port":"80","endpoint":"end2","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256}]}}]}' service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc end2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" --- apiVersion: apps/v1 diff --git a/generator/examples/simple/cluster2/service2.yaml b/generator/examples/simple/cluster2/service2.yaml index 5f326740..d12fd836 100644 --- a/generator/examples/simple/cluster2/service2.yaml +++ b/generator/examples/simple/cluster2/service2.yaml @@ -7,7 +7,7 @@ metadata: version: cluster2 namespace: edge-namespace data: - conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end2","protocol":"http","cpu_complexity":{"execution_time":"5s","method":"fibonacci","workers":2,"cpu_affinity":[22,23],"cpu_load":"100%"},"memory_complexity":{"execution_time":"5s","method":"swap","workers":24,"bytes_load":"100%"},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' + conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end2","protocol":"http","execution_mode":"sequential","cpu_complexity":{"execution_time":"5s","method":"fibonacci","workers":2,"cpu_affinity":[22,23],"cpu_load":"100%"},"memory_complexity":{"execution_time":"5s","method":"swap","workers":24,"bytes_load":"100%"},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc end2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" --- apiVersion: apps/v1 diff --git a/generator/input/new_description.json b/generator/input/new_description.json index 35d9c9b9..61034ddb 100644 --- a/generator/input/new_description.json +++ b/generator/input/new_description.json @@ -33,6 +33,7 @@ { "name": "end1", "protocol": "http", + "execution_mode": "sequential", "cpu_complexity": { "execution_time": "10s", "method": "fibonacci", diff --git a/generator/k8s/cluster1/service1.yaml b/generator/k8s/cluster1/service1.yaml deleted file mode 100644 index d04d1ccb..00000000 --- a/generator/k8s/cluster1/service1.yaml +++ /dev/null @@ -1,78 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: config-service1 - labels: - name: config-service1 - version: cluster1 - namespace: edge-namespace -data: - conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end1","protocol":"http","cpu_complexity":{"execution_time":10,"methods":["fibonacci"],"workers":2,"execution_mode":"parallel","cpu_affinity":[0,2],"cpu_load":"80%"},"memory_complexity":0.003,"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[{"service":"service2","port":"80","endpoint":"end2","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256}]}}]}' - service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc end2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: service1 - namespace: edge-namespace - labels: - version: cluster1 -spec: - selector: - matchLabels: - app: service1 - version: cluster1 - replicas: 1 - template: - metadata: - labels: - app: service1 - version: cluster1 - spec: - nodeName: cluster1-control-plane - containers: - - name: app - image: app-demo:latest - imagePullPolicy: Never - env: - - name: SERVICE_NAME - value: service1 - ports: - - containerPort: 5000 - volumeMounts: - - mountPath: /usr/src/app/config - name: config-data-volume - readinessProbe: - httpGet: - path: / - port: 5000 - initialDelaySeconds: 5 - periodSeconds: 1 - resources: - limits: - cpu: 1000m - memory: 1024M - requests: - cpu: 500m - memory: 512M - volumes: - - name: config-data-volume - configMap: - name: config-service1 ---- -apiVersion: v1 -kind: Service -metadata: - name: service1 - namespace: edge-namespace - labels: - version: cluster1 - annotations: - http: / -spec: - selector: - app: service1 - ports: - - name: http - port: 80 - targetPort: 5000 diff --git a/generator/k8s/cluster2/service2.yaml b/generator/k8s/cluster2/service2.yaml deleted file mode 100644 index 07d222a8..00000000 --- a/generator/k8s/cluster2/service2.yaml +++ /dev/null @@ -1,78 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: config-service2 - labels: - name: config-service2 - version: cluster2 - namespace: edge-namespace -data: - conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end2","protocol":"http","cpu_complexity":{"execution_time":10,"methods":["fibonacci"],"workers":2,"execution_mode":"parallel","cpu_affinity":[0,2],"cpu_load":"80%"},"memory_complexity":0.003,"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' - service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc end2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: service2 - namespace: edge-namespace - labels: - version: cluster2 -spec: - selector: - matchLabels: - app: service2 - version: cluster2 - replicas: 1 - template: - metadata: - labels: - app: service2 - version: cluster2 - spec: - nodeName: cluster2-control-plane - containers: - - name: app - image: app-demo:latest - imagePullPolicy: Never - env: - - name: SERVICE_NAME - value: service2 - ports: - - containerPort: 5000 - volumeMounts: - - mountPath: /usr/src/app/config - name: config-data-volume - readinessProbe: - httpGet: - path: / - port: 5000 - initialDelaySeconds: 5 - periodSeconds: 1 - resources: - limits: - cpu: 1000m - memory: 1024M - requests: - cpu: 500m - memory: 512M - volumes: - - name: config-data-volume - configMap: - name: config-service2 ---- -apiVersion: v1 -kind: Service -metadata: - name: service2 - namespace: edge-namespace - labels: - version: cluster2 - annotations: - http: / -spec: - selector: - app: service2 - ports: - - name: http - port: 80 - targetPort: 5000 diff --git a/generator/src/pkg/model/input.go b/generator/src/pkg/model/input.go index 37dc9af9..4bd9b29c 100644 --- a/generator/src/pkg/model/input.go +++ b/generator/src/pkg/model/input.go @@ -49,6 +49,7 @@ type NetworkComplexity struct { type Endpoint struct { Name string `json:"name"` Protocol string `json:"protocol"` + ExecutionMode string `json:"execution_mode"` CpuComplexity CpuComplexity `json:"cpu_complexity"` MemoryComplexity MemoryComplexity `json:"memory_complexity"` NetworkComplexity NetworkComplexity `json:"network_complexity"` diff --git a/generator/src/pkg/service/util.go b/generator/src/pkg/service/util.go index 5363f097..ca69bf85 100644 --- a/generator/src/pkg/service/util.go +++ b/generator/src/pkg/service/util.go @@ -47,6 +47,7 @@ const ( SvcReadinessProbeDefault = 5 EpNamePrefix = "/end" + EpExecModeDefault = "sequential" EpNwResponseSizeDefault = 512 EpExecTimeDefault = "10s" @@ -318,6 +319,8 @@ func CreateInputEndpoint() model.Endpoint { ep.Protocol = defaultProtocol + ep.ExecutionMode = EpExecModeDefault + ep.CpuComplexity.ExecutionTime = EpExecTimeDefault ep.CpuComplexity.Method = EpMethodDefault ep.CpuComplexity.Workers = EpWorkersDefault diff --git a/model/restful/utils/task.py b/model/restful/utils/task.py index 96dfe264..123e42bb 100644 --- a/model/restful/utils/task.py +++ b/model/restful/utils/task.py @@ -23,6 +23,7 @@ import uuid import subprocess import sys +from concurrent.futures import ThreadPoolExecutor, as_completed FORMATTED_REMOTE_URL = "http://{0}:{1}/{2}" @@ -55,15 +56,84 @@ def run_task(service_name, service_endpoint): source_svc["service"] = service_name source_svc["endpoint"] = service_endpoint["name"] - # CPU task - if service_endpoint["cpu_complexity"]: - cpu_response = execute_cpu_bounded_task(conf=service_endpoint["cpu_complexity"]) + execution_mode = service_endpoint["execution_mode"] + + if execution_mode == "sequential": + # CPU task + if service_endpoint["cpu_complexity"]: + cpu_response, _ = execute_cpu_bounded_task(conf=service_endpoint["cpu_complexity"]) + + # Memory task + if service_endpoint["memory_complexity"]: + mem_response, _ = execute_memory_bounded_task(conf=service_endpoint["memory_complexity"]) + + # Network task + if service_endpoint["network_complexity"]: + nw_response, _ = run_network_task(source_svc, service_endpoint, headers, res_payload) + + # TODO: Change this way and create a new map + nw_response["cpu_task"]["statuses"].append(cpu_response["status"]) + nw_response["memory_task"]["statuses"].append(mem_response["status"]) + + else: # "parallel" + executor = ThreadPoolExecutor(max_workers=3) + task_futures = [] + + # CPU task + if service_endpoint["cpu_complexity"]: + cpu_future = executor.submit(execute_cpu_bounded_task, service_endpoint["cpu_complexity"]) + task_futures.append(cpu_future) + + # Memory task + if service_endpoint["memory_complexity"]: + mem_future = executor.submit(execute_memory_bounded_task, service_endpoint["memory_complexity"]) + task_futures.append(mem_future) + + # Network task + if service_endpoint["network_complexity"]: + nw_future = executor.submit(run_network_task, source_svc, service_endpoint, headers, res_payload) + task_futures.append(nw_future) + + # Wait until all threads are done with their tasks + for future in as_completed(task_futures): + response, task_type = future.result() + if task_type == "cpu": + cpu_response = response + elif task_type == "memory": + mem_response = response + elif task_type == "network": + nw_response = response + + # TODO: Change this way and create a new map + nw_response["cpu_task"]["statuses"].append(cpu_response["status"]) + nw_response["memory_task"]["statuses"].append(mem_response["status"]) + + executor.shutdown() + + return nw_response + + +def execute_cpu_bounded_task(conf): + res = subprocess.run(['stress-ng --class cpu --cpu %s --cpu-method %s --taskset %s --cpu-load %s --timeout %s --metrics-brief' % (conf["workers"], conf["method"], ",".join(str(cpu_id) for cpu_id in conf["cpu_affinity"]), conf["cpu_load"], conf["execution_time"])], capture_output=True, shell=True) + + response = { + "status": res.stderr.decode("utf-8") + } + + return response, "cpu" + + +def execute_memory_bounded_task(conf): + res = subprocess.run(['stress-ng --class memory --vm %s --vm-method %s --vm-bytes %s --timeout %s --metrics-brief' % (conf["workers"], conf["method"], conf["bytes_load"], conf["execution_time"])], capture_output=True, shell=True) + + response = { + "status": res.stderr.decode("utf-8") + } + + return response, "memory" - # Memory task - if service_endpoint["memory_complexity"]: - mem_response = execute_memory_bounded_task(conf=service_endpoint["memory_complexity"]) - # Network task +def run_network_task(source_svc, service_endpoint, headers, res_payload): if service_endpoint["network_complexity"]["forward_requests"] == "asynchronous": asyncio.set_event_loop(asyncio.new_event_loop()) loop = asyncio.get_event_loop() @@ -73,10 +143,7 @@ def run_task(service_name, service_endpoint): loop = asyncio.get_event_loop() nw_response = loop.run_until_complete(sync_network_task(source_svc, service_endpoint, headers, res_payload)) - nw_response["cpu_task"]["statuses"].append(cpu_response["status"]) - nw_response["memory_task"]["statuses"].append(mem_response["status"]) - - return nw_response + return nw_response, "network" async def async_network_task(source_svc, service_endpoint, headers, res_payload): @@ -142,8 +209,8 @@ async def sync_network_task(source_svc, service_endpoint, headers, res_payload): response["cpu_task"]["services"] += res["cpu_task"]["services"] response["cpu_task"]["statuses"] += res["cpu_task"]["statuses"] - response["memory_task"]["services"] += svc["memory_task"]["services"] - response["memory_task"]["statuses"] += svc["memory_task"]["statuses"] + response["memory_task"]["services"] += res["memory_task"]["services"] + response["memory_task"]["statuses"] += res["memory_task"]["statuses"] response["network_task"]["services"] += res["network_task"]["services"] response["network_task"]["statuses"] += res["network_task"]["statuses"] @@ -151,24 +218,6 @@ async def sync_network_task(source_svc, service_endpoint, headers, res_payload): return response -def execute_cpu_bounded_task(conf): - res = subprocess.run(['stress-ng --class cpu --cpu %s --cpu-method %s --taskset %s --cpu-load %s --timeout %s --metrics-brief' % (conf["workers"], conf["method"], ",".join(str(cpu_id) for cpu_id in conf["cpu_affinity"]), conf["cpu_load"], conf["execution_time"])], capture_output=True, shell=True) - - response = { - "status": res.stderr.decode("utf-8") - } - return response - - -def execute_memory_bounded_task(conf): - res = subprocess.run(['stress-ng --class memory --vm %s --vm-method %s --vm-bytes %s --timeout %s --metrics-brief' % (conf["workers"], conf["method"], conf["bytes_load"], conf["execution_time"])], capture_output=True, shell=True) - - response = { - "status": res.stderr.decode("utf-8") - } - return response - - async def execute_io_bounded_task(session, source_service, target_service, sync, forward_headers={}): dst = FORMATTED_REMOTE_URL.format(target_service["service"], target_service["port"], target_service["endpoint"]) From 6f028604baef11d8492c92a281a0f632141b0151 Mon Sep 17 00:00:00 2001 From: Aleksandra Obeso Duque Date: Tue, 26 Apr 2022 07:15:43 +0000 Subject: [PATCH 12/18] removing async code in sync mode --- model/restful/utils/task.py | 245 +++++++++++++++++------------------- 1 file changed, 115 insertions(+), 130 deletions(-) diff --git a/model/restful/utils/task.py b/model/restful/utils/task.py index 123e42bb..5c8e5918 100644 --- a/model/restful/utils/task.py +++ b/model/restful/utils/task.py @@ -14,16 +14,14 @@ limitations under the License. """ -import logging from wsgiref import headers from flask import Blueprint, jsonify, request -import path from aiohttp import ClientSession import asyncio -import uuid import subprocess import sys from concurrent.futures import ThreadPoolExecutor, as_completed +import requests FORMATTED_REMOTE_URL = "http://{0}:{1}/{2}" @@ -48,37 +46,40 @@ def getForwardHeaders(request): def run_task(service_name, service_endpoint): headers = getForwardHeaders(request) - response_payload_size = service_endpoint["network_complexity"]["response_payload_size"] - response_payload = subprocess.run(['cat /dev/urandom | tr -dc "[:alnum:]" | head -c${1:-%s}' % response_payload_size], capture_output=True, shell=True) - res_payload = response_payload.stdout.decode("utf-8") + res_payload = create_payload(service_endpoint["network_complexity"]["response_payload_size"]) source_svc = {} source_svc["service"] = service_name source_svc["endpoint"] = service_endpoint["name"] - execution_mode = service_endpoint["execution_mode"] + response = create_response() + execution_mode = service_endpoint["execution_mode"] if execution_mode == "sequential": + # Network task + # NOTE: Network complexity shall not be optional + nw_response, _ = run_network_task(source_svc, service_endpoint, headers, res_payload) + response = concatenate_response_simple(response, nw_response) + # CPU task if service_endpoint["cpu_complexity"]: cpu_response, _ = execute_cpu_bounded_task(conf=service_endpoint["cpu_complexity"]) + response["cpu_task"]["statuses"].append(cpu_response["status"]) # Memory task if service_endpoint["memory_complexity"]: mem_response, _ = execute_memory_bounded_task(conf=service_endpoint["memory_complexity"]) - - # Network task - if service_endpoint["network_complexity"]: - nw_response, _ = run_network_task(source_svc, service_endpoint, headers, res_payload) - - # TODO: Change this way and create a new map - nw_response["cpu_task"]["statuses"].append(cpu_response["status"]) - nw_response["memory_task"]["statuses"].append(mem_response["status"]) + response["memory_task"]["statuses"].append(mem_response["status"]) else: # "parallel" executor = ThreadPoolExecutor(max_workers=3) task_futures = [] + # Network task + # NOTE: Network complexity shall not be optional + nw_future = executor.submit(run_network_task, source_svc, service_endpoint, headers, res_payload) + task_futures.append(nw_future) + # CPU task if service_endpoint["cpu_complexity"]: cpu_future = executor.submit(execute_cpu_bounded_task, service_endpoint["cpu_complexity"]) @@ -89,11 +90,6 @@ def run_task(service_name, service_endpoint): mem_future = executor.submit(execute_memory_bounded_task, service_endpoint["memory_complexity"]) task_futures.append(mem_future) - # Network task - if service_endpoint["network_complexity"]: - nw_future = executor.submit(run_network_task, source_svc, service_endpoint, headers, res_payload) - task_futures.append(nw_future) - # Wait until all threads are done with their tasks for future in as_completed(task_futures): response, task_type = future.result() @@ -104,13 +100,15 @@ def run_task(service_name, service_endpoint): elif task_type == "network": nw_response = response - # TODO: Change this way and create a new map - nw_response["cpu_task"]["statuses"].append(cpu_response["status"]) - nw_response["memory_task"]["statuses"].append(mem_response["status"]) + response = concatenate_response_simple(response, nw_response) + if service_endpoint["cpu_complexity"]: + response["cpu_task"]["statuses"].append(cpu_response["status"]) + if service_endpoint["memory_complexity"]: + response["memory_task"]["statuses"].append(mem_response["status"]) executor.shutdown() - return nw_response + return response def execute_cpu_bounded_task(conf): @@ -134,100 +132,107 @@ def execute_memory_bounded_task(conf): def run_network_task(source_svc, service_endpoint, headers, res_payload): + if service_endpoint["network_complexity"]["forward_requests"] == "asynchronous": asyncio.set_event_loop(asyncio.new_event_loop()) loop = asyncio.get_event_loop() nw_response = loop.run_until_complete(async_network_task(source_svc, service_endpoint, headers, res_payload)) else: # "synchronous" - asyncio.set_event_loop(asyncio.new_event_loop()) - loop = asyncio.get_event_loop() - nw_response = loop.run_until_complete(sync_network_task(source_svc, service_endpoint, headers, res_payload)) + nw_response = sync_network_task(source_svc, service_endpoint, headers, res_payload) return nw_response, "network" async def async_network_task(source_svc, service_endpoint, headers, res_payload): + async with ClientSession() as session: + response = create_response() + response["network_task"]["payload"] = res_payload + io_tasks = [] if len(service_endpoint["network_complexity"]["called_services"]) > 0: for target_svc in service_endpoint["network_complexity"]["called_services"]: - io_task = asyncio.create_task(execute_io_bounded_task(session=session, source_service=source_svc, target_service=target_svc, sync=False, forward_headers=headers)) + io_task = asyncio.create_task(async_execute_io_bounded_task(session=session, source_service=source_svc, target_service=target_svc, forward_headers=headers)) io_tasks.append(io_task) services = await asyncio.gather(*io_tasks) - # Concatenate json responses - response = {} - response["cpu_task"] = {} - response["cpu_task"]["services"] = [] - response["cpu_task"]["statuses"] = [] + if len(service_endpoint["network_complexity"]["called_services"]) > 0: + for svc in services: + response = concatenate_response_simple(response, svc) - response["memory_task"] = {} - response["memory_task"]["services"] = [] - response["memory_task"]["statuses"] = [] + return response - response["network_task"] = {} - response["network_task"]["services"] = [] - response["network_task"]["statuses"] = [] - response["network_task"]["payload"] = res_payload +def sync_network_task(source_svc, service_endpoint, headers, res_payload): + + response = create_response() + response["network_task"]["payload"] = res_payload + if len(service_endpoint["network_complexity"]["called_services"]) > 0: - for svc in services: - response["cpu_task"]["services"] += svc["cpu_task"]["services"] - response["cpu_task"]["statuses"] += svc["cpu_task"]["statuses"] + for target_svc in service_endpoint["network_complexity"]["called_services"]: + res = sync_execute_io_bounded_task(source_service=source_svc, target_service=target_svc, forward_headers=headers) + response = concatenate_response_simple(response, res) + + return response - response["memory_task"]["services"] += svc["memory_task"]["services"] - response["memory_task"]["statuses"] += svc["memory_task"]["statuses"] - response["network_task"]["services"] += svc["network_task"]["services"] - response["network_task"]["statuses"] += svc["network_task"]["statuses"] +async def async_execute_io_bounded_task(session, source_service, target_service, forward_headers={}): - return response + dst = FORMATTED_REMOTE_URL.format(target_service["service"], target_service["port"], target_service["endpoint"]) + forward_headers.update({'Content-type' : 'application/json'}) + response = create_response() -async def sync_network_task(source_svc, service_endpoint, headers, res_payload): - async with ClientSession() as session: - response = {} - response["cpu_task"] = {} - response["cpu_task"]["services"] = [] - response["cpu_task"]["statuses"] = [] - - response["memory_task"] = {} - response["memory_task"]["services"] = [] - response["memory_task"]["statuses"] = [] - - response["network_task"] = {} - response["network_task"]["services"] = [] - response["network_task"]["statuses"] = [] - response["network_task"]["payload"] = res_payload - - if len(service_endpoint["network_complexity"]["called_services"]) > 0: - for target_svc in service_endpoint["network_complexity"]["called_services"]: - res = await execute_io_bounded_task(session=session, source_service=source_svc, target_service=target_svc, sync=True, forward_headers=headers) - - # Concatenate json responses - response["cpu_task"]["services"] += res["cpu_task"]["services"] - response["cpu_task"]["statuses"] += res["cpu_task"]["statuses"] + json_data = {} + json_data["payload"] = create_payload(target_service["request_payload_size"]) - response["memory_task"]["services"] += res["memory_task"]["services"] - response["memory_task"]["statuses"] += res["memory_task"]["statuses"] + forward_ratio = target_service["traffic_forward_ratio"] + if forward_ratio > 0: + async with ClientSession() as session: + io_tasks = [] - response["network_task"]["services"] += res["network_task"]["services"] - response["network_task"]["statuses"] += res["network_task"]["statuses"] + for i in range(forward_ratio): + io_task = asyncio.create_task(session.post(dst, data=json_data, headers=forward_headers)) + io_tasks.append(io_task) + calls = await asyncio.gather(*io_tasks) + + for res in calls: + res_payload = await res.json() + response = concatenate_response(response, res_payload, source_service, target_service) + response["network_task"]['statuses'].append(res.status) return response -async def execute_io_bounded_task(session, source_service, target_service, sync, forward_headers={}): +def sync_execute_io_bounded_task(source_service, target_service, forward_headers={}): dst = FORMATTED_REMOTE_URL.format(target_service["service"], target_service["port"], target_service["endpoint"]) forward_headers.update({'Content-type' : 'application/json'}) + response = create_response() + json_data = {} - json_data["payload"] = "" + json_data["payload"] = create_payload(target_service["request_payload_size"]) + + forward_ratio = target_service["traffic_forward_ratio"] + if forward_ratio > 0: + for i in range(forward_ratio): + res = requests.post(dst, data=json_data, headers=forward_headers) + response = concatenate_response(response, res.json(), source_service, target_service) + response["network_task"]['statuses'].append(res.status_code) + + return response - forward_ratio = target_service["traffic_forward_ratio"] - request_payload_size = target_service["request_payload_size"] + +def create_payload(payload_size): + + request_payload = subprocess.run(['cat /dev/urandom | tr -dc "[:alnum:]" | head -c${1:-%s}' % payload_size], capture_output=True, shell=True) + + return request_payload.stdout.decode("utf-8") + + +def create_response(): response = {} response["cpu_task"] = {} @@ -242,56 +247,36 @@ async def execute_io_bounded_task(session, source_service, target_service, sync, response["network_task"]["services"] = [] response["network_task"]["statuses"] = [] + return response + + +def concatenate_response_simple(response, res): + + response["cpu_task"]["services"] += res["cpu_task"]["services"] + response["cpu_task"]["statuses"] += res["cpu_task"]["statuses"] + + response["memory_task"]["services"] += res["memory_task"]["services"] + response["memory_task"]["statuses"] += res["memory_task"]["statuses"] + + response["network_task"]["services"] += res["network_task"]["services"] + response["network_task"]["statuses"] += res["network_task"]["statuses"] + + return response + + +def concatenate_response(response, res_payload, source_service, target_service): + + response["cpu_task"]["services"] += res_payload["cpu_task"]["services"] + #response["cpu_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") + response["cpu_task"]['services'].append(target_service["service"]+"/"+target_service["endpoint"]) + response["cpu_task"]["statuses"] += res_payload["cpu_task"]["statuses"] + + response["memory_task"]["services"] += res_payload["memory_task"]["services"] + response["memory_task"]['services'].append(target_service["service"]+"/"+target_service["endpoint"]) + response["memory_task"]["statuses"] += res_payload["memory_task"]["statuses"] + + response["network_task"]["services"] += res_payload["network_task"]["services"] + response["network_task"]['services'].append(target_service["service"]+"/"+target_service["endpoint"]) + response["network_task"]["statuses"] += res_payload["network_task"]["statuses"] - if request_payload_size: - request_payload = subprocess.run(['cat /dev/urandom | tr -dc "[:alnum:]" | head -c${1:-%s}' % request_payload_size], capture_output=True, shell=True) - json_data["payload"] = request_payload.stdout.decode("utf-8") - - if forward_ratio > 0: - if not sync: - async with ClientSession() as session: - io_tasks = [] - - for i in range(forward_ratio): - io_task = asyncio.create_task(session.post(dst, data=json_data, headers=forward_headers)) - io_tasks.append(io_task) - calls = await asyncio.gather(*io_tasks) - - # Concatenate json responses - for res in calls: - res_payload = await res.json() - - response["cpu_task"]["services"] += res_payload["cpu_task"]["services"] - response["cpu_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") - response["cpu_task"]["statuses"] += res_payload["cpu_task"]["statuses"] - - response["memory_task"]["services"] += res_payload["memory_task"]["services"] - response["memory_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") - response["memory_task"]["statuses"] += res_payload["memory_task"]["statuses"] - - response["network_task"]["services"] += res_payload["network_task"]["services"] - response["network_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") - response["network_task"]["statuses"] += res_payload["network_task"]["statuses"] - response["network_task"]['statuses'].append(res.status) - - else: # "synchronous" - async with ClientSession() as session: - for i in range(forward_ratio): - res = await session.post(dst, data=json_data, headers=forward_headers) - res_payload = await res.json() - - # Concatenate json responses - response["cpu_task"]["services"] += res_payload["cpu_task"]["services"] - response["cpu_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") - response["cpu_task"]["statuses"] += res_payload["cpu_task"]["statuses"] - - response["memory_task"]["services"] += res_payload["memory_task"]["services"] - response["memory_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") - response["memory_task"]["statuses"] += res_payload["memory_task"]["statuses"] - - response["network_task"]["services"] += res_payload["network_task"]["services"] - response["network_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") - response["network_task"]["statuses"] += res_payload["network_task"]["statuses"] - response["network_task"]['statuses'].append(res.status) - return response \ No newline at end of file From c6859c5ee5bd37bc19a2508863212fefffb8dd1e Mon Sep 17 00:00:00 2001 From: Aleksandra Obeso Duque Date: Tue, 26 Apr 2022 11:28:57 +0000 Subject: [PATCH 13/18] fixing response format --- model/restful/utils/task.py | 66 ++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 38 deletions(-) diff --git a/model/restful/utils/task.py b/model/restful/utils/task.py index 5c8e5918..fcf72935 100644 --- a/model/restful/utils/task.py +++ b/model/restful/utils/task.py @@ -57,54 +57,52 @@ def run_task(service_name, service_endpoint): execution_mode = service_endpoint["execution_mode"] if execution_mode == "sequential": # Network task - # NOTE: Network complexity shall not be optional - nw_response, _ = run_network_task(source_svc, service_endpoint, headers, res_payload) - response = concatenate_response_simple(response, nw_response) + if ("network_complexity" in service_endpoint) and (len(service_endpoint["network_complexity"]["called_services"]) > 0): + nw_response, _ = run_network_task(source_svc, service_endpoint, headers, res_payload) + response = concatenate_response_simple(response, nw_response) # CPU task - if service_endpoint["cpu_complexity"]: + if ("cpu_complexity" in service_endpoint) and len(service_endpoint["cpu_complexity"]["execution_time"]) > 0: cpu_response, _ = execute_cpu_bounded_task(conf=service_endpoint["cpu_complexity"]) - response["cpu_task"]["statuses"].append(cpu_response["status"]) + response["cpu_task"]["services"].append(source_svc["service"]+"/"+source_svc["endpoint"]) + response["cpu_task"]["statuses"].append(cpu_response) # Memory task - if service_endpoint["memory_complexity"]: + if ("memory_complexity" in service_endpoint) and len(service_endpoint["memory_complexity"]["execution_time"]) > 0: mem_response, _ = execute_memory_bounded_task(conf=service_endpoint["memory_complexity"]) - response["memory_task"]["statuses"].append(mem_response["status"]) + response["memory_task"]["services"].append(source_svc["service"]+"/"+source_svc["endpoint"]) + response["memory_task"]["statuses"].append(mem_response) else: # "parallel" executor = ThreadPoolExecutor(max_workers=3) task_futures = [] # Network task - # NOTE: Network complexity shall not be optional - nw_future = executor.submit(run_network_task, source_svc, service_endpoint, headers, res_payload) - task_futures.append(nw_future) + if ("network_complexity" in service_endpoint) and (len(service_endpoint["network_complexity"]["called_services"]) > 0): + nw_future = executor.submit(run_network_task, source_svc, service_endpoint, headers, res_payload) + task_futures.append(nw_future) # CPU task - if service_endpoint["cpu_complexity"]: + if ("cpu_complexity" in service_endpoint) and len(service_endpoint["cpu_complexity"]["execution_time"]) > 0: cpu_future = executor.submit(execute_cpu_bounded_task, service_endpoint["cpu_complexity"]) task_futures.append(cpu_future) # Memory task - if service_endpoint["memory_complexity"]: + if ("memory_complexity" in service_endpoint) and len(service_endpoint["memory_complexity"]["execution_time"]) > 0: mem_future = executor.submit(execute_memory_bounded_task, service_endpoint["memory_complexity"]) task_futures.append(mem_future) # Wait until all threads are done with their tasks for future in as_completed(task_futures): - response, task_type = future.result() - if task_type == "cpu": - cpu_response = response + r, task_type = future.result() + if task_type == "network": + response = concatenate_response_simple(response, r) + elif task_type == "cpu": + response["cpu_task"]["services"].append(source_svc["service"]+"/"+source_svc["endpoint"]) + response["cpu_task"]["statuses"].append(r) elif task_type == "memory": - mem_response = response - elif task_type == "network": - nw_response = response - - response = concatenate_response_simple(response, nw_response) - if service_endpoint["cpu_complexity"]: - response["cpu_task"]["statuses"].append(cpu_response["status"]) - if service_endpoint["memory_complexity"]: - response["memory_task"]["statuses"].append(mem_response["status"]) + response["memory_task"]["services"].append(source_svc["service"]+"/"+source_svc["endpoint"]) + response["memory_task"]["statuses"].append(r) executor.shutdown() @@ -114,21 +112,13 @@ def run_task(service_name, service_endpoint): def execute_cpu_bounded_task(conf): res = subprocess.run(['stress-ng --class cpu --cpu %s --cpu-method %s --taskset %s --cpu-load %s --timeout %s --metrics-brief' % (conf["workers"], conf["method"], ",".join(str(cpu_id) for cpu_id in conf["cpu_affinity"]), conf["cpu_load"], conf["execution_time"])], capture_output=True, shell=True) - response = { - "status": res.stderr.decode("utf-8") - } - - return response, "cpu" + return res.stderr.decode("utf-8"), "cpu" def execute_memory_bounded_task(conf): res = subprocess.run(['stress-ng --class memory --vm %s --vm-method %s --vm-bytes %s --timeout %s --metrics-brief' % (conf["workers"], conf["method"], conf["bytes_load"], conf["execution_time"])], capture_output=True, shell=True) - response = { - "status": res.stderr.decode("utf-8") - } - - return response, "memory" + return res.stderr.decode("utf-8"), "memory" def run_network_task(source_svc, service_endpoint, headers, res_payload): @@ -246,6 +236,7 @@ def create_response(): response["network_task"] = {} response["network_task"]["services"] = [] response["network_task"]["statuses"] = [] + response["network_task"]["payload"] = "" return response @@ -260,6 +251,7 @@ def concatenate_response_simple(response, res): response["network_task"]["services"] += res["network_task"]["services"] response["network_task"]["statuses"] += res["network_task"]["statuses"] + response["network_task"]["payload"] += res["network_task"]["payload"] return response @@ -267,16 +259,14 @@ def concatenate_response_simple(response, res): def concatenate_response(response, res_payload, source_service, target_service): response["cpu_task"]["services"] += res_payload["cpu_task"]["services"] - #response["cpu_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") - response["cpu_task"]['services'].append(target_service["service"]+"/"+target_service["endpoint"]) response["cpu_task"]["statuses"] += res_payload["cpu_task"]["statuses"] response["memory_task"]["services"] += res_payload["memory_task"]["services"] - response["memory_task"]['services'].append(target_service["service"]+"/"+target_service["endpoint"]) response["memory_task"]["statuses"] += res_payload["memory_task"]["statuses"] response["network_task"]["services"] += res_payload["network_task"]["services"] - response["network_task"]['services'].append(target_service["service"]+"/"+target_service["endpoint"]) + response["network_task"]['services'].append("("+source_service["service"]+"/"+source_service["endpoint"]+", "+target_service["service"]+"/"+target_service["endpoint"]+")") response["network_task"]["statuses"] += res_payload["network_task"]["statuses"] + response["network_task"]["payload"] = res_payload["network_task"]["payload"] return response \ No newline at end of file From 44a74152c7324383be86cd96fa3c3e3452b1343f Mon Sep 17 00:00:00 2001 From: Aleksandra Obeso Duque Date: Tue, 26 Apr 2022 11:38:19 +0000 Subject: [PATCH 14/18] updating simple example --- .../simple/application_description.json | 59 +++++++++++-------- .../examples/simple/cluster1/service1.yaml | 4 +- .../{cluster2 => cluster1}/service2.yaml | 16 ++--- 3 files changed, 46 insertions(+), 33 deletions(-) rename generator/examples/simple/{cluster2 => cluster1}/service2.yaml (58%) diff --git a/generator/examples/simple/application_description.json b/generator/examples/simple/application_description.json index 2069929e..30ac48f4 100644 --- a/generator/examples/simple/application_description.json +++ b/generator/examples/simple/application_description.json @@ -31,25 +31,9 @@ "readiness_probe": 1, "endpoints": [ { - "name": "end1", + "name": "endpoint1", "protocol": "http", - "execution_mode": "parallel", - "cpu_complexity": { - "execution_time": "5s", - "method": "fibonacci", - "workers": 2, - "cpu_affinity": [ - 0, - 1 - ], - "cpu_load": "100%" - }, - "memory_complexity": { - "execution_time": "5s", - "method": "swap", - "workers": 24, - "bytes_load": "100%" - }, + "execution_mode": "sequential", "network_complexity": { "forward_requests": "asynchronous", "response_payload_size": 512, @@ -57,13 +41,32 @@ { "service": "service2", "port": "80", - "endpoint": "end2", + "endpoint": "endpoint1", + "protocol": "http", + "traffic_forward_ratio": 1, + "request_payload_size": 256 + }, + { + "service": "service2", + "port": "80", + "endpoint": "endpoint2", "protocol": "http", "traffic_forward_ratio": 1, "request_payload_size": 256 } ] } + }, + { + "name": "endpoint2", + "protocol": "http", + "execution_mode": "parallel", + "network_complexity": { + "forward_requests": "asynchronous", + "response_payload_size": 512, + "called_services": [ + ] + } } ] }, @@ -71,9 +74,9 @@ "name": "service2", "clusters": [ { - "cluster": "cluster2", + "cluster": "cluster1", "namespace": "edge-namespace", - "node": "cluster2-control-plane" + "node": "cluster1-control-plane" } ], "resources": { @@ -91,9 +94,9 @@ "readiness_probe": 1, "endpoints": [ { - "name": "end2", + "name": "endpoint1", "protocol": "http", - "execution_mode": "sequential", + "execution_mode": "parallel", "cpu_complexity": { "execution_time": "5s", "method": "fibonacci", @@ -115,6 +118,16 @@ "response_payload_size": 512, "called_services": [] } + }, + { + "name": "endpoint2", + "protocol": "http", + "execution_mode": "parallel", + "network_complexity": { + "forward_requests": "asynchronous", + "response_payload_size": 512, + "called_services": [] + } } ] } diff --git a/generator/examples/simple/cluster1/service1.yaml b/generator/examples/simple/cluster1/service1.yaml index 6edd9f6b..26f23243 100644 --- a/generator/examples/simple/cluster1/service1.yaml +++ b/generator/examples/simple/cluster1/service1.yaml @@ -7,8 +7,8 @@ metadata: version: cluster1 namespace: edge-namespace data: - conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end1","protocol":"http","execution_mode":"parallel","cpu_complexity":{"execution_time":"5s","method":"fibonacci","workers":2,"cpu_affinity":[0,1],"cpu_load":"100%"},"memory_complexity":{"execution_time":"5s","method":"swap","workers":24,"bytes_load":"100%"},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[{"service":"service2","port":"80","endpoint":"end2","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256}]}}]}' - service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc end2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" + conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"endpoint1","protocol":"http","execution_mode":"sequential","cpu_complexity":{"execution_time":"","method":"","workers":0,"cpu_affinity":null,"cpu_load":""},"memory_complexity":{"execution_time":"","method":"","workers":0,"bytes_load":""},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[{"service":"service2","port":"80","endpoint":"endpoint1","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256},{"service":"service2","port":"80","endpoint":"endpoint2","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256}]}},{"name":"endpoint2","protocol":"http","execution_mode":"parallel","cpu_complexity":{"execution_time":"","method":"","workers":0,"cpu_affinity":null,"cpu_load":""},"memory_complexity":{"execution_time":"","method":"","workers":0,"bytes_load":""},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' + service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc endpoint1 (Request) returns (Response) {}\n \n rpc endpoint2 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc endpoint1 (Request) returns (Response) {}\n \n rpc endpoint2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" --- apiVersion: apps/v1 kind: Deployment diff --git a/generator/examples/simple/cluster2/service2.yaml b/generator/examples/simple/cluster1/service2.yaml similarity index 58% rename from generator/examples/simple/cluster2/service2.yaml rename to generator/examples/simple/cluster1/service2.yaml index d12fd836..2c03c16b 100644 --- a/generator/examples/simple/cluster2/service2.yaml +++ b/generator/examples/simple/cluster1/service2.yaml @@ -4,11 +4,11 @@ metadata: name: config-service2 labels: name: config-service2 - version: cluster2 + version: cluster1 namespace: edge-namespace data: - conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"end2","protocol":"http","execution_mode":"sequential","cpu_complexity":{"execution_time":"5s","method":"fibonacci","workers":2,"cpu_affinity":[22,23],"cpu_load":"100%"},"memory_complexity":{"execution_time":"5s","method":"swap","workers":24,"bytes_load":"100%"},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' - service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc end1 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc end2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" + conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"endpoint1","protocol":"http","execution_mode":"parallel","cpu_complexity":{"execution_time":"5s","method":"fibonacci","workers":2,"cpu_affinity":[22,23],"cpu_load":"100%"},"memory_complexity":{"execution_time":"5s","method":"swap","workers":24,"bytes_load":"100%"},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}},{"name":"endpoint2","protocol":"http","execution_mode":"parallel","cpu_complexity":{"execution_time":"","method":"","workers":0,"cpu_affinity":null,"cpu_load":""},"memory_complexity":{"execution_time":"","method":"","workers":0,"bytes_load":""},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' + service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc endpoint1 (Request) returns (Response) {}\n \n rpc endpoint2 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc endpoint1 (Request) returns (Response) {}\n \n rpc endpoint2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" --- apiVersion: apps/v1 kind: Deployment @@ -16,20 +16,20 @@ metadata: name: service2 namespace: edge-namespace labels: - version: cluster2 + version: cluster1 spec: selector: matchLabels: app: service2 - version: cluster2 + version: cluster1 replicas: 1 template: metadata: labels: app: service2 - version: cluster2 + version: cluster1 spec: - nodeName: cluster2-control-plane + nodeName: cluster1-control-plane containers: - name: app image: app-demo:latest @@ -66,7 +66,7 @@ metadata: name: service2 namespace: edge-namespace labels: - version: cluster2 + version: cluster1 annotations: http: / spec: From d3fda0d40be6d64c419bb1bad4b7eb61f781aece Mon Sep 17 00:00:00 2001 From: Aleksandra Obeso Duque Date: Tue, 26 Apr 2022 16:13:17 +0000 Subject: [PATCH 15/18] minimizing default values --- generator/src/pkg/service/util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/src/pkg/service/util.go b/generator/src/pkg/service/util.go index ca69bf85..2e78fa7d 100644 --- a/generator/src/pkg/service/util.go +++ b/generator/src/pkg/service/util.go @@ -50,10 +50,10 @@ const ( EpExecModeDefault = "sequential" EpNwResponseSizeDefault = 512 - EpExecTimeDefault = "10s" + EpExecTimeDefault = "1s" EpMethodDefault = "all" EpWorkersDefault = 1 - EpLoadDefault = "100%" + EpLoadDefault = "20%" EpNwForwardRequests = "asynchronous" From 15ef4183433c14a2979ec7b9cb4009d6ffad61ff Mon Sep 17 00:00:00 2001 From: Aleksandra Obeso Duque Date: Tue, 26 Apr 2022 17:16:11 +0000 Subject: [PATCH 16/18] fixing bug with random generation of empty lists --- generator/src/pkg/generate/generate.go | 2 +- generator/src/pkg/service/util.go | 8 +++++--- model/restful/utils/task.py | 5 ++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/generator/src/pkg/generate/generate.go b/generator/src/pkg/generate/generate.go index 86326496..eb875346 100644 --- a/generator/src/pkg/generate/generate.go +++ b/generator/src/pkg/generate/generate.go @@ -141,7 +141,7 @@ func CreateK8sYaml(config model.FileConfig, clusters []string) { for j := 0; j < len(config.Services[i].Clusters); j++ { directory := config.Services[i].Clusters[j].Cluster directory_path := fmt.Sprintf(path+"/%s", directory) - c_id := fmt.Sprintf("%s", config.Services[i].Clusters[j].Cluster) + c_id := config.Services[i].Clusters[j].Cluster nodeAffinity := config.Services[i].Clusters[j].Node namespace := config.Services[i].Clusters[j].Namespace manifestFilePath := fmt.Sprintf(directory_path+"/%s.yaml", serv) diff --git a/generator/src/pkg/service/util.go b/generator/src/pkg/service/util.go index 2e78fa7d..69eb6751 100644 --- a/generator/src/pkg/service/util.go +++ b/generator/src/pkg/service/util.go @@ -28,9 +28,9 @@ const ( ImageName = "app" ImageURL = "app-demo:latest" - DefaultExtPort = 80 // + DefaultExtPort = 80 DefaultPort = 5000 - defaultProtocol = "http" // + defaultProtocol = "http" Uri = "/" @@ -46,7 +46,7 @@ const ( SvcThreadsDefault = 2 SvcReadinessProbeDefault = 5 - EpNamePrefix = "/end" + EpNamePrefix = "end" EpExecModeDefault = "sequential" EpNwResponseSizeDefault = 512 @@ -324,6 +324,7 @@ func CreateInputEndpoint() model.Endpoint { ep.CpuComplexity.ExecutionTime = EpExecTimeDefault ep.CpuComplexity.Method = EpMethodDefault ep.CpuComplexity.Workers = EpWorkersDefault + ep.CpuComplexity.CpuAffinity = []int{} ep.CpuComplexity.CpuLoad = EpLoadDefault ep.MemoryComplexity.ExecutionTime = EpExecTimeDefault @@ -333,6 +334,7 @@ func CreateInputEndpoint() model.Endpoint { ep.NetworkComplexity.ForwardRequests = EpNwForwardRequests ep.NetworkComplexity.ResponsePayloadSize = EpNwResponseSizeDefault + ep.NetworkComplexity.CalledServices = []model.CalledService{} return ep } diff --git a/model/restful/utils/task.py b/model/restful/utils/task.py index fcf72935..f5930887 100644 --- a/model/restful/utils/task.py +++ b/model/restful/utils/task.py @@ -110,7 +110,10 @@ def run_task(service_name, service_endpoint): def execute_cpu_bounded_task(conf): - res = subprocess.run(['stress-ng --class cpu --cpu %s --cpu-method %s --taskset %s --cpu-load %s --timeout %s --metrics-brief' % (conf["workers"], conf["method"], ",".join(str(cpu_id) for cpu_id in conf["cpu_affinity"]), conf["cpu_load"], conf["execution_time"])], capture_output=True, shell=True) + if len(conf["cpu_affinity"]) > 0: + res = subprocess.run(['stress-ng --class cpu --cpu %s --cpu-method %s --taskset %s --cpu-load %s --timeout %s --metrics-brief' % (conf["workers"], conf["method"], ",".join(str(cpu_id) for cpu_id in conf["cpu_affinity"]), conf["cpu_load"], conf["execution_time"])], capture_output=True, shell=True) + else: + res = subprocess.run(['stress-ng --class cpu --cpu %s --cpu-method %s --cpu-load %s --timeout %s --metrics-brief' % (conf["workers"], conf["method"], conf["cpu_load"], conf["execution_time"])], capture_output=True, shell=True) return res.stderr.decode("utf-8"), "cpu" From ed9ff7de8fa6ed6ae61bafc2aa50903411a99376 Mon Sep 17 00:00:00 2001 From: Aleksandra Obeso Duque Date: Mon, 2 May 2022 10:57:49 +0000 Subject: [PATCH 17/18] fixing PR feedback --- .../examples/simple/application_description.json | 16 ++++++++-------- generator/examples/simple/cluster1/service1.yaml | 2 +- generator/examples/simple/cluster1/service2.yaml | 2 +- generator/input/new_description.json | 8 ++++---- generator/src/pkg/model/input.go | 4 ++-- generator/src/pkg/service/util.go | 8 ++++---- model/Dockerfile | 2 -- model/restful/utils/task.py | 1 - 8 files changed, 20 insertions(+), 23 deletions(-) diff --git a/generator/examples/simple/application_description.json b/generator/examples/simple/application_description.json index 30ac48f4..2e8cfe36 100644 --- a/generator/examples/simple/application_description.json +++ b/generator/examples/simple/application_description.json @@ -12,7 +12,7 @@ "clusters": [ { "cluster": "cluster1", - "namespace": "edge-namespace", + "namespace": "default", "node": "cluster1-control-plane" } ], @@ -75,7 +75,7 @@ "clusters": [ { "cluster": "cluster1", - "namespace": "edge-namespace", + "namespace": "default", "node": "cluster1-control-plane" } ], @@ -98,20 +98,20 @@ "protocol": "http", "execution_mode": "parallel", "cpu_complexity": { - "execution_time": "5s", + "execution_time": "1s", "method": "fibonacci", "workers": 2, "cpu_affinity": [ - 22, - 23 + 1, + 2 ], - "cpu_load": "100%" + "cpu_load": "10%" }, "memory_complexity": { - "execution_time": "5s", + "execution_time": "1s", "method": "swap", "workers": 24, - "bytes_load": "100%" + "bytes_load": "10%" }, "network_complexity": { "forward_requests": "asynchronous", diff --git a/generator/examples/simple/cluster1/service1.yaml b/generator/examples/simple/cluster1/service1.yaml index 26f23243..de6b2ede 100644 --- a/generator/examples/simple/cluster1/service1.yaml +++ b/generator/examples/simple/cluster1/service1.yaml @@ -7,7 +7,7 @@ metadata: version: cluster1 namespace: edge-namespace data: - conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"endpoint1","protocol":"http","execution_mode":"sequential","cpu_complexity":{"execution_time":"","method":"","workers":0,"cpu_affinity":null,"cpu_load":""},"memory_complexity":{"execution_time":"","method":"","workers":0,"bytes_load":""},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[{"service":"service2","port":"80","endpoint":"endpoint1","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256},{"service":"service2","port":"80","endpoint":"endpoint2","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256}]}},{"name":"endpoint2","protocol":"http","execution_mode":"parallel","cpu_complexity":{"execution_time":"","method":"","workers":0,"cpu_affinity":null,"cpu_load":""},"memory_complexity":{"execution_time":"","method":"","workers":0,"bytes_load":""},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' + conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"endpoint1","protocol":"http","execution_mode":"sequential","network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[{"service":"service2","port":"80","endpoint":"endpoint1","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256},{"service":"service2","port":"80","endpoint":"endpoint2","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256}]}},{"name":"endpoint2","protocol":"http","execution_mode":"parallel","network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc endpoint1 (Request) returns (Response) {}\n \n rpc endpoint2 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc endpoint1 (Request) returns (Response) {}\n \n rpc endpoint2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" --- apiVersion: apps/v1 diff --git a/generator/examples/simple/cluster1/service2.yaml b/generator/examples/simple/cluster1/service2.yaml index 2c03c16b..cd4f6d07 100644 --- a/generator/examples/simple/cluster1/service2.yaml +++ b/generator/examples/simple/cluster1/service2.yaml @@ -7,7 +7,7 @@ metadata: version: cluster1 namespace: edge-namespace data: - conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"endpoint1","protocol":"http","execution_mode":"parallel","cpu_complexity":{"execution_time":"5s","method":"fibonacci","workers":2,"cpu_affinity":[22,23],"cpu_load":"100%"},"memory_complexity":{"execution_time":"5s","method":"swap","workers":24,"bytes_load":"100%"},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}},{"name":"endpoint2","protocol":"http","execution_mode":"parallel","cpu_complexity":{"execution_time":"","method":"","workers":0,"cpu_affinity":null,"cpu_load":""},"memory_complexity":{"execution_time":"","method":"","workers":0,"bytes_load":""},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' + conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"endpoint1","protocol":"http","execution_mode":"parallel","cpu_complexity":{"execution_time":"1s","method":"fibonacci","workers":2,"cpu_affinity":[1,2],"cpu_load":"10%"},"memory_complexity":{"execution_time":"1s","method":"swap","workers":24,"bytes_load":"10%"},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}},{"name":"endpoint2","protocol":"http","execution_mode":"parallel","network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc endpoint1 (Request) returns (Response) {}\n \n rpc endpoint2 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc endpoint1 (Request) returns (Response) {}\n \n rpc endpoint2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" --- apiVersion: apps/v1 diff --git a/generator/input/new_description.json b/generator/input/new_description.json index 61034ddb..a80e23bc 100644 --- a/generator/input/new_description.json +++ b/generator/input/new_description.json @@ -35,20 +35,20 @@ "protocol": "http", "execution_mode": "sequential", "cpu_complexity": { - "execution_time": "10s", + "execution_time": "1s", "method": "fibonacci", "workers": 2, "cpu_affinity": [ 0, 2 ], - "cpu_load": "80%" + "cpu_load": "10%" }, "memory_complexity": { - "execution_time": "10s", + "execution_time": "1s", "method": "swap", "workers": 2, - "bytes_load": "80%" + "bytes_load": "10%" }, "network_complexity": { "forward_requests": "asynchronous", diff --git a/generator/src/pkg/model/input.go b/generator/src/pkg/model/input.go index 4bd9b29c..7b31e49d 100644 --- a/generator/src/pkg/model/input.go +++ b/generator/src/pkg/model/input.go @@ -50,8 +50,8 @@ type Endpoint struct { Name string `json:"name"` Protocol string `json:"protocol"` ExecutionMode string `json:"execution_mode"` - CpuComplexity CpuComplexity `json:"cpu_complexity"` - MemoryComplexity MemoryComplexity `json:"memory_complexity"` + CpuComplexity *CpuComplexity `json:"cpu_complexity,omitempty"` + MemoryComplexity *MemoryComplexity `json:"memory_complexity,omitempty"` NetworkComplexity NetworkComplexity `json:"network_complexity"` } diff --git a/generator/src/pkg/service/util.go b/generator/src/pkg/service/util.go index 69eb6751..5f3fc057 100644 --- a/generator/src/pkg/service/util.go +++ b/generator/src/pkg/service/util.go @@ -42,9 +42,9 @@ const ( LimitsMemoryDefault = "1024M" SvcNamePrefix = "service" - SvcProcessesDefault = 2 - SvcThreadsDefault = 2 - SvcReadinessProbeDefault = 5 + SvcProcessesDefault = 1 + SvcThreadsDefault = 1 + SvcReadinessProbeDefault = 2 EpNamePrefix = "end" EpExecModeDefault = "sequential" @@ -53,7 +53,7 @@ const ( EpExecTimeDefault = "1s" EpMethodDefault = "all" EpWorkersDefault = 1 - EpLoadDefault = "20%" + EpLoadDefault = "5%" EpNwForwardRequests = "asynchronous" diff --git a/model/Dockerfile b/model/Dockerfile index 502bdf61..7c460a8d 100644 --- a/model/Dockerfile +++ b/model/Dockerfile @@ -20,8 +20,6 @@ RUN mkdir -p /usr/src/app RUN apt update RUN apt install -y jq \ wget \ - curl \ - vim \ stress-ng \ 2to3 diff --git a/model/restful/utils/task.py b/model/restful/utils/task.py index f5930887..b8f2f468 100644 --- a/model/restful/utils/task.py +++ b/model/restful/utils/task.py @@ -14,7 +14,6 @@ limitations under the License. """ -from wsgiref import headers from flask import Blueprint, jsonify, request from aiohttp import ClientSession import asyncio From 2b77f269cbabc0356aa81aec11b6b6f048074ea0 Mon Sep 17 00:00:00 2001 From: Aleksandra Obeso Duque Date: Tue, 3 May 2022 08:48:43 +0000 Subject: [PATCH 18/18] updating namespace in yaml examples --- generator/examples/simple/cluster1/service1.yaml | 6 +++--- generator/examples/simple/cluster1/service2.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/generator/examples/simple/cluster1/service1.yaml b/generator/examples/simple/cluster1/service1.yaml index de6b2ede..70a34fbe 100644 --- a/generator/examples/simple/cluster1/service1.yaml +++ b/generator/examples/simple/cluster1/service1.yaml @@ -5,7 +5,7 @@ metadata: labels: name: config-service1 version: cluster1 - namespace: edge-namespace + namespace: default data: conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"endpoint1","protocol":"http","execution_mode":"sequential","network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[{"service":"service2","port":"80","endpoint":"endpoint1","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256},{"service":"service2","port":"80","endpoint":"endpoint2","protocol":"http","traffic_forward_ratio":1,"request_payload_size":256}]}},{"name":"endpoint2","protocol":"http","execution_mode":"parallel","network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc endpoint1 (Request) returns (Response) {}\n \n rpc endpoint2 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc endpoint1 (Request) returns (Response) {}\n \n rpc endpoint2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" @@ -14,7 +14,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: service1 - namespace: edge-namespace + namespace: default labels: version: cluster1 spec: @@ -64,7 +64,7 @@ apiVersion: v1 kind: Service metadata: name: service1 - namespace: edge-namespace + namespace: default labels: version: cluster1 annotations: diff --git a/generator/examples/simple/cluster1/service2.yaml b/generator/examples/simple/cluster1/service2.yaml index cd4f6d07..1af87453 100644 --- a/generator/examples/simple/cluster1/service2.yaml +++ b/generator/examples/simple/cluster1/service2.yaml @@ -5,7 +5,7 @@ metadata: labels: name: config-service2 version: cluster1 - namespace: edge-namespace + namespace: default data: conf.json: '{"processes":2,"threads":2,"endpoints":[{"name":"endpoint1","protocol":"http","execution_mode":"parallel","cpu_complexity":{"execution_time":"1s","method":"fibonacci","workers":2,"cpu_affinity":[1,2],"cpu_load":"10%"},"memory_complexity":{"execution_time":"1s","method":"swap","workers":24,"bytes_load":"10%"},"network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}},{"name":"endpoint2","protocol":"http","execution_mode":"parallel","network_complexity":{"forward_requests":"asynchronous","response_payload_size":512,"called_services":[]}}]}' service.proto: "syntax = \"proto3\";\n\n\nservice service1 {\n \n rpc endpoint1 (Request) returns (Response) {}\n \n rpc endpoint2 (Request) returns (Response) {}\n \n}\n\nservice service2 {\n \n rpc endpoint1 (Request) returns (Response) {}\n \n rpc endpoint2 (Request) returns (Response) {}\n \n}\n\n\nmessage Request {\n string data = 1;\n}\n\nmessage Response {\n string data = 1;\n}" @@ -14,7 +14,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: service2 - namespace: edge-namespace + namespace: default labels: version: cluster1 spec: @@ -64,7 +64,7 @@ apiVersion: v1 kind: Service metadata: name: service2 - namespace: edge-namespace + namespace: default labels: version: cluster1 annotations: