From 48744bd71b51d1439035c5adadba293439f0599f Mon Sep 17 00:00:00 2001 From: Mohammad Reza Saleh Date: Tue, 21 Jun 2022 15:33:02 +0200 Subject: [PATCH 1/6] remove stress-ng installation --- model/Dockerfile | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/model/Dockerfile b/model/Dockerfile index 3dbfea8b..0ec2cfa3 100644 --- a/model/Dockerfile +++ b/model/Dockerfile @@ -23,30 +23,9 @@ RUN apt install -y jq \ wget \ 2to3 -RUN apt install -y libbsd-dev \ - libcap-dev \ - libipsec-mb-dev \ - libjudy-dev \ - libkeyutils-dev \ - libsctp-dev \ - libatomic1 \ - zlib1g-dev \ - libkmod-dev \ - libxxhash-dev \ - git \ - build-essential WORKDIR /usr/src/app -RUN git clone https://github.com/alekodu/stress-ng.git &&\ - cd stress-ng/ &&\ - git checkout cloudsim &&\ - make clean &&\ - make &&\ - mv stress-ng my-stress-ng &&\ - cp my-stress-ng /usr/src/app &&\ - cd .. &&\ - rm -R stress-ng/ ADD ./requirements.txt /usr/src/app/requirements.txt From 24890a87aafa8a77875b897331b28781259ff6da Mon Sep 17 00:00:00 2001 From: Mohammad Reza Saleh Date: Tue, 21 Jun 2022 15:33:57 +0200 Subject: [PATCH 2/6] add cpu_load function and remove payload from udev/random --- model/restful/utils/task.py | 40 +++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/model/restful/utils/task.py b/model/restful/utils/task.py index a9f8b446..7643153d 100644 --- a/model/restful/utils/task.py +++ b/model/restful/utils/task.py @@ -18,9 +18,11 @@ from aiohttp import ClientSession import asyncio import subprocess -import sys from concurrent.futures import ThreadPoolExecutor, as_completed import requests +import time +import random +import string FORMATTED_REMOTE_URL = "http://{0}:{1}/{2}" @@ -87,10 +89,10 @@ def run_task(service_name, service_endpoint): cpu_future = executor.submit(execute_cpu_bounded_task, service_endpoint["cpu_complexity"]) task_futures.append(cpu_future) - # Memory task - if ("memory_complexity" in service_endpoint) and len(service_endpoint["memory_complexity"]["execution_time"]) > 0: - mem_future = executor.submit(execute_memory_bounded_task, service_endpoint["memory_complexity"]) - task_futures.append(mem_future) + # # Memory task + # if ("memory_complexity" in service_endpoint) and len(service_endpoint["memory_complexity"]["execution_time"]) > 0: + # mem_future = executor.submit(execute_memory_bounded_task, service_endpoint["memory_complexity"]) + # task_futures.append(mem_future) # Wait until all threads are done with their tasks for future in as_completed(task_futures): @@ -100,28 +102,31 @@ def run_task(service_name, service_endpoint): elif task_type == "cpu": response["cpu_task"]["services"].append(source_svc["service"]+"/"+source_svc["endpoint"]) response["cpu_task"]["statuses"].append(r) - elif task_type == "memory": - response["memory_task"]["services"].append(source_svc["service"]+"/"+source_svc["endpoint"]) - response["memory_task"]["statuses"].append(r) + # elif task_type == "memory": + # response["memory_task"]["services"].append(source_svc["service"]+"/"+source_svc["endpoint"]) + # response["memory_task"]["statuses"].append(r) executor.shutdown() return response -def execute_cpu_bounded_task(conf): - if len(conf["cpu_affinity"]) > 0: - res = subprocess.run(['/usr/src/app/my-stress-ng --class cpu --cpu %s --cpu-method %s --taskset %s --cpu-load %s --timeout %s --metrics-brief' % (conf["workers"], conf["method"], ",".join(str(cpu_id) for cpu_id in conf["cpu_affinity"]), conf["cpu_load"], conf["execution_time"])], capture_output=True, shell=True) - else: - res = subprocess.run(['/usr/src/app/my-stress-ng --class cpu --cpu %s --cpu-method %s --cpu-load %s --timeout %s --metrics-brief' % (conf["workers"], conf["method"], conf["cpu_load"], conf["execution_time"])], capture_output=True, shell=True) +def cpu_load(duration): + start = time.clock_gettime_ns(time.CLOCK_THREAD_CPUTIME_ID) + until = start + duration * 1000000000 + while time.clock_gettime_ns(time.CLOCK_THREAD_CPUTIME_ID) < until: + pass - return res.stderr.decode("utf-8"), "cpu" + +def execute_cpu_bounded_task(conf): + cpu_load(float(conf["execution_time"])) + return "execution_time: "+str(conf["execution_time"]), "cpu" def execute_memory_bounded_task(conf): - res = subprocess.run(['/usr/src/app/my-stress-ng --class memory --vm %s --vm-method %s --vm-bytes %s --timeout %s --metrics-brief' % (conf["workers"], conf["method"], conf["bytes_load"], conf["execution_time"])], capture_output=True, shell=True) + #res = subprocess.run(['/usr/src/app/my-stress-ng --class memory --vm %s --vm-method %s --vm-bytes %s --timeout %s --metrics-brief' % (conf["workers"], conf["method"], conf["bytes_load"], conf["execution_time"])], capture_output=True, shell=True) - return res.stderr.decode("utf-8"), "memory" + return "Not implemented yet", "memory" def run_network_task(source_svc, service_endpoint, headers, res_payload): @@ -221,7 +226,8 @@ def sync_execute_io_bounded_task(source_service, target_service, forward_headers def create_payload(payload_size): request_payload = subprocess.run(['cat /dev/urandom | tr -dc "[:alnum:]" | head -c${1:-%s}' % payload_size], capture_output=True, shell=True) - + letters = string.ascii_lowercase + result_str = ''.join(random.choice(letters) for i in range(int(payload_size))) return request_payload.stdout.decode("utf-8") From 231c9c0c8e83a2c9664f47dc624cb7dc3caf518c Mon Sep 17 00:00:00 2001 From: Mohammad Reza Saleh Date: Tue, 21 Jun 2022 15:48:46 +0200 Subject: [PATCH 3/6] remove memorycomplexity and update cpucomplexity --- generator/src/pkg/model/input.go | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/generator/src/pkg/model/input.go b/generator/src/pkg/model/input.go index 59bfb061..1a4ca130 100644 --- a/generator/src/pkg/model/input.go +++ b/generator/src/pkg/model/input.go @@ -26,18 +26,7 @@ type CalledService struct { } type CpuComplexity struct { - ExecutionTime string `json:"execution_time"` - Method string `json:"method"` - Workers int `json:"workers"` - CpuAffinity []int `json:"cpu_affinity"` - CpuLoad string `json:"cpu_load"` -} - -type MemoryComplexity struct { - ExecutionTime string `json:"execution_time"` - Method string `json:"method"` - Workers int `json:"workers"` - BytesLoad string `json:"bytes_load"` + ExecutionTime float32 `json:"execution_time"` } type NetworkComplexity struct { @@ -51,7 +40,6 @@ type Endpoint struct { Protocol string `json:"protocol"` ExecutionMode string `json:"execution_mode"` CpuComplexity *CpuComplexity `json:"cpu_complexity,omitempty"` - MemoryComplexity *MemoryComplexity `json:"memory_complexity,omitempty"` NetworkComplexity NetworkComplexity `json:"network_complexity"` } @@ -82,7 +70,7 @@ type Service struct { type Cluster struct { Cluster string `json:"cluster"` - Replicas int `json:"replicas,omitempty"` + Replicas int `json:"replicas,omitempty"` Namespace string `json:"namespace"` Node string `json:"node,omitempty"` Annotations []Annotation `json:"annotations,omitempty"` From a5ccb47643d193646aeb457f2319ac04d09b9650 Mon Sep 17 00:00:00 2001 From: Mohammad Reza Saleh Date: Tue, 21 Jun 2022 15:48:56 +0200 Subject: [PATCH 4/6] remove memorycomplexity and update cpucomplexity --- generator/src/pkg/service/util.go | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/generator/src/pkg/service/util.go b/generator/src/pkg/service/util.go index a87891d4..f92efb0d 100644 --- a/generator/src/pkg/service/util.go +++ b/generator/src/pkg/service/util.go @@ -48,10 +48,7 @@ const ( EpExecModeDefault = "sequential" EpNwResponseSizeDefault = 512 - EpExecTimeDefault = "0.1s" - EpMethodDefault = "all" - EpWorkersDefault = 1 - EpLoadDefault = "5%" + EpExecTimeDefault = 0.001 EpNwForwardRequests = "asynchronous" @@ -324,22 +321,11 @@ func CreateInputEndpoint() model.Endpoint { var ep model.Endpoint ep.Protocol = defaultProtocol var cpuComplexity model.CpuComplexity - var memoryComplexity model.MemoryComplexity ep.CpuComplexity = &cpuComplexity - ep.MemoryComplexity = &memoryComplexity ep.ExecutionMode = EpExecModeDefault cpuComplexity.ExecutionTime = EpExecTimeDefault - cpuComplexity.Method = EpMethodDefault - cpuComplexity.Workers = EpWorkersDefault - cpuComplexity.CpuAffinity = []int{} - cpuComplexity.CpuLoad = EpLoadDefault - - memoryComplexity.ExecutionTime = EpExecTimeDefault - memoryComplexity.Method = EpMethodDefault - memoryComplexity.Workers = EpWorkersDefault - memoryComplexity.BytesLoad = EpLoadDefault ep.NetworkComplexity.ForwardRequests = EpNwForwardRequests ep.NetworkComplexity.ResponsePayloadSize = EpNwResponseSizeDefault From 79641e7ea2bb8fe4ead0a00239f9eb54ca49fa70 Mon Sep 17 00:00:00 2001 From: Mohammad Reza Saleh Date: Tue, 21 Jun 2022 15:49:06 +0200 Subject: [PATCH 5/6] update service description file --- generator/input/new_description.json | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/generator/input/new_description.json b/generator/input/new_description.json index b217a587..47442838 100644 --- a/generator/input/new_description.json +++ b/generator/input/new_description.json @@ -49,20 +49,7 @@ "protocol": "http", "execution_mode": "sequential", "cpu_complexity": { - "execution_time": "1s", - "method": "fibonacci", - "workers": 2, - "cpu_affinity": [ - 0, - 2 - ], - "cpu_load": "10%" - }, - "memory_complexity": { - "execution_time": "1s", - "method": "swap", - "workers": 2, - "bytes_load": "10%" + "execution_time": 0.001 }, "network_complexity": { "forward_requests": "asynchronous", From 2febf895287cb3473bf3c9f08b9bbd8c822429d2 Mon Sep 17 00:00:00 2001 From: Mohammad Reza Saleh Date: Wed, 22 Jun 2022 05:13:09 +0200 Subject: [PATCH 6/6] fix the requested changes mostly in create_payload --- model/restful/utils/task.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/model/restful/utils/task.py b/model/restful/utils/task.py index 7643153d..ac8aa8eb 100644 --- a/model/restful/utils/task.py +++ b/model/restful/utils/task.py @@ -17,7 +17,6 @@ from flask import Blueprint, jsonify, request, current_app from aiohttp import ClientSession import asyncio -import subprocess from concurrent.futures import ThreadPoolExecutor, as_completed import requests import time @@ -224,11 +223,9 @@ def sync_execute_io_bounded_task(source_service, target_service, forward_headers def create_payload(payload_size): - - request_payload = subprocess.run(['cat /dev/urandom | tr -dc "[:alnum:]" | head -c${1:-%s}' % payload_size], capture_output=True, shell=True) letters = string.ascii_lowercase result_str = ''.join(random.choice(letters) for i in range(int(payload_size))) - return request_payload.stdout.decode("utf-8") + return result_str def create_response():