JakeHillion/dissertation-3-evaluation
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

added standard structures

Browse Source
This commit is contained in:
Jake Hillion 2020-11-13 16:37:06 +00:00
parent e594f6f06e
commit b36556bb71
3 changed files with 262 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
evaluation.ipynb
View File

@ -6,7 +6,7 @@
"source": [
"# Project Evaluation\n",
"\n",
"This file interfaces with a Proxmox server to automatically generate VM structures and graphs for testing the\n",
"This file interfaces with a Proxmox server to automatically generate VM structures and plots for testing the\n",
"success criteria of my project."
]
},
@ -31,10 +31,12 @@
"import os\n",
"import ipaddress\n",
"import threading\n",
"from typing import Dict\n",
"\n",
"import runners\n",
"from structure import Bridge, Interface, IpMethod\n",
"from structure import RemotePortal, LocalPortal, SpeedTestServer\n",
"from structure import StandardEnvironment, StandardTest, IperfResult\n",
"\n",
"%load_ext dotenv\n",
"%dotenv"
@ -84,8 +86,23 @@
" 'branch': os.getenv('TARGET_BRANCH'),\n",
"}\n",
"\n",
"directionInbound = {}\n",
"directionOutbound = {}"
"directionInbound: Dict[str, IperfResult] = {}\n",
"directionOutbound: Dict[str, IperfResult] = {}\n",
"\n",
"def run_and_save_test(env: StandardEnvironment, test: StandardTest):\n",
" (directionInbound[test.name()], directionOutbound[test.name()]) = env.test(test)"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
},
"source": [
"### Direct Server to Server Testing"
]
},
{
@ -124,6 +141,48 @@
" runner.teardown()"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
},
"source": [
"### Local Portal with 2 Interfaces"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"is_executing": true,
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"with StandardEnvironment(2, runner, setup_params) as env:\n",
" run_and_save_test(env, StandardTest([1,1]))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"is_executing": true,
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"from pprint import pprint\n",
"pprint(directionInbound[StandardTest([1,1]).name()].standard_deviation())"
]
},
{
"cell_type": "code",
"execution_count": null,
@ -196,6 +255,15 @@
" runner.teardown()"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false
},
"source": [
"### Local Portal with 3 Interfaces"
]
},
{
"cell_type": "code",
"execution_count": null,
@ -241,6 +309,15 @@
" runner.teardown()"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false
},
"source": [
"### Local Portal with 4 Interfaces"
]
},
{
"cell_type": "code",
"execution_count": null,
@ -428,27 +505,8 @@
" },\n",
" 'Network Slow',\n",
" events={0: 'Y=2', 15: 'Y=1', 30: 'Y=2'}\n",
")"
")\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"pycharm": {
"name": "#%% md\n"
}
},
"source": [
"## Criteria\n",
"This section automatically verifies some criteria with assertions."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {

1
structure/__init__.py
View File

@ -3,3 +3,4 @@ from .structure import Node
from .structure import IpMethod, Interface, Bridge
from .structure import SpeedTestServer, LocalPortal, RemotePortal
from .structure import StandardEnvironment, StandardTest, IperfResult

189
structure/structure.py
View File

@ -1,9 +1,12 @@
import ipaddress
import json
import textwrap
import threading
from enum import Enum
import random
from typing import List, Optional, Union, Dict
import numpy as np
from typing import List, Optional, Union, Dict, Tuple
class IpMethod(Enum):
@ -160,8 +163,7 @@ class SpeedTestServer(Node):
target = target.get_interfaces()[0].get_address()
command = 'iperf3 -c {target} -t {time} -O 5 -J'.format(target=target, time=time)
out = self.ssh(command, error_stdout=True, error_stderr=True, return_stdout=True)
return json.loads(out)
return self.ssh(command, error_stdout=True, error_stderr=True, return_stdout=True)
class RemotePortal(Node):
@ -210,7 +212,10 @@ class RemotePortal(Node):
(nohup sudo ./mpbl3p > mpbl3p.log 2>&1 & echo $! > mpbl3p.pid)
sleep 1
sleep 10
ps $(cat mpbl3p.pid) || cat mpbl3p.log
sudo ip addr add 172.19.152.2/31 dev nc0
sudo ip link set up nc0
@ -223,7 +228,7 @@ class RemotePortal(Node):
sudo ip route add table 10 to {local_host} via 172.19.152.3 dev nc0
sudo ip rule add to {local_host} table 10 priority 10
ps $(cat mpbl3p.pid)
ps $(cat mpbl3p.pid) || cat mpbl3p.log
''').format(
local_host=self.get_interfaces()[0].get_address(),
**self.setup_params,
@ -284,7 +289,7 @@ class LocalPortal(Node):
''')
policy_routing = '\n\n'.join([policy_routing_string.format(
table_number=i+10,
table_number=i + 10,
device='eth{}'.format(i),
network=iface.get_bridge().get_network(),
local_address=iface.get_address(),
@ -312,7 +317,10 @@ class LocalPortal(Node):
(nohup sudo ./mpbl3p > mpbl3p.log 2>&1 & echo $! > mpbl3p.pid)
sleep 1
sleep 10
ps $(cat mpbl3p.pid) || cat mpbl3p.log
sudo ip addr add 172.19.152.3/31 dev nc0
sudo ip link set up nc0
@ -324,11 +332,174 @@ class LocalPortal(Node):
sudo ip route add to {remote_host} dev {local_interface} table 19
sudo ip rule add to {remote_host} table 19 priority 19
ps $(cat mpbl3p.pid)
ps $(cat mpbl3p.pid) || cat mpbl3p.log
''').format(
**self.setup_params,
peers=peers,
policy_routing=policy_routing,
remote_host=self.remote_portal.get_interfaces()[0].get_address(),
local_interface='eth{}'.format(len(self.get_interfaces())-2),
local_interface='eth{}'.format(len(self.get_interfaces()) - 2),
)
class StandardTest:
def __init__(
self,
rates: List[int],
events: Dict[float, Tuple[int, int]] = None,
duration: int = 30,
variation_target: float = 0.4,
max_failures: int = 3,
max_attempts: int = 30,
):
self.rates = rates
self.events = events if events is not None else dict()
self.duration = duration
self.variation_target = variation_target
self.max_failures = max_failures
self.max_attempts = max_attempts
def name(self) -> str:
name_builder = ['R{}-{}'.format(*y) for y in enumerate(self.rates)]
name_builder += ['E{}R{}-{}'.format(x, *y) for (x, y) in self.events.items()]
name_builder.append('D{}'.format(self.duration))
return ''.join(name_builder)
class IperfResult:
def __init__(self, iperf: str, interval_size=1.0, duration=30):
self.interval_size = interval_size
self.duration = duration
# list containing an exact time and a value
self.data: List[Tuple[float, float]] = []
self.add_results(iperf)
def add_results(self, iperf: str):
data = json.loads(iperf)
# grab the sum data of all non omitted intervals, excluding any that are smaller than expected
intervals = [
x['sum'] for x in data['intervals'] if
(not x['sum']['omitted']) and (x['sum']['end'] - x['sum']['start'] > self.interval_size / 2)
]
for (time, result) in zip(
[((x['start'] + x['end']) / 2) for x in intervals],
[x['bits_per_second'] for x in intervals],
):
self.data.append((time, result))
def bins(self) -> List[List[Tuple[float, float]]]:
# Binning phase
bins: List[List[Tuple[float, float]]] = [[] for _ in np.arange(0, self.duration, self.interval_size)]
for time, result in self.data:
index = int((time - self.interval_size / 2) / self.interval_size)
bins[index].append((time, result))
return bins
def summarise(self) -> Dict[float, float]:
bins = self.bins()
means = [np.mean(x, axis=0)[1] for x in bins]
times = [i + self.interval_size / 2 for i in np.arange(0, self.duration, self.interval_size)]
return dict(zip(times, means))
def standard_deviation(self) -> Dict[float, float]:
bins = self.bins()
stds = [np.std(x, axis=0)[1] for x in bins]
times = [i + self.interval_size / 2 for i in np.arange(0, self.duration, self.interval_size)]
return dict(zip(times, stds))
def coefficient_variance(self) -> Dict[float, float]:
stds = self.standard_deviation()
means = self.summarise()
return {k: stds[k] / means[k] for k in stds.keys()}
def time_range(self) -> Dict[float, Tuple[float, float]]:
bins = self.bins()
times = [i + self.interval_size / 2 for i in np.arange(0, self.duration, self.interval_size)]
ranges = [(np.min(x, axis=0)[0] - time, np.max(x, axis=0)[0] - time) for (x, time) in zip(bins, times)]
return dict(zip(times, ranges))
class StandardEnvironment:
def __init__(self, interfaces: int, runner, setup_params: dict):
self._interfaces = interfaces
self._runner = runner
self.rp = RemotePortal([Interface(IpMethod.Auto4)], setup_params=setup_params)
self.st = SpeedTestServer()
self.cl = SpeedTestServer(clone_interface=self.rp.get_interfaces()[0])
self.lp = LocalPortal(
[Interface(IpMethod.Auto4) for _ in range(interfaces)],
self.cl,
setup_params=setup_params,
)
self.rp.set_local_portal(self.lp)
self.lp.set_remote_portal(self.rp)
self.top_level_bridge = Bridge(
self.st.get_interfaces()[0],
self.rp.get_interfaces()[0],
*self.lp.get_interfaces()[0:interfaces],
)
def __enter__(self):
try:
self._runner.build(self.top_level_bridge)
except Exception as e:
self._runner.teardown()
raise e
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self._runner.teardown()
def test(self, test: StandardTest) -> Tuple[IperfResult, IperfResult]:
if len(test.rates) != self._interfaces:
raise RuntimeError('mismatched number of interfaces')
for i, r in enumerate(test.rates):
self.lp.get_interfaces()[i].set_rate(r)
results = []
for server, client in [(self.cl, self.st), (self.st, self.cl)]:
result: Optional[IperfResult] = None
for i in range(test.max_attempts):
if i > 2 and max(result.coefficient_variance().values()) < test.variation_target:
break
for j in range(test.max_failures):
try:
server.server()
for t, (iface, rate) in test.events.items():
threading.Timer(5 + t, lambda: self.lp.get_interfaces()[iface].set_rate(rate))
iperf = client.client(server, time=test.duration)
if result is None:
result = IperfResult(iperf)
else:
result.add_results(iperf)
break
except Exception as e:
print('failed with {}'.format(e))
if j == test.max_failures - 1:
raise e
if max(result.coefficient_variance().values()) > test.variation_target:
raise RuntimeError('too many attempts')
results.append(result)
# Return a tuple of (inbound, outbound)
return results[0], results[1]