JakeHillion/dissertation-3-evaluation
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

improved variation

Browse Source
This commit is contained in:
Jake Hillion 2020-11-14 10:50:20 +00:00
parent d700800a78
commit e47e058c3f
3 changed files with 70 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
evaluation.ipynb
View File

@ -30,13 +30,10 @@
"source": [
"import os\n",
"import ipaddress\n",
"import threading\n",
"from typing import Dict\n",
"\n",
"import runners\n",
"from structure import Bridge, Interface, IpMethod\n",
"from structure import RemotePortal, LocalPortal, SpeedTestServer\n",
"from structure import StandardEnvironment, StandardTest, IperfResult\n",
"from structure import StandardEnvironment, StandardTest, StandardIperfResult\n",
"\n",
"%load_ext dotenv\n",
"%dotenv"
@ -86,8 +83,8 @@
" 'branch': os.getenv('TARGET_BRANCH'),\n",
"}\n",
"\n",
"directionInbound: Dict[str, IperfResult] = {}\n",
"directionOutbound: Dict[str, IperfResult] = {}\n",
"directionInbound: Dict[str, StandardIperfResult] = {}\n",
"directionOutbound: Dict[str, StandardIperfResult] = {}\n",
"\n",
"def run_and_save_test(env: StandardEnvironment, test: StandardTest):\n",
" (directionInbound[test.name()], directionOutbound[test.name()]) = env.test(test)"
@ -105,42 +102,6 @@
"### Direct Server to Server Testing"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"st1 = SpeedTestServer()\n",
"st2 = SpeedTestServer()\n",
"\n",
"top_level_bridge = Bridge(\n",
" st1.get_interfaces()[0],\n",
" st2.get_interfaces()[0],\n",
")\n",
"\n",
"try:\n",
" runner.build(top_level_bridge)\n",
"\n",
" st2.get_interfaces()[0].set_rate(1)\n",
" st2.server()\n",
" directionInbound['One1MBNotProxied'] = st1.client(st2)\n",
" st1.server()\n",
" directionOutbound['One1MBNotProxied'] = st2.client(st1)\n",
"\n",
" st2.get_interfaces()[0].set_rate(2)\n",
" st2.server()\n",
" directionInbound['One2MBNotProxied'] = st1.client(st2)\n",
" st1.server()\n",
" directionOutbound['One2MBNotProxied'] = st2.client(st1)\n",
"finally:\n",
" runner.teardown()"
]
},
{
"cell_type": "markdown",
"metadata": {
@ -190,7 +151,8 @@
"outputs": [],
"source": [
"with StandardEnvironment(3, runner, setup_params) as env:\n",
" run_and_save_test(env, StandardTest([1,1,1]))\n"
" run_and_save_test(env, StandardTest([1,1,1]))\n",
" run_and_save_test(env, StandardTest([2,2,2]))\n"
]
},
{
@ -213,7 +175,8 @@
"outputs": [],
"source": [
"with StandardEnvironment(4, runner, setup_params) as env:\n",
" run_and_save_test(env, StandardTest([1,1,1,1]))"
" run_and_save_test(env, StandardTest([1,1,1,1]))\n",
" run_and_save_test(env, StandardTest([2,2,2,2]))\n"
]
},
{
@ -237,22 +200,43 @@
"from itertools import cycle\n",
"import matplotlib.pyplot as plt\n",
"\n",
"def plot_iperf_results(data, title, events=None, filename=None, start_at_zero=True):\n",
"def plot_iperf_results(\n",
" series: Dict[str, StandardTest],\n",
" title: str = None,\n",
" direction = 'inbound',\n",
" error_bars_x=False,\n",
" error_bars_y=False,\n",
" filename=None,\n",
" start_at_zero=True,\n",
"):\n",
" series = {\n",
" k: (directionInbound if direction == 'inbound' else directionOutbound)[v.name()] for (k, v) in series.items()\n",
" }\n",
"\n",
" cycol = cycle('brgy')\n",
"\n",
" fig = plt.figure()\n",
" axes = fig.add_axes([0,0,1,1])\n",
"\n",
" axes.set_title(title, pad=20.0 if events is not None else None)\n",
" if title is not None:\n",
" axes.set_title(title, pad=20.0 if True in [len(x.test.events) > 0 for x in series.values()] else None)\n",
"\n",
" axes.set_xlabel('Time (s)')\n",
" axes.set_ylabel('Throughput (Mbps)')\n",
"\n",
" for k, v in data.items():\n",
" intervals = [x['sum'] for x in v['intervals'] if not x['sum']['omitted']]\n",
" for k, v in series.items():\n",
" data = v.summarise()\n",
"\n",
" x_axis = [((x['start'] + x['end'])/2) for x in intervals]\n",
" y_axis = [x['bits_per_second']/1e6 for x in intervals]\n",
" axes.plot(x_axis, y_axis, next(cycol), label=k)\n",
" axes.errorbar(\n",
" data.keys(),\n",
" [x/1e6 for x in data.values()],\n",
" xerr=([x[0] for x in v.time_range().values()], [x[1] for x in v.time_range().values()]) if error_bars_x else None,\n",
" yerr=[x*1.5/1e6 for x in v.standard_deviation().values()] if error_bars_y else None,\n",
" capsize=3,\n",
" ecolor='grey',\n",
" color=next(cycol),\n",
" label=k,\n",
" )\n",
"\n",
" legend = axes.legend()\n",
"\n",
@ -260,7 +244,7 @@
" axes.set_ylim(bottom=0)\n",
" axes.set_xlim(left=0)\n",
"\n",
" if events is not None:\n",
" if False:\n",
" for k, v in events.items():\n",
" axes.axvline(k, linestyle='--', color='grey')\n",
" axes.annotate(v, (k, 1.02), xycoords=axes.get_xaxis_transform(), ha='center')\n",
@ -293,11 +277,13 @@
"source": [
"plot_iperf_results(\n",
" {\n",
" '1x2MB Connection (not proxied)': directionInbound['One2MBNotProxied'],\n",
" '2x1MB Connections (proxied)': directionInbound['Two1MBProxied'],\n",
" '1x1MB Connection (not proxied)': directionInbound['One1MBNotProxied'],\n",
" '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n",
" '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" },\n",
" 'Two Equal 1MB Connections',\n",
" 'Scaling of Equal Connections',\n",
" error_bars_x=True,\n",
" error_bars_y=True,\n",
")"
]
},
@ -313,10 +299,13 @@
"source": [
"plot_iperf_results(\n",
" {\n",
" '2x2MB Connections (proxied)': directionInbound['Two2MBProxied'],\n",
" '1x2MB Connection (not proxied)': directionInbound['One2MBNotProxied'],\n",
" '4x2MB Connections (proxied)': StandardTest([2,2,2,2]),\n",
" '3x2MB Connections (proxied)': StandardTest([2,2,2]),\n",
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
" },\n",
" 'Two Equal 2MB Connections',\n",
" 'Scaling of Equal Connections',\n",
" error_bars_x=True,\n",
" error_bars_y=True,\n",
")"
]
},
@ -332,11 +321,9 @@
"source": [
"plot_iperf_results(\n",
" {\n",
" '4x1MBps Connections (proxied)': directionInbound['Four1MBProxied'],\n",
" '3x1MBps Connections (proxied)': directionInbound['Three1MBProxied'],\n",
" '2x1MBps Connections (proxied)': directionInbound['Two1MBProxied'],\n",
" 'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n",
" },\n",
" 'More Equal Connections',\n",
" error_bars_y=True,\n",
")"
]
},
@ -349,15 +336,7 @@
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '1x2MBps + 1xYMBps Connections (proxied)': directionInbound['One2MBOneYMBProxiedSlow15Return30'],\n",
" },\n",
" 'Network Slow',\n",
" events={0: 'Y=2', 15: 'Y=1', 30: 'Y=2'}\n",
")\n"
]
"source": []
}
],
"metadata": {

2
structure/__init__.py
View File

@ -3,4 +3,4 @@ from .structure import Node
from .structure import IpMethod, Interface, Bridge
from .structure import SpeedTestServer, LocalPortal, RemotePortal
from .structure import StandardEnvironment, StandardTest, IperfResult
from .structure import StandardEnvironment, StandardTest, StandardIperfResult

33
structure/structure.py
View File

@ -348,7 +348,7 @@ class StandardTest:
rates: List[int],
events: Dict[float, Tuple[int, int]] = None,
duration: int = 10,
variation_target: float = 0.4,
variation_target: float = 0.3,
max_failures: int = 3,
max_attempts: int = 60,
):
@ -367,10 +367,11 @@ class StandardTest:
return ''.join(name_builder)
class IperfResult:
def __init__(self, iperf: str, interval_size=1.0, duration=30):
class StandardIperfResult:
def __init__(self, test: StandardTest, iperf: str, interval_size=1.0):
self.test = test
self.interval_size = interval_size
self.duration = duration
# list containing an exact time and a value
self.data: List[Tuple[float, float]] = []
@ -392,11 +393,10 @@ class IperfResult:
self.data.append((time, result))
def bins(self) -> List[List[Tuple[float, float]]]:
# Binning phase
bins: List[List[Tuple[float, float]]] = [[] for _ in np.arange(0, self.duration, self.interval_size)]
bins: List[List[Tuple[float, float]]] = [[] for _ in np.arange(0, self.test.duration, self.interval_size)]
for time, result in self.data:
index = int((time - self.interval_size / 2) / self.interval_size)
index = int(np.round((time - self.interval_size / 2) / self.interval_size))
bins[index].append((time, result))
return bins
@ -404,13 +404,13 @@ class IperfResult:
def summarise(self) -> Dict[float, float]:
bins = self.bins()
means = [np.mean(x, axis=0)[1] for x in bins]
times = [i + self.interval_size / 2 for i in np.arange(0, self.duration, self.interval_size)]
times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)]
return dict(zip(times, means))
def standard_deviation(self) -> Dict[float, float]:
bins = self.bins()
stds = [np.std(x, axis=0)[1] for x in bins]
times = [i + self.interval_size / 2 for i in np.arange(0, self.duration, self.interval_size)]
times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)]
return dict(zip(times, stds))
def coefficient_variance(self) -> Dict[float, float]:
@ -421,8 +421,8 @@ class IperfResult:
def time_range(self) -> Dict[float, Tuple[float, float]]:
bins = self.bins()
times = [i + self.interval_size / 2 for i in np.arange(0, self.duration, self.interval_size)]
ranges = [(np.min(x, axis=0)[0] - time, np.max(x, axis=0)[0] - time) for (x, time) in zip(bins, times)]
times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)]
ranges = [(-np.min(x, axis=0)[0] + time, np.max(x, axis=0)[0] - time) for (x, time) in zip(bins, times)]
return dict(zip(times, ranges))
@ -462,7 +462,7 @@ class StandardEnvironment:
def __exit__(self, exc_type, exc_val, exc_tb):
self._runner.teardown()
def test(self, test: StandardTest) -> Tuple[IperfResult, IperfResult]:
def test(self, test: StandardTest) -> Tuple[StandardIperfResult, StandardIperfResult]:
if len(test.rates) != self._interfaces:
raise RuntimeError('mismatched number of interfaces')
@ -471,7 +471,7 @@ class StandardEnvironment:
results = []
for server, client in [(self.cl, self.st), (self.st, self.cl)]:
result: Optional[IperfResult] = None
result: Optional[StandardIperfResult] = None
for i in range(test.max_attempts):
if i > 2 and max(result.coefficient_variance().values()) < test.variation_target:
@ -482,11 +482,14 @@ class StandardEnvironment:
server.server()
for t, (iface, rate) in test.events.items():
threading.Timer(5 + t, lambda: self.lp.get_interfaces()[iface].set_rate(rate))
threading.Timer(
5 + t,
(lambda s: lambda: s.lp.get_interfaces()[iface].set_rate(rate))(self),
)
iperf = client.client(server, time=test.duration)
if result is None:
result = IperfResult(iperf)
result = StandardIperfResult(test, iperf)
else:
result.add_results(iperf)