improved variation
This commit is contained in:
parent
d700800a78
commit
e47e058c3f
123
evaluation.ipynb
123
evaluation.ipynb
@ -30,13 +30,10 @@
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import ipaddress\n",
|
||||
"import threading\n",
|
||||
"from typing import Dict\n",
|
||||
"\n",
|
||||
"import runners\n",
|
||||
"from structure import Bridge, Interface, IpMethod\n",
|
||||
"from structure import RemotePortal, LocalPortal, SpeedTestServer\n",
|
||||
"from structure import StandardEnvironment, StandardTest, IperfResult\n",
|
||||
"from structure import StandardEnvironment, StandardTest, StandardIperfResult\n",
|
||||
"\n",
|
||||
"%load_ext dotenv\n",
|
||||
"%dotenv"
|
||||
@ -86,8 +83,8 @@
|
||||
" 'branch': os.getenv('TARGET_BRANCH'),\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"directionInbound: Dict[str, IperfResult] = {}\n",
|
||||
"directionOutbound: Dict[str, IperfResult] = {}\n",
|
||||
"directionInbound: Dict[str, StandardIperfResult] = {}\n",
|
||||
"directionOutbound: Dict[str, StandardIperfResult] = {}\n",
|
||||
"\n",
|
||||
"def run_and_save_test(env: StandardEnvironment, test: StandardTest):\n",
|
||||
" (directionInbound[test.name()], directionOutbound[test.name()]) = env.test(test)"
|
||||
@ -105,42 +102,6 @@
|
||||
"### Direct Server to Server Testing"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"st1 = SpeedTestServer()\n",
|
||||
"st2 = SpeedTestServer()\n",
|
||||
"\n",
|
||||
"top_level_bridge = Bridge(\n",
|
||||
" st1.get_interfaces()[0],\n",
|
||||
" st2.get_interfaces()[0],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" runner.build(top_level_bridge)\n",
|
||||
"\n",
|
||||
" st2.get_interfaces()[0].set_rate(1)\n",
|
||||
" st2.server()\n",
|
||||
" directionInbound['One1MBNotProxied'] = st1.client(st2)\n",
|
||||
" st1.server()\n",
|
||||
" directionOutbound['One1MBNotProxied'] = st2.client(st1)\n",
|
||||
"\n",
|
||||
" st2.get_interfaces()[0].set_rate(2)\n",
|
||||
" st2.server()\n",
|
||||
" directionInbound['One2MBNotProxied'] = st1.client(st2)\n",
|
||||
" st1.server()\n",
|
||||
" directionOutbound['One2MBNotProxied'] = st2.client(st1)\n",
|
||||
"finally:\n",
|
||||
" runner.teardown()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
@ -190,7 +151,8 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"with StandardEnvironment(3, runner, setup_params) as env:\n",
|
||||
" run_and_save_test(env, StandardTest([1,1,1]))\n"
|
||||
" run_and_save_test(env, StandardTest([1,1,1]))\n",
|
||||
" run_and_save_test(env, StandardTest([2,2,2]))\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -213,7 +175,8 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"with StandardEnvironment(4, runner, setup_params) as env:\n",
|
||||
" run_and_save_test(env, StandardTest([1,1,1,1]))"
|
||||
" run_and_save_test(env, StandardTest([1,1,1,1]))\n",
|
||||
" run_and_save_test(env, StandardTest([2,2,2,2]))\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -237,22 +200,43 @@
|
||||
"from itertools import cycle\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"\n",
|
||||
"def plot_iperf_results(data, title, events=None, filename=None, start_at_zero=True):\n",
|
||||
"def plot_iperf_results(\n",
|
||||
" series: Dict[str, StandardTest],\n",
|
||||
" title: str = None,\n",
|
||||
" direction = 'inbound',\n",
|
||||
" error_bars_x=False,\n",
|
||||
" error_bars_y=False,\n",
|
||||
" filename=None,\n",
|
||||
" start_at_zero=True,\n",
|
||||
"):\n",
|
||||
" series = {\n",
|
||||
" k: (directionInbound if direction == 'inbound' else directionOutbound)[v.name()] for (k, v) in series.items()\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" cycol = cycle('brgy')\n",
|
||||
"\n",
|
||||
" fig = plt.figure()\n",
|
||||
" axes = fig.add_axes([0,0,1,1])\n",
|
||||
"\n",
|
||||
" axes.set_title(title, pad=20.0 if events is not None else None)\n",
|
||||
" if title is not None:\n",
|
||||
" axes.set_title(title, pad=20.0 if True in [len(x.test.events) > 0 for x in series.values()] else None)\n",
|
||||
"\n",
|
||||
" axes.set_xlabel('Time (s)')\n",
|
||||
" axes.set_ylabel('Throughput (Mbps)')\n",
|
||||
"\n",
|
||||
" for k, v in data.items():\n",
|
||||
" intervals = [x['sum'] for x in v['intervals'] if not x['sum']['omitted']]\n",
|
||||
" for k, v in series.items():\n",
|
||||
" data = v.summarise()\n",
|
||||
"\n",
|
||||
" x_axis = [((x['start'] + x['end'])/2) for x in intervals]\n",
|
||||
" y_axis = [x['bits_per_second']/1e6 for x in intervals]\n",
|
||||
" axes.plot(x_axis, y_axis, next(cycol), label=k)\n",
|
||||
" axes.errorbar(\n",
|
||||
" data.keys(),\n",
|
||||
" [x/1e6 for x in data.values()],\n",
|
||||
" xerr=([x[0] for x in v.time_range().values()], [x[1] for x in v.time_range().values()]) if error_bars_x else None,\n",
|
||||
" yerr=[x*1.5/1e6 for x in v.standard_deviation().values()] if error_bars_y else None,\n",
|
||||
" capsize=3,\n",
|
||||
" ecolor='grey',\n",
|
||||
" color=next(cycol),\n",
|
||||
" label=k,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" legend = axes.legend()\n",
|
||||
"\n",
|
||||
@ -260,7 +244,7 @@
|
||||
" axes.set_ylim(bottom=0)\n",
|
||||
" axes.set_xlim(left=0)\n",
|
||||
"\n",
|
||||
" if events is not None:\n",
|
||||
" if False:\n",
|
||||
" for k, v in events.items():\n",
|
||||
" axes.axvline(k, linestyle='--', color='grey')\n",
|
||||
" axes.annotate(v, (k, 1.02), xycoords=axes.get_xaxis_transform(), ha='center')\n",
|
||||
@ -293,11 +277,13 @@
|
||||
"source": [
|
||||
"plot_iperf_results(\n",
|
||||
" {\n",
|
||||
" '1x2MB Connection (not proxied)': directionInbound['One2MBNotProxied'],\n",
|
||||
" '2x1MB Connections (proxied)': directionInbound['Two1MBProxied'],\n",
|
||||
" '1x1MB Connection (not proxied)': directionInbound['One1MBNotProxied'],\n",
|
||||
" '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n",
|
||||
" '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
|
||||
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
|
||||
" },\n",
|
||||
" 'Two Equal 1MB Connections',\n",
|
||||
" 'Scaling of Equal Connections',\n",
|
||||
" error_bars_x=True,\n",
|
||||
" error_bars_y=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
@ -313,10 +299,13 @@
|
||||
"source": [
|
||||
"plot_iperf_results(\n",
|
||||
" {\n",
|
||||
" '2x2MB Connections (proxied)': directionInbound['Two2MBProxied'],\n",
|
||||
" '1x2MB Connection (not proxied)': directionInbound['One2MBNotProxied'],\n",
|
||||
" '4x2MB Connections (proxied)': StandardTest([2,2,2,2]),\n",
|
||||
" '3x2MB Connections (proxied)': StandardTest([2,2,2]),\n",
|
||||
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
|
||||
" },\n",
|
||||
" 'Two Equal 2MB Connections',\n",
|
||||
" 'Scaling of Equal Connections',\n",
|
||||
" error_bars_x=True,\n",
|
||||
" error_bars_y=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
@ -332,11 +321,9 @@
|
||||
"source": [
|
||||
"plot_iperf_results(\n",
|
||||
" {\n",
|
||||
" '4x1MBps Connections (proxied)': directionInbound['Four1MBProxied'],\n",
|
||||
" '3x1MBps Connections (proxied)': directionInbound['Three1MBProxied'],\n",
|
||||
" '2x1MBps Connections (proxied)': directionInbound['Two1MBProxied'],\n",
|
||||
" 'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n",
|
||||
" },\n",
|
||||
" 'More Equal Connections',\n",
|
||||
" error_bars_y=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
@ -349,15 +336,7 @@
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"plot_iperf_results(\n",
|
||||
" {\n",
|
||||
" '1x2MBps + 1xYMBps Connections (proxied)': directionInbound['One2MBOneYMBProxiedSlow15Return30'],\n",
|
||||
" },\n",
|
||||
" 'Network Slow',\n",
|
||||
" events={0: 'Y=2', 15: 'Y=1', 30: 'Y=2'}\n",
|
||||
")\n"
|
||||
]
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
@ -3,4 +3,4 @@ from .structure import Node
|
||||
from .structure import IpMethod, Interface, Bridge
|
||||
|
||||
from .structure import SpeedTestServer, LocalPortal, RemotePortal
|
||||
from .structure import StandardEnvironment, StandardTest, IperfResult
|
||||
from .structure import StandardEnvironment, StandardTest, StandardIperfResult
|
||||
|
@ -348,7 +348,7 @@ class StandardTest:
|
||||
rates: List[int],
|
||||
events: Dict[float, Tuple[int, int]] = None,
|
||||
duration: int = 10,
|
||||
variation_target: float = 0.4,
|
||||
variation_target: float = 0.3,
|
||||
max_failures: int = 3,
|
||||
max_attempts: int = 60,
|
||||
):
|
||||
@ -367,10 +367,11 @@ class StandardTest:
|
||||
return ''.join(name_builder)
|
||||
|
||||
|
||||
class IperfResult:
|
||||
def __init__(self, iperf: str, interval_size=1.0, duration=30):
|
||||
class StandardIperfResult:
|
||||
def __init__(self, test: StandardTest, iperf: str, interval_size=1.0):
|
||||
self.test = test
|
||||
|
||||
self.interval_size = interval_size
|
||||
self.duration = duration
|
||||
|
||||
# list containing an exact time and a value
|
||||
self.data: List[Tuple[float, float]] = []
|
||||
@ -392,11 +393,10 @@ class IperfResult:
|
||||
self.data.append((time, result))
|
||||
|
||||
def bins(self) -> List[List[Tuple[float, float]]]:
|
||||
# Binning phase
|
||||
bins: List[List[Tuple[float, float]]] = [[] for _ in np.arange(0, self.duration, self.interval_size)]
|
||||
bins: List[List[Tuple[float, float]]] = [[] for _ in np.arange(0, self.test.duration, self.interval_size)]
|
||||
|
||||
for time, result in self.data:
|
||||
index = int((time - self.interval_size / 2) / self.interval_size)
|
||||
index = int(np.round((time - self.interval_size / 2) / self.interval_size))
|
||||
bins[index].append((time, result))
|
||||
|
||||
return bins
|
||||
@ -404,13 +404,13 @@ class IperfResult:
|
||||
def summarise(self) -> Dict[float, float]:
|
||||
bins = self.bins()
|
||||
means = [np.mean(x, axis=0)[1] for x in bins]
|
||||
times = [i + self.interval_size / 2 for i in np.arange(0, self.duration, self.interval_size)]
|
||||
times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)]
|
||||
return dict(zip(times, means))
|
||||
|
||||
def standard_deviation(self) -> Dict[float, float]:
|
||||
bins = self.bins()
|
||||
stds = [np.std(x, axis=0)[1] for x in bins]
|
||||
times = [i + self.interval_size / 2 for i in np.arange(0, self.duration, self.interval_size)]
|
||||
times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)]
|
||||
return dict(zip(times, stds))
|
||||
|
||||
def coefficient_variance(self) -> Dict[float, float]:
|
||||
@ -421,8 +421,8 @@ class IperfResult:
|
||||
|
||||
def time_range(self) -> Dict[float, Tuple[float, float]]:
|
||||
bins = self.bins()
|
||||
times = [i + self.interval_size / 2 for i in np.arange(0, self.duration, self.interval_size)]
|
||||
ranges = [(np.min(x, axis=0)[0] - time, np.max(x, axis=0)[0] - time) for (x, time) in zip(bins, times)]
|
||||
times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)]
|
||||
ranges = [(-np.min(x, axis=0)[0] + time, np.max(x, axis=0)[0] - time) for (x, time) in zip(bins, times)]
|
||||
return dict(zip(times, ranges))
|
||||
|
||||
|
||||
@ -462,7 +462,7 @@ class StandardEnvironment:
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self._runner.teardown()
|
||||
|
||||
def test(self, test: StandardTest) -> Tuple[IperfResult, IperfResult]:
|
||||
def test(self, test: StandardTest) -> Tuple[StandardIperfResult, StandardIperfResult]:
|
||||
if len(test.rates) != self._interfaces:
|
||||
raise RuntimeError('mismatched number of interfaces')
|
||||
|
||||
@ -471,7 +471,7 @@ class StandardEnvironment:
|
||||
|
||||
results = []
|
||||
for server, client in [(self.cl, self.st), (self.st, self.cl)]:
|
||||
result: Optional[IperfResult] = None
|
||||
result: Optional[StandardIperfResult] = None
|
||||
|
||||
for i in range(test.max_attempts):
|
||||
if i > 2 and max(result.coefficient_variance().values()) < test.variation_target:
|
||||
@ -482,11 +482,14 @@ class StandardEnvironment:
|
||||
server.server()
|
||||
|
||||
for t, (iface, rate) in test.events.items():
|
||||
threading.Timer(5 + t, lambda: self.lp.get_interfaces()[iface].set_rate(rate))
|
||||
threading.Timer(
|
||||
5 + t,
|
||||
(lambda s: lambda: s.lp.get_interfaces()[iface].set_rate(rate))(self),
|
||||
)
|
||||
|
||||
iperf = client.client(server, time=test.duration)
|
||||
if result is None:
|
||||
result = IperfResult(iperf)
|
||||
result = StandardIperfResult(test, iperf)
|
||||
else:
|
||||
result.add_results(iperf)
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user