improved variation
This commit is contained in:
parent
d700800a78
commit
e47e058c3f
123
evaluation.ipynb
123
evaluation.ipynb
@ -30,13 +30,10 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"import os\n",
|
"import os\n",
|
||||||
"import ipaddress\n",
|
"import ipaddress\n",
|
||||||
"import threading\n",
|
|
||||||
"from typing import Dict\n",
|
"from typing import Dict\n",
|
||||||
"\n",
|
"\n",
|
||||||
"import runners\n",
|
"import runners\n",
|
||||||
"from structure import Bridge, Interface, IpMethod\n",
|
"from structure import StandardEnvironment, StandardTest, StandardIperfResult\n",
|
||||||
"from structure import RemotePortal, LocalPortal, SpeedTestServer\n",
|
|
||||||
"from structure import StandardEnvironment, StandardTest, IperfResult\n",
|
|
||||||
"\n",
|
"\n",
|
||||||
"%load_ext dotenv\n",
|
"%load_ext dotenv\n",
|
||||||
"%dotenv"
|
"%dotenv"
|
||||||
@ -86,8 +83,8 @@
|
|||||||
" 'branch': os.getenv('TARGET_BRANCH'),\n",
|
" 'branch': os.getenv('TARGET_BRANCH'),\n",
|
||||||
"}\n",
|
"}\n",
|
||||||
"\n",
|
"\n",
|
||||||
"directionInbound: Dict[str, IperfResult] = {}\n",
|
"directionInbound: Dict[str, StandardIperfResult] = {}\n",
|
||||||
"directionOutbound: Dict[str, IperfResult] = {}\n",
|
"directionOutbound: Dict[str, StandardIperfResult] = {}\n",
|
||||||
"\n",
|
"\n",
|
||||||
"def run_and_save_test(env: StandardEnvironment, test: StandardTest):\n",
|
"def run_and_save_test(env: StandardEnvironment, test: StandardTest):\n",
|
||||||
" (directionInbound[test.name()], directionOutbound[test.name()]) = env.test(test)"
|
" (directionInbound[test.name()], directionOutbound[test.name()]) = env.test(test)"
|
||||||
@ -105,42 +102,6 @@
|
|||||||
"### Direct Server to Server Testing"
|
"### Direct Server to Server Testing"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"pycharm": {
|
|
||||||
"name": "#%%\n"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"st1 = SpeedTestServer()\n",
|
|
||||||
"st2 = SpeedTestServer()\n",
|
|
||||||
"\n",
|
|
||||||
"top_level_bridge = Bridge(\n",
|
|
||||||
" st1.get_interfaces()[0],\n",
|
|
||||||
" st2.get_interfaces()[0],\n",
|
|
||||||
")\n",
|
|
||||||
"\n",
|
|
||||||
"try:\n",
|
|
||||||
" runner.build(top_level_bridge)\n",
|
|
||||||
"\n",
|
|
||||||
" st2.get_interfaces()[0].set_rate(1)\n",
|
|
||||||
" st2.server()\n",
|
|
||||||
" directionInbound['One1MBNotProxied'] = st1.client(st2)\n",
|
|
||||||
" st1.server()\n",
|
|
||||||
" directionOutbound['One1MBNotProxied'] = st2.client(st1)\n",
|
|
||||||
"\n",
|
|
||||||
" st2.get_interfaces()[0].set_rate(2)\n",
|
|
||||||
" st2.server()\n",
|
|
||||||
" directionInbound['One2MBNotProxied'] = st1.client(st2)\n",
|
|
||||||
" st1.server()\n",
|
|
||||||
" directionOutbound['One2MBNotProxied'] = st2.client(st1)\n",
|
|
||||||
"finally:\n",
|
|
||||||
" runner.teardown()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {
|
"metadata": {
|
||||||
@ -190,7 +151,8 @@
|
|||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"with StandardEnvironment(3, runner, setup_params) as env:\n",
|
"with StandardEnvironment(3, runner, setup_params) as env:\n",
|
||||||
" run_and_save_test(env, StandardTest([1,1,1]))\n"
|
" run_and_save_test(env, StandardTest([1,1,1]))\n",
|
||||||
|
" run_and_save_test(env, StandardTest([2,2,2]))\n"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -213,7 +175,8 @@
|
|||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"with StandardEnvironment(4, runner, setup_params) as env:\n",
|
"with StandardEnvironment(4, runner, setup_params) as env:\n",
|
||||||
" run_and_save_test(env, StandardTest([1,1,1,1]))"
|
" run_and_save_test(env, StandardTest([1,1,1,1]))\n",
|
||||||
|
" run_and_save_test(env, StandardTest([2,2,2,2]))\n"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -237,22 +200,43 @@
|
|||||||
"from itertools import cycle\n",
|
"from itertools import cycle\n",
|
||||||
"import matplotlib.pyplot as plt\n",
|
"import matplotlib.pyplot as plt\n",
|
||||||
"\n",
|
"\n",
|
||||||
"def plot_iperf_results(data, title, events=None, filename=None, start_at_zero=True):\n",
|
"def plot_iperf_results(\n",
|
||||||
|
" series: Dict[str, StandardTest],\n",
|
||||||
|
" title: str = None,\n",
|
||||||
|
" direction = 'inbound',\n",
|
||||||
|
" error_bars_x=False,\n",
|
||||||
|
" error_bars_y=False,\n",
|
||||||
|
" filename=None,\n",
|
||||||
|
" start_at_zero=True,\n",
|
||||||
|
"):\n",
|
||||||
|
" series = {\n",
|
||||||
|
" k: (directionInbound if direction == 'inbound' else directionOutbound)[v.name()] for (k, v) in series.items()\n",
|
||||||
|
" }\n",
|
||||||
|
"\n",
|
||||||
" cycol = cycle('brgy')\n",
|
" cycol = cycle('brgy')\n",
|
||||||
"\n",
|
"\n",
|
||||||
" fig = plt.figure()\n",
|
" fig = plt.figure()\n",
|
||||||
" axes = fig.add_axes([0,0,1,1])\n",
|
" axes = fig.add_axes([0,0,1,1])\n",
|
||||||
"\n",
|
"\n",
|
||||||
" axes.set_title(title, pad=20.0 if events is not None else None)\n",
|
" if title is not None:\n",
|
||||||
|
" axes.set_title(title, pad=20.0 if True in [len(x.test.events) > 0 for x in series.values()] else None)\n",
|
||||||
|
"\n",
|
||||||
" axes.set_xlabel('Time (s)')\n",
|
" axes.set_xlabel('Time (s)')\n",
|
||||||
" axes.set_ylabel('Throughput (Mbps)')\n",
|
" axes.set_ylabel('Throughput (Mbps)')\n",
|
||||||
"\n",
|
"\n",
|
||||||
" for k, v in data.items():\n",
|
" for k, v in series.items():\n",
|
||||||
" intervals = [x['sum'] for x in v['intervals'] if not x['sum']['omitted']]\n",
|
" data = v.summarise()\n",
|
||||||
"\n",
|
"\n",
|
||||||
" x_axis = [((x['start'] + x['end'])/2) for x in intervals]\n",
|
" axes.errorbar(\n",
|
||||||
" y_axis = [x['bits_per_second']/1e6 for x in intervals]\n",
|
" data.keys(),\n",
|
||||||
" axes.plot(x_axis, y_axis, next(cycol), label=k)\n",
|
" [x/1e6 for x in data.values()],\n",
|
||||||
|
" xerr=([x[0] for x in v.time_range().values()], [x[1] for x in v.time_range().values()]) if error_bars_x else None,\n",
|
||||||
|
" yerr=[x*1.5/1e6 for x in v.standard_deviation().values()] if error_bars_y else None,\n",
|
||||||
|
" capsize=3,\n",
|
||||||
|
" ecolor='grey',\n",
|
||||||
|
" color=next(cycol),\n",
|
||||||
|
" label=k,\n",
|
||||||
|
" )\n",
|
||||||
"\n",
|
"\n",
|
||||||
" legend = axes.legend()\n",
|
" legend = axes.legend()\n",
|
||||||
"\n",
|
"\n",
|
||||||
@ -260,7 +244,7 @@
|
|||||||
" axes.set_ylim(bottom=0)\n",
|
" axes.set_ylim(bottom=0)\n",
|
||||||
" axes.set_xlim(left=0)\n",
|
" axes.set_xlim(left=0)\n",
|
||||||
"\n",
|
"\n",
|
||||||
" if events is not None:\n",
|
" if False:\n",
|
||||||
" for k, v in events.items():\n",
|
" for k, v in events.items():\n",
|
||||||
" axes.axvline(k, linestyle='--', color='grey')\n",
|
" axes.axvline(k, linestyle='--', color='grey')\n",
|
||||||
" axes.annotate(v, (k, 1.02), xycoords=axes.get_xaxis_transform(), ha='center')\n",
|
" axes.annotate(v, (k, 1.02), xycoords=axes.get_xaxis_transform(), ha='center')\n",
|
||||||
@ -293,11 +277,13 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"plot_iperf_results(\n",
|
"plot_iperf_results(\n",
|
||||||
" {\n",
|
" {\n",
|
||||||
" '1x2MB Connection (not proxied)': directionInbound['One2MBNotProxied'],\n",
|
" '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n",
|
||||||
" '2x1MB Connections (proxied)': directionInbound['Two1MBProxied'],\n",
|
" '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
|
||||||
" '1x1MB Connection (not proxied)': directionInbound['One1MBNotProxied'],\n",
|
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
|
||||||
" },\n",
|
" },\n",
|
||||||
" 'Two Equal 1MB Connections',\n",
|
" 'Scaling of Equal Connections',\n",
|
||||||
|
" error_bars_x=True,\n",
|
||||||
|
" error_bars_y=True,\n",
|
||||||
")"
|
")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@ -313,10 +299,13 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"plot_iperf_results(\n",
|
"plot_iperf_results(\n",
|
||||||
" {\n",
|
" {\n",
|
||||||
" '2x2MB Connections (proxied)': directionInbound['Two2MBProxied'],\n",
|
" '4x2MB Connections (proxied)': StandardTest([2,2,2,2]),\n",
|
||||||
" '1x2MB Connection (not proxied)': directionInbound['One2MBNotProxied'],\n",
|
" '3x2MB Connections (proxied)': StandardTest([2,2,2]),\n",
|
||||||
|
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
|
||||||
" },\n",
|
" },\n",
|
||||||
" 'Two Equal 2MB Connections',\n",
|
" 'Scaling of Equal Connections',\n",
|
||||||
|
" error_bars_x=True,\n",
|
||||||
|
" error_bars_y=True,\n",
|
||||||
")"
|
")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@ -332,11 +321,9 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"plot_iperf_results(\n",
|
"plot_iperf_results(\n",
|
||||||
" {\n",
|
" {\n",
|
||||||
" '4x1MBps Connections (proxied)': directionInbound['Four1MBProxied'],\n",
|
" 'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n",
|
||||||
" '3x1MBps Connections (proxied)': directionInbound['Three1MBProxied'],\n",
|
|
||||||
" '2x1MBps Connections (proxied)': directionInbound['Two1MBProxied'],\n",
|
|
||||||
" },\n",
|
" },\n",
|
||||||
" 'More Equal Connections',\n",
|
" error_bars_y=True,\n",
|
||||||
")"
|
")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@ -349,15 +336,7 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": []
|
||||||
"plot_iperf_results(\n",
|
|
||||||
" {\n",
|
|
||||||
" '1x2MBps + 1xYMBps Connections (proxied)': directionInbound['One2MBOneYMBProxiedSlow15Return30'],\n",
|
|
||||||
" },\n",
|
|
||||||
" 'Network Slow',\n",
|
|
||||||
" events={0: 'Y=2', 15: 'Y=1', 30: 'Y=2'}\n",
|
|
||||||
")\n"
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"metadata": {
|
"metadata": {
|
||||||
|
@ -3,4 +3,4 @@ from .structure import Node
|
|||||||
from .structure import IpMethod, Interface, Bridge
|
from .structure import IpMethod, Interface, Bridge
|
||||||
|
|
||||||
from .structure import SpeedTestServer, LocalPortal, RemotePortal
|
from .structure import SpeedTestServer, LocalPortal, RemotePortal
|
||||||
from .structure import StandardEnvironment, StandardTest, IperfResult
|
from .structure import StandardEnvironment, StandardTest, StandardIperfResult
|
||||||
|
@ -348,7 +348,7 @@ class StandardTest:
|
|||||||
rates: List[int],
|
rates: List[int],
|
||||||
events: Dict[float, Tuple[int, int]] = None,
|
events: Dict[float, Tuple[int, int]] = None,
|
||||||
duration: int = 10,
|
duration: int = 10,
|
||||||
variation_target: float = 0.4,
|
variation_target: float = 0.3,
|
||||||
max_failures: int = 3,
|
max_failures: int = 3,
|
||||||
max_attempts: int = 60,
|
max_attempts: int = 60,
|
||||||
):
|
):
|
||||||
@ -367,10 +367,11 @@ class StandardTest:
|
|||||||
return ''.join(name_builder)
|
return ''.join(name_builder)
|
||||||
|
|
||||||
|
|
||||||
class IperfResult:
|
class StandardIperfResult:
|
||||||
def __init__(self, iperf: str, interval_size=1.0, duration=30):
|
def __init__(self, test: StandardTest, iperf: str, interval_size=1.0):
|
||||||
|
self.test = test
|
||||||
|
|
||||||
self.interval_size = interval_size
|
self.interval_size = interval_size
|
||||||
self.duration = duration
|
|
||||||
|
|
||||||
# list containing an exact time and a value
|
# list containing an exact time and a value
|
||||||
self.data: List[Tuple[float, float]] = []
|
self.data: List[Tuple[float, float]] = []
|
||||||
@ -392,11 +393,10 @@ class IperfResult:
|
|||||||
self.data.append((time, result))
|
self.data.append((time, result))
|
||||||
|
|
||||||
def bins(self) -> List[List[Tuple[float, float]]]:
|
def bins(self) -> List[List[Tuple[float, float]]]:
|
||||||
# Binning phase
|
bins: List[List[Tuple[float, float]]] = [[] for _ in np.arange(0, self.test.duration, self.interval_size)]
|
||||||
bins: List[List[Tuple[float, float]]] = [[] for _ in np.arange(0, self.duration, self.interval_size)]
|
|
||||||
|
|
||||||
for time, result in self.data:
|
for time, result in self.data:
|
||||||
index = int((time - self.interval_size / 2) / self.interval_size)
|
index = int(np.round((time - self.interval_size / 2) / self.interval_size))
|
||||||
bins[index].append((time, result))
|
bins[index].append((time, result))
|
||||||
|
|
||||||
return bins
|
return bins
|
||||||
@ -404,13 +404,13 @@ class IperfResult:
|
|||||||
def summarise(self) -> Dict[float, float]:
|
def summarise(self) -> Dict[float, float]:
|
||||||
bins = self.bins()
|
bins = self.bins()
|
||||||
means = [np.mean(x, axis=0)[1] for x in bins]
|
means = [np.mean(x, axis=0)[1] for x in bins]
|
||||||
times = [i + self.interval_size / 2 for i in np.arange(0, self.duration, self.interval_size)]
|
times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)]
|
||||||
return dict(zip(times, means))
|
return dict(zip(times, means))
|
||||||
|
|
||||||
def standard_deviation(self) -> Dict[float, float]:
|
def standard_deviation(self) -> Dict[float, float]:
|
||||||
bins = self.bins()
|
bins = self.bins()
|
||||||
stds = [np.std(x, axis=0)[1] for x in bins]
|
stds = [np.std(x, axis=0)[1] for x in bins]
|
||||||
times = [i + self.interval_size / 2 for i in np.arange(0, self.duration, self.interval_size)]
|
times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)]
|
||||||
return dict(zip(times, stds))
|
return dict(zip(times, stds))
|
||||||
|
|
||||||
def coefficient_variance(self) -> Dict[float, float]:
|
def coefficient_variance(self) -> Dict[float, float]:
|
||||||
@ -421,8 +421,8 @@ class IperfResult:
|
|||||||
|
|
||||||
def time_range(self) -> Dict[float, Tuple[float, float]]:
|
def time_range(self) -> Dict[float, Tuple[float, float]]:
|
||||||
bins = self.bins()
|
bins = self.bins()
|
||||||
times = [i + self.interval_size / 2 for i in np.arange(0, self.duration, self.interval_size)]
|
times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)]
|
||||||
ranges = [(np.min(x, axis=0)[0] - time, np.max(x, axis=0)[0] - time) for (x, time) in zip(bins, times)]
|
ranges = [(-np.min(x, axis=0)[0] + time, np.max(x, axis=0)[0] - time) for (x, time) in zip(bins, times)]
|
||||||
return dict(zip(times, ranges))
|
return dict(zip(times, ranges))
|
||||||
|
|
||||||
|
|
||||||
@ -462,7 +462,7 @@ class StandardEnvironment:
|
|||||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||||
self._runner.teardown()
|
self._runner.teardown()
|
||||||
|
|
||||||
def test(self, test: StandardTest) -> Tuple[IperfResult, IperfResult]:
|
def test(self, test: StandardTest) -> Tuple[StandardIperfResult, StandardIperfResult]:
|
||||||
if len(test.rates) != self._interfaces:
|
if len(test.rates) != self._interfaces:
|
||||||
raise RuntimeError('mismatched number of interfaces')
|
raise RuntimeError('mismatched number of interfaces')
|
||||||
|
|
||||||
@ -471,7 +471,7 @@ class StandardEnvironment:
|
|||||||
|
|
||||||
results = []
|
results = []
|
||||||
for server, client in [(self.cl, self.st), (self.st, self.cl)]:
|
for server, client in [(self.cl, self.st), (self.st, self.cl)]:
|
||||||
result: Optional[IperfResult] = None
|
result: Optional[StandardIperfResult] = None
|
||||||
|
|
||||||
for i in range(test.max_attempts):
|
for i in range(test.max_attempts):
|
||||||
if i > 2 and max(result.coefficient_variance().values()) < test.variation_target:
|
if i > 2 and max(result.coefficient_variance().values()) < test.variation_target:
|
||||||
@ -482,11 +482,14 @@ class StandardEnvironment:
|
|||||||
server.server()
|
server.server()
|
||||||
|
|
||||||
for t, (iface, rate) in test.events.items():
|
for t, (iface, rate) in test.events.items():
|
||||||
threading.Timer(5 + t, lambda: self.lp.get_interfaces()[iface].set_rate(rate))
|
threading.Timer(
|
||||||
|
5 + t,
|
||||||
|
(lambda s: lambda: s.lp.get_interfaces()[iface].set_rate(rate))(self),
|
||||||
|
)
|
||||||
|
|
||||||
iperf = client.client(server, time=test.duration)
|
iperf = client.client(server, time=test.duration)
|
||||||
if result is None:
|
if result is None:
|
||||||
result = IperfResult(iperf)
|
result = StandardIperfResult(test, iperf)
|
||||||
else:
|
else:
|
||||||
result.add_results(iperf)
|
result.add_results(iperf)
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user