JakeHillion/dissertation-3-evaluation
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

improved variation

Browse Source
This commit is contained in:
Jake Hillion 2020-11-14 10:50:20 +00:00
parent d700800a78
commit e47e058c3f
3 changed files with 70 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
evaluation.ipynb
View File

@ -30,13 +30,10 @@
"source": [ "source": [
"import os\n", "import os\n",
"import ipaddress\n", "import ipaddress\n",
"import threading\n",
"from typing import Dict\n", "from typing import Dict\n",
"\n", "\n",
"import runners\n", "import runners\n",
"from structure import Bridge, Interface, IpMethod\n", "from structure import StandardEnvironment, StandardTest, StandardIperfResult\n",
"from structure import RemotePortal, LocalPortal, SpeedTestServer\n",
"from structure import StandardEnvironment, StandardTest, IperfResult\n",
"\n", "\n",
"%load_ext dotenv\n", "%load_ext dotenv\n",
"%dotenv" "%dotenv"
@ -86,8 +83,8 @@
" 'branch': os.getenv('TARGET_BRANCH'),\n", " 'branch': os.getenv('TARGET_BRANCH'),\n",
"}\n", "}\n",
"\n", "\n",
"directionInbound: Dict[str, IperfResult] = {}\n", "directionInbound: Dict[str, StandardIperfResult] = {}\n",
"directionOutbound: Dict[str, IperfResult] = {}\n", "directionOutbound: Dict[str, StandardIperfResult] = {}\n",
"\n", "\n",
"def run_and_save_test(env: StandardEnvironment, test: StandardTest):\n", "def run_and_save_test(env: StandardEnvironment, test: StandardTest):\n",
" (directionInbound[test.name()], directionOutbound[test.name()]) = env.test(test)" " (directionInbound[test.name()], directionOutbound[test.name()]) = env.test(test)"
@ -105,42 +102,6 @@
"### Direct Server to Server Testing" "### Direct Server to Server Testing"
] ]
}, },
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"st1 = SpeedTestServer()\n",
"st2 = SpeedTestServer()\n",
"\n",
"top_level_bridge = Bridge(\n",
" st1.get_interfaces()[0],\n",
" st2.get_interfaces()[0],\n",
")\n",
"\n",
"try:\n",
" runner.build(top_level_bridge)\n",
"\n",
" st2.get_interfaces()[0].set_rate(1)\n",
" st2.server()\n",
" directionInbound['One1MBNotProxied'] = st1.client(st2)\n",
" st1.server()\n",
" directionOutbound['One1MBNotProxied'] = st2.client(st1)\n",
"\n",
" st2.get_interfaces()[0].set_rate(2)\n",
" st2.server()\n",
" directionInbound['One2MBNotProxied'] = st1.client(st2)\n",
" st1.server()\n",
" directionOutbound['One2MBNotProxied'] = st2.client(st1)\n",
"finally:\n",
" runner.teardown()"
]
},
{ {
"cell_type": "markdown", "cell_type": "markdown",
"metadata": { "metadata": {
@ -190,7 +151,8 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"with StandardEnvironment(3, runner, setup_params) as env:\n", "with StandardEnvironment(3, runner, setup_params) as env:\n",
" run_and_save_test(env, StandardTest([1,1,1]))\n" " run_and_save_test(env, StandardTest([1,1,1]))\n",
" run_and_save_test(env, StandardTest([2,2,2]))\n"
] ]
}, },
{ {
@ -213,7 +175,8 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"with StandardEnvironment(4, runner, setup_params) as env:\n", "with StandardEnvironment(4, runner, setup_params) as env:\n",
" run_and_save_test(env, StandardTest([1,1,1,1]))" " run_and_save_test(env, StandardTest([1,1,1,1]))\n",
" run_and_save_test(env, StandardTest([2,2,2,2]))\n"
] ]
}, },
{ {
@ -237,22 +200,43 @@
"from itertools import cycle\n", "from itertools import cycle\n",
"import matplotlib.pyplot as plt\n", "import matplotlib.pyplot as plt\n",
"\n", "\n",
"def plot_iperf_results(data, title, events=None, filename=None, start_at_zero=True):\n", "def plot_iperf_results(\n",
" series: Dict[str, StandardTest],\n",
" title: str = None,\n",
" direction = 'inbound',\n",
" error_bars_x=False,\n",
" error_bars_y=False,\n",
" filename=None,\n",
" start_at_zero=True,\n",
"):\n",
" series = {\n",
" k: (directionInbound if direction == 'inbound' else directionOutbound)[v.name()] for (k, v) in series.items()\n",
" }\n",
"\n",
" cycol = cycle('brgy')\n", " cycol = cycle('brgy')\n",
"\n", "\n",
" fig = plt.figure()\n", " fig = plt.figure()\n",
" axes = fig.add_axes([0,0,1,1])\n", " axes = fig.add_axes([0,0,1,1])\n",
"\n", "\n",
" axes.set_title(title, pad=20.0 if events is not None else None)\n", " if title is not None:\n",
" axes.set_title(title, pad=20.0 if True in [len(x.test.events) > 0 for x in series.values()] else None)\n",
"\n",
" axes.set_xlabel('Time (s)')\n", " axes.set_xlabel('Time (s)')\n",
" axes.set_ylabel('Throughput (Mbps)')\n", " axes.set_ylabel('Throughput (Mbps)')\n",
"\n", "\n",
" for k, v in data.items():\n", " for k, v in series.items():\n",
" intervals = [x['sum'] for x in v['intervals'] if not x['sum']['omitted']]\n", " data = v.summarise()\n",
"\n", "\n",
" x_axis = [((x['start'] + x['end'])/2) for x in intervals]\n", " axes.errorbar(\n",
" y_axis = [x['bits_per_second']/1e6 for x in intervals]\n", " data.keys(),\n",
" axes.plot(x_axis, y_axis, next(cycol), label=k)\n", " [x/1e6 for x in data.values()],\n",
" xerr=([x[0] for x in v.time_range().values()], [x[1] for x in v.time_range().values()]) if error_bars_x else None,\n",
" yerr=[x*1.5/1e6 for x in v.standard_deviation().values()] if error_bars_y else None,\n",
" capsize=3,\n",
" ecolor='grey',\n",
" color=next(cycol),\n",
" label=k,\n",
" )\n",
"\n", "\n",
" legend = axes.legend()\n", " legend = axes.legend()\n",
"\n", "\n",
@ -260,7 +244,7 @@
" axes.set_ylim(bottom=0)\n", " axes.set_ylim(bottom=0)\n",
" axes.set_xlim(left=0)\n", " axes.set_xlim(left=0)\n",
"\n", "\n",
" if events is not None:\n", " if False:\n",
" for k, v in events.items():\n", " for k, v in events.items():\n",
" axes.axvline(k, linestyle='--', color='grey')\n", " axes.axvline(k, linestyle='--', color='grey')\n",
" axes.annotate(v, (k, 1.02), xycoords=axes.get_xaxis_transform(), ha='center')\n", " axes.annotate(v, (k, 1.02), xycoords=axes.get_xaxis_transform(), ha='center')\n",
@ -293,11 +277,13 @@
"source": [ "source": [
"plot_iperf_results(\n", "plot_iperf_results(\n",
" {\n", " {\n",
" '1x2MB Connection (not proxied)': directionInbound['One2MBNotProxied'],\n", " '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n",
" '2x1MB Connections (proxied)': directionInbound['Two1MBProxied'],\n", " '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
" '1x1MB Connection (not proxied)': directionInbound['One1MBNotProxied'],\n", " '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" },\n", " },\n",
" 'Two Equal 1MB Connections',\n", " 'Scaling of Equal Connections',\n",
" error_bars_x=True,\n",
" error_bars_y=True,\n",
")" ")"
] ]
}, },
@ -313,10 +299,13 @@
"source": [ "source": [
"plot_iperf_results(\n", "plot_iperf_results(\n",
" {\n", " {\n",
" '2x2MB Connections (proxied)': directionInbound['Two2MBProxied'],\n", " '4x2MB Connections (proxied)': StandardTest([2,2,2,2]),\n",
" '1x2MB Connection (not proxied)': directionInbound['One2MBNotProxied'],\n", " '3x2MB Connections (proxied)': StandardTest([2,2,2]),\n",
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
" },\n", " },\n",
" 'Two Equal 2MB Connections',\n", " 'Scaling of Equal Connections',\n",
" error_bars_x=True,\n",
" error_bars_y=True,\n",
")" ")"
] ]
}, },
@ -332,11 +321,9 @@
"source": [ "source": [
"plot_iperf_results(\n", "plot_iperf_results(\n",
" {\n", " {\n",
" '4x1MBps Connections (proxied)': directionInbound['Four1MBProxied'],\n", " 'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n",
" '3x1MBps Connections (proxied)': directionInbound['Three1MBProxied'],\n",
" '2x1MBps Connections (proxied)': directionInbound['Two1MBProxied'],\n",
" },\n", " },\n",
" 'More Equal Connections',\n", " error_bars_y=True,\n",
")" ")"
] ]
}, },
@ -349,15 +336,7 @@
} }
}, },
"outputs": [], "outputs": [],
"source": [ "source": []
"plot_iperf_results(\n",
" {\n",
" '1x2MBps + 1xYMBps Connections (proxied)': directionInbound['One2MBOneYMBProxiedSlow15Return30'],\n",
" },\n",
" 'Network Slow',\n",
" events={0: 'Y=2', 15: 'Y=1', 30: 'Y=2'}\n",
")\n"
]
} }
], ],
"metadata": { "metadata": {

2
structure/__init__.py
View File

@ -3,4 +3,4 @@ from .structure import Node
from .structure import IpMethod, Interface, Bridge from .structure import IpMethod, Interface, Bridge
from .structure import SpeedTestServer, LocalPortal, RemotePortal from .structure import SpeedTestServer, LocalPortal, RemotePortal
from .structure import StandardEnvironment, StandardTest, IperfResult from .structure import StandardEnvironment, StandardTest, StandardIperfResult

33
structure/structure.py
View File

@ -348,7 +348,7 @@ class StandardTest:
rates: List[int], rates: List[int],
events: Dict[float, Tuple[int, int]] = None, events: Dict[float, Tuple[int, int]] = None,
duration: int = 10, duration: int = 10,
variation_target: float = 0.4, variation_target: float = 0.3,
max_failures: int = 3, max_failures: int = 3,
max_attempts: int = 60, max_attempts: int = 60,
): ):
@ -367,10 +367,11 @@ class StandardTest:
return ''.join(name_builder) return ''.join(name_builder)
class IperfResult: class StandardIperfResult:
def __init__(self, iperf: str, interval_size=1.0, duration=30): def __init__(self, test: StandardTest, iperf: str, interval_size=1.0):
self.test = test
self.interval_size = interval_size self.interval_size = interval_size
self.duration = duration
# list containing an exact time and a value # list containing an exact time and a value
self.data: List[Tuple[float, float]] = [] self.data: List[Tuple[float, float]] = []
@ -392,11 +393,10 @@ class IperfResult:
self.data.append((time, result)) self.data.append((time, result))
def bins(self) -> List[List[Tuple[float, float]]]: def bins(self) -> List[List[Tuple[float, float]]]:
# Binning phase bins: List[List[Tuple[float, float]]] = [[] for _ in np.arange(0, self.test.duration, self.interval_size)]
bins: List[List[Tuple[float, float]]] = [[] for _ in np.arange(0, self.duration, self.interval_size)]
for time, result in self.data: for time, result in self.data:
index = int((time - self.interval_size / 2) / self.interval_size) index = int(np.round((time - self.interval_size / 2) / self.interval_size))
bins[index].append((time, result)) bins[index].append((time, result))
return bins return bins
@ -404,13 +404,13 @@ class IperfResult:
def summarise(self) -> Dict[float, float]: def summarise(self) -> Dict[float, float]:
bins = self.bins() bins = self.bins()
means = [np.mean(x, axis=0)[1] for x in bins] means = [np.mean(x, axis=0)[1] for x in bins]
times = [i + self.interval_size / 2 for i in np.arange(0, self.duration, self.interval_size)] times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)]
return dict(zip(times, means)) return dict(zip(times, means))
def standard_deviation(self) -> Dict[float, float]: def standard_deviation(self) -> Dict[float, float]:
bins = self.bins() bins = self.bins()
stds = [np.std(x, axis=0)[1] for x in bins] stds = [np.std(x, axis=0)[1] for x in bins]
times = [i + self.interval_size / 2 for i in np.arange(0, self.duration, self.interval_size)] times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)]
return dict(zip(times, stds)) return dict(zip(times, stds))
def coefficient_variance(self) -> Dict[float, float]: def coefficient_variance(self) -> Dict[float, float]:
@ -421,8 +421,8 @@ class IperfResult:
def time_range(self) -> Dict[float, Tuple[float, float]]: def time_range(self) -> Dict[float, Tuple[float, float]]:
bins = self.bins() bins = self.bins()
times = [i + self.interval_size / 2 for i in np.arange(0, self.duration, self.interval_size)] times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)]
ranges = [(np.min(x, axis=0)[0] - time, np.max(x, axis=0)[0] - time) for (x, time) in zip(bins, times)] ranges = [(-np.min(x, axis=0)[0] + time, np.max(x, axis=0)[0] - time) for (x, time) in zip(bins, times)]
return dict(zip(times, ranges)) return dict(zip(times, ranges))
@ -462,7 +462,7 @@ class StandardEnvironment:
def __exit__(self, exc_type, exc_val, exc_tb): def __exit__(self, exc_type, exc_val, exc_tb):
self._runner.teardown() self._runner.teardown()
def test(self, test: StandardTest) -> Tuple[IperfResult, IperfResult]: def test(self, test: StandardTest) -> Tuple[StandardIperfResult, StandardIperfResult]:
if len(test.rates) != self._interfaces: if len(test.rates) != self._interfaces:
raise RuntimeError('mismatched number of interfaces') raise RuntimeError('mismatched number of interfaces')
@ -471,7 +471,7 @@ class StandardEnvironment:
results = [] results = []
for server, client in [(self.cl, self.st), (self.st, self.cl)]: for server, client in [(self.cl, self.st), (self.st, self.cl)]:
result: Optional[IperfResult] = None result: Optional[StandardIperfResult] = None
for i in range(test.max_attempts): for i in range(test.max_attempts):
if i > 2 and max(result.coefficient_variance().values()) < test.variation_target: if i > 2 and max(result.coefficient_variance().values()) < test.variation_target:
@ -482,11 +482,14 @@ class StandardEnvironment:
server.server() server.server()
for t, (iface, rate) in test.events.items(): for t, (iface, rate) in test.events.items():
threading.Timer(5 + t, lambda: self.lp.get_interfaces()[iface].set_rate(rate)) threading.Timer(
5 + t,
(lambda s: lambda: s.lp.get_interfaces()[iface].set_rate(rate))(self),
)
iperf = client.client(server, time=test.duration) iperf = client.client(server, time=test.duration)
if result is None: if result is None:
result = IperfResult(iperf) result = StandardIperfResult(test, iperf)
else: else:
result.add_results(iperf) result.add_results(iperf)