JakeHillion/dissertation-3-evaluation
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request 'bar-graphs' (#7) from bar-graphs into develop

Browse Source 
Reviewed-on: #7
This commit is contained in:
JakeHillion 2021-02-16 15:47:45 +00:00
commit fe75cc59b4
11 changed files with 701 additions and 868 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,4 +1,4 @@
graphs/
output/
# Created by https://www.toptal.com/developers/gitignore/api/python,intellij+all,dotenv
# Edit at https://www.toptal.com/developers/gitignore?templates=python,intellij+all,dotenv

600
evaluation.ipynb
View File

@ -2,7 +2,9 @@
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"metadata": {
"collapsed": false
},
"source": [
"# Project Evaluation\n",
"\n",
@ -13,9 +15,7 @@
{
"cell_type": "markdown",
"metadata": {
"pycharm": {
"name": "#%% md\n"
}
"collapsed": false
},
"source": [
"## Setup\n",
@ -25,16 +25,20 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"import os\n",
"import ipaddress\n",
"from typing import Dict\n",
"\n",
"import runners\n",
"from structure import StandardEnvironment, StandardTest, StandardIperfResult\n",
"from structure.structure import DirectEnvironment, DirectTest\n",
"from structure import StandardEnvironment, StandardTest\n",
"from structure import DirectEnvironment, DirectTest\n",
"from structure import BaseEnvironment\n",
"\n",
"%load_ext dotenv\n",
"%dotenv"
@ -43,9 +47,7 @@
{
"cell_type": "markdown",
"metadata": {
"pycharm": {
"name": "#%% md\n"
}
"collapsed": false
},
"source": [
"## Testing\n",
@ -82,13 +84,27 @@
" 'access_key': os.getenv('S3_ACCESS_KEY'),\n",
" 'secret_key': os.getenv('S3_SECRET_KEY'),\n",
" 'branch': os.getenv('TARGET_BRANCH'),\n",
"}\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"from graphs import TestStore\n",
"\n",
"directionInbound: Dict[str, StandardIperfResult] = {}\n",
"directionOutbound: Dict[str, StandardIperfResult] = {}\n",
"test_store = TestStore()\n",
"\n",
"def run_and_save_test(env: StandardEnvironment, test: StandardTest):\n",
" (directionInbound[test.name()], directionOutbound[test.name()]) = env.test(test)\n",
"def run_and_save_test(env: BaseEnvironment, test: StandardTest):\n",
" (inbound, outbound) = env.test(test)\n",
" test_store.save_inbound(test, inbound)\n",
" test_store.save_outbound(test, outbound)\n",
"\n",
"def attempt_n_times(foo, n=3):\n",
" for i in range(n):\n",
@ -107,10 +123,7 @@
{
"cell_type": "markdown",
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
"collapsed": false
},
"source": [
"### Direct Server to Server"
@ -128,10 +141,10 @@
"source": [
"def direct_tests():\n",
" with DirectEnvironment(runner) as env:\n",
" run_and_save_test(env, DirectTest(1, variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, DirectTest(2, variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, DirectTest(3, variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, DirectTest(4, variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, DirectTest(1, bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
" run_and_save_test(env, DirectTest(2, bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
" run_and_save_test(env, DirectTest(3, bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
" run_and_save_test(env, DirectTest(4, bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
"\n",
"attempt_n_times(direct_tests)"
]
@ -139,10 +152,7 @@
{
"cell_type": "markdown",
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
"collapsed": false
},
"source": [
"### Local Portal with 2 Interfaces"
@ -160,14 +170,14 @@
"source": [
"def two_interface_tests():\n",
" with StandardEnvironment(2, runner, setup_params) as env:\n",
" run_and_save_test(env, StandardTest([1,1], variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, StandardTest([1,2], variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, StandardTest([2,2], variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, StandardTest([1,1], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
" run_and_save_test(env, StandardTest([1,2], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
" run_and_save_test(env, StandardTest([2,2], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
" run_and_save_test(env, StandardTest(\n",
" [2,2],\n",
" events={10: (0,1), 15: (0,2)},\n",
" events={10: (0,1), 20: (0,2)},\n",
" duration=30,\n",
" variation_target=0.8 if fast_tests else 0.5,\n",
" interval_variation_target=0.8 if fast_tests else 0.5,\n",
" ))\n",
"\n",
"attempt_n_times(two_interface_tests)"
@ -194,8 +204,8 @@
"source": [
"def three_interface_tests():\n",
" with StandardEnvironment(3, runner, setup_params) as env:\n",
" run_and_save_test(env, StandardTest([1,1,1], variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, StandardTest([2,2,2], variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, StandardTest([1,1,1], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
" run_and_save_test(env, StandardTest([2,2,2], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
"\n",
"attempt_n_times(three_interface_tests)"
]
@ -221,8 +231,8 @@
"source": [
"def four_interface_tests():\n",
" with StandardEnvironment(4, runner, setup_params) as env:\n",
" run_and_save_test(env, StandardTest([1,1,1,1], variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, StandardTest([2,2,2,2], variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, StandardTest([1,1,1,1], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
" run_and_save_test(env, StandardTest([2,2,2,2], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
"\n",
"attempt_n_times(four_interface_tests)"
]
@ -230,9 +240,7 @@
{
"cell_type": "markdown",
"metadata": {
"pycharm": {
"name": "#%% md\n"
}
"collapsed": false
},
"source": [
"## Graphs\n",
@ -249,107 +257,24 @@
},
"outputs": [],
"source": [
"from itertools import cycle\n",
"import matplotlib.pyplot as plt\n",
"from graphs import plot_iperf_results, plot_iperf_results_time\n",
"\n",
"def plot_iperf_results(\n",
" series: Dict[str, StandardTest],\n",
" title: str = None,\n",
" direction = 'both',\n",
" error_bars_x=False,\n",
" error_bars_y=True,\n",
" filename=None,\n",
" start_at_zero=True,\n",
"):\n",
" if direction == 'both':\n",
" plot_iperf_results(series, title, 'outbound', error_bars_x, error_bars_y, filename, start_at_zero)\n",
" plot_iperf_results(series, title, 'inbound', error_bars_x, error_bars_y, filename, start_at_zero)\n",
" return\n",
"if not os.path.exists('output/'):\n",
" os.makedirs('output/')\n",
"\n",
" if filename in ['png', 'eps']:\n",
" filename = 'graphs/{}{}{}{}.{}'.format(\n",
" 'I' if direction == 'inbound' else 'O',\n",
" 'Ex' if error_bars_x else '',\n",
" 'Ey' if error_bars_y else '',\n",
" ''.join(['S{}-{}'.format(i,x.name()) for (i, x) in enumerate(series.values())]),\n",
" filename,\n",
" )\n",
"\n",
" series = {\n",
" k: (directionInbound if direction == 'inbound' else directionOutbound)[v.name()] for (k, v) in series.items()\n",
" }\n",
"\n",
" cycol = cycle('brgy')\n",
"\n",
" fig = plt.figure()\n",
" axes = fig.add_axes([0,0,1,1])\n",
"\n",
" if title is not None:\n",
" axes.set_title(title, pad=20.0 if True in [len(x.test.events) > 0 for x in series.values()] else None)\n",
"\n",
" axes.set_xlabel('Time (s)')\n",
" axes.set_ylabel('Throughput (Mbps)')\n",
"\n",
" for k, v in series.items():\n",
" data = v.summarise()\n",
"\n",
" axes.errorbar(\n",
" data.keys(),\n",
" [x/1e6 for x in data.values()],\n",
" xerr=([x[0] for x in v.time_range().values()], [x[1] for x in v.time_range().values()]) if error_bars_x else None,\n",
" yerr=[x*1.5/1e6 for x in v.standard_deviation().values()] if error_bars_y else None,\n",
" capsize=3,\n",
" ecolor='grey',\n",
" color=next(cycol),\n",
" label=k,\n",
" )\n",
"\n",
" legend = axes.legend()\n",
"\n",
" if start_at_zero:\n",
" axes.set_ylim(bottom=0)\n",
" axes.set_xlim(left=0)\n",
"\n",
" if False:\n",
" for k, v in events.items():\n",
" axes.axvline(k, linestyle='--', color='grey')\n",
" axes.annotate(v, (k, 1.02), xycoords=axes.get_xaxis_transform(), ha='center')\n",
"\n",
" if fast_tests:\n",
" fig.text(0.95, 0.05, 'Draft', fontsize=50, color='gray', ha='right', va='bottom', alpha=0.5)\n",
"\n",
" if filename is not None:\n",
" fig.savefig(filename, bbox_extra_artists=(legend,), bbox_inches='tight', pad_inches=0.3)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"if not os.path.exists('graphs/'):\n",
" os.makedirs('graphs/')\n",
"\n",
"for filename in os.listdir('graphs'):\n",
" file_path = os.path.join('graphs/', filename)\n",
"for filename in os.listdir('output'):\n",
" file_path = os.path.join('output/', filename)\n",
" os.unlink(file_path)"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
"collapsed": false
},
"source": [
"### Error Bar Evaluation"
"### Section 4.2: Graphs\n",
"#### Subsection 4.2.2 Line Graphs"
]
},
{
@ -362,12 +287,13 @@
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
"plot_iperf_results_time(test_store,\n",
" {\n",
" '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n",
" '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" },\n",
" fast_tests,\n",
" error_bars_x=False,\n",
" error_bars_y=False,\n",
" filename='png',\n",
@ -384,27 +310,19 @@
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
"plot_iperf_results_time(test_store,\n",
" {\n",
" '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n",
" '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" },\n",
" fast_tests,\n",
" error_bars_x=True,\n",
" error_bars_y=False,\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false
},
"source": [
"### More than 2 connections evaluation"
]
},
{
"cell_type": "code",
"execution_count": null,
@ -415,16 +333,29 @@
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
"plot_iperf_results_time(test_store,\n",
" {\n",
" '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n",
" '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" },\n",
" fast_tests,\n",
" error_bars_x=False,\n",
" error_bars_y=True,\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false
},
"source": [
"### Section 4.3: Success Criteria\n",
"#### Subsection 4.3.2: Bidirectional Performance Gains"
]
},
{
"cell_type": "code",
"execution_count": null,
@ -435,12 +366,13 @@
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
"plot_iperf_results(test_store,\n",
" {\n",
" '4x2MB Connections (proxied)': StandardTest([2,2,2,2]),\n",
" '3x2MB Connections (proxied)': StandardTest([2,2,2]),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" '1x1MB + 1x2MB\\nConnections (proxied)': StandardTest([1,2]),\n",
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
" },\n",
" fast_tests,\n",
" filename='png',\n",
")"
]
@ -451,7 +383,7 @@
"collapsed": false
},
"source": [
"### Mixed Performance Evaluation"
"#### Subsection 4.3.5: More Bandwidth over Two Equal Connections"
]
},
{
@ -464,137 +396,14 @@
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
"plot_iperf_results(test_store,\n",
" {\n",
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
" '1x1MB + 1x2MB Connections (proxied)': StandardTest([1,2]),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false
},
"source": [
"### Eventful Evaluation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" 'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false
},
"source": [
"### Comparisons to a Direct Connection"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '1x2MB Connection (not proxied)': DirectTest(2),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" '1x1MB Connection (not proxied)': DirectTest(1),\n",
" '1x1MB Connection\\n(direct)': DirectTest(1),\n",
" '2x1MB Connections\\n(proxied)': StandardTest([1,1]),\n",
" '1x2MB Connection\\n(direct)': DirectTest(2),\n",
"\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '1x4MB Connection (not proxied)': DirectTest(4),\n",
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
" '1x2MB Connection (not proxied)': DirectTest(2),\n",
"\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false
},
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n",
" '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '4x2MB Connections (proxied)': StandardTest([2,2,2,2]),\n",
" '3x2MB Connections (proxied)': StandardTest([2,2,2]),\n",
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
" },\n",
" fast_tests,\n",
" filename='png',\n",
")"
]
@ -605,7 +414,8 @@
"collapsed": false
},
"source": [
"### Mixed Performance Evaluation"
"### Section 4.4: Extended Goals\n",
"#### Subsection 4.4.1: More Bandwidth over Unequal Connections"
]
},
{
@ -618,12 +428,13 @@
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
"plot_iperf_results(test_store,\n",
" {\n",
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
" '1x1MB + 1x2MB Connections (proxied)': StandardTest([1,2]),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" '2x2MB Connections\\n(proxied)': StandardTest([2,2]),\n",
" '1x1MB + 1x2MB\\nConnections (proxied)': StandardTest([1,2]),\n",
" '2x1MB Connections\\n(proxied)': StandardTest([1,1]),\n",
" },\n",
" fast_tests,\n",
" filename='png',\n",
")"
]
@ -634,7 +445,7 @@
"collapsed": false
},
"source": [
"### Eventful Evaluation"
"#### Subsection 4.4.2: More Bandwidth over Four Equal Connections"
]
},
{
@ -647,10 +458,34 @@
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
"plot_iperf_results(test_store,\n",
" {\n",
" 'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n",
" '4x1MB Connections\\n(proxied)': StandardTest([1,1,1,1]),\n",
" '3x1MB Connections\\n(proxied)': StandardTest([1,1,1]),\n",
" '2x1MB Connections\\n(proxied)': StandardTest([1,1]),\n",
" },\n",
" fast_tests,\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(test_store,\n",
" {\n",
" '4x2MB Connections\\n(proxied)': StandardTest([2,2,2,2]),\n",
" '3x2MB Connections\\n(proxied)': StandardTest([2,2,2]),\n",
" '2x2MB Connections\\n(proxied)': StandardTest([2,2]),\n",
" },\n",
" fast_tests,\n",
" filename='png',\n",
")"
]
@ -661,7 +496,7 @@
"collapsed": false
},
"source": [
"### Comparisons to a Direct Connection"
"#### Subsection 4.4.3: Bandwidth Variation"
]
},
{
@ -674,200 +509,11 @@
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
"plot_iperf_results_time(test_store,\n",
" {\n",
" '1x2MB Connection (not proxied)': DirectTest(2),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" '1x1MB Connection (not proxied)': DirectTest(1),\n",
"\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '1x4MB Connection (not proxied)': DirectTest(4),\n",
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
" '1x2MB Connection (not proxied)': DirectTest(2),\n",
"\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
},
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n",
" '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '4x2MB Connections (proxied)': StandardTest([2,2,2,2]),\n",
" '3x2MB Connections (proxied)': StandardTest([2,2,2]),\n",
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
},
"source": [
"### Mixed Performance Evaluation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
" '1x1MB + 1x2MB Connections (proxied)': StandardTest([1,2]),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
},
"source": [
"### Eventful Evaluation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" 'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
},
"source": [
"### Comparisons to a Direct Connection"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '1x2MB Connection (not proxied)': DirectTest(2),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" '1x1MB Connection (not proxied)': DirectTest(1),\n",
"\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '1x4MB Connection (not proxied)': DirectTest(4),\n",
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
" '1x2MB Connection (not proxied)': DirectTest(2),\n",
"\n",
" 'Varied Connection': StandardTest([2,2], events={10: (0,1), 20: (0,2)}, duration=30),\n",
" },\n",
" fast_tests,\n",
" filename='png',\n",
")"
]
@ -900,7 +546,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.6"
"version": "3.9.0"
}
},
"nbformat": 4,

2
graphs/__init__.py Normal file
View File

@ -0,0 +1,2 @@
from .store import TestStore
from .graphs import plot_iperf_results, plot_iperf_results_time

132
graphs/graphs.py Normal file
View File

@ -0,0 +1,132 @@
from itertools import cycle
from typing import Dict, Union
from matplotlib import pyplot as plt
from graphs import TestStore
from structure import StandardTest
def plot_iperf_results(
store: TestStore,
series: Dict[str, StandardTest],
fast_tests: bool,
title: str = None,
direction='both',
error_bars=True,
filename=None,
):
if direction == 'both':
plot_iperf_results(store, series, fast_tests, title, 'inbound', error_bars, filename)
plot_iperf_results(store, series, fast_tests, title, 'outbound', error_bars, filename)
return
if filename in ['png', 'eps']:
filename = 'output/{}{}{}.{}'.format(
'I' if direction == 'inbound' else 'O',
'E' if error_bars else '',
''.join(['S{}-{}'.format(i, x.name()) for (i, x) in enumerate(series.values())]),
filename,
)
print(filename)
series = {
k: (store.get_inbound(v) if direction == 'inbound' else store.get_outbound(v))
for (k, v) in series.items()
}
fig = plt.figure()
axes = fig.add_axes([0, 0, 1, 1])
if title is not None:
axes.set_title(title, pad=20.0 if True in [len(x.test.events) > 0 for x in series.values()] else None)
axes.set_ylabel('Throughput (Mbps)')
for k, v in series.items():
axes.bar(
k,
v.bandwidth_mean() / 1e6,
yerr=1.5 * v.bandwidth_standard_deviation() / 1e6,
width=0.4,
capsize=15,
)
if fast_tests:
fig.text(0.95, 0.05, 'Draft', fontsize=50, color='gray', ha='right', va='bottom', alpha=0.5)
if filename is not None:
fig.savefig(filename, bbox_inches='tight', pad_inches=0.3)
def plot_iperf_results_time(
store: TestStore,
series: Dict[str, StandardTest],
fast_tests: bool,
title: str = None,
direction='both',
error_bars_x=False,
error_bars_y=True,
filename=None,
start_at_zero=True,
):
if direction == 'both':
plot_iperf_results_time(store, series, fast_tests, title, 'outbound', error_bars_x, error_bars_y, filename,
start_at_zero)
plot_iperf_results_time(store, series, fast_tests, title, 'inbound', error_bars_x, error_bars_y, filename,
start_at_zero)
return
if filename in ['png', 'eps']:
filename = 'output/T{}{}{}{}.{}'.format(
'I' if direction == 'inbound' else 'O',
'Ex' if error_bars_x else '',
'Ey' if error_bars_y else '',
''.join(['S{}-{}'.format(i, x.name()) for (i, x) in enumerate(series.values())]),
filename,
)
print(filename)
series = {
k: (store.get_inbound(v) if direction == 'inbound' else store.get_outbound(v))
for (k, v) in series.items()
}
cycol = cycle('brgy')
fig = plt.figure()
axes = fig.add_axes([0, 0, 1, 1])
if title is not None:
axes.set_title(title, pad=20.0 if True in [len(x.test.events) > 0 for x in series.values()] else None)
axes.set_xlabel('Time (s)')
axes.set_ylabel('Throughput (Mbps)')
for k, v in series.items():
data = v.interval_means()
axes.errorbar(
data.keys(),
[x / 1e6 for x in data.values()],
xerr=(
[x[0] for x in v.interval_time_ranges().values()],
[x[1] for x in v.interval_time_ranges().values()]) if error_bars_x else None,
yerr=[x * 1.5 / 1e6 for x in v.interval_standard_deviations().values()] if error_bars_y else None,
capsize=3,
ecolor='grey',
color=next(cycol),
label=k,
)
legend = axes.legend()
if start_at_zero:
axes.set_ylim(bottom=0)
axes.set_xlim(left=0)
if fast_tests:
fig.text(0.95, 0.05, 'Draft', fontsize=50, color='gray', ha='right', va='bottom', alpha=0.5)
if filename is not None:
fig.savefig(filename, bbox_extra_artists=(legend,), bbox_inches='tight', pad_inches=0.3)

21
graphs/store.py Normal file
View File

@ -0,0 +1,21 @@
from typing import Dict
from structure import IperfResult, StandardTest
class TestStore:
def __init__(self):
self.inbound: Dict[str, IperfResult] = dict()
self.outbound: Dict[str, IperfResult] = dict()
def save_inbound(self, test: StandardTest, result: IperfResult):
self.inbound[test.name()] = result
def save_outbound(self, test: StandardTest, result: IperfResult):
self.outbound[test.name()] = result
def get_inbound(self, test: StandardTest) -> IperfResult:
return self.inbound[test.name()]
def get_outbound(self, test: StandardTest) -> IperfResult:
return self.outbound[test.name()]

62
runners/runners.py
View File

@ -11,6 +11,7 @@ import proxmoxer
import paramiko
import structure
import structure.generic
def check_env(*names: str) -> bool:
@ -21,14 +22,14 @@ def check_env(*names: str) -> bool:
def bridge_node_search(
first: structure.Bridge,
bridge_name_generator: Callable[[structure.Bridge], str],
node_id_generator: Callable[[structure.Node], int],
) -> Tuple[List[structure.Bridge], List[structure.Node]]:
bridges: List[structure.Bridge] = []
nodes: List[structure.Node] = []
first: structure.generic.Bridge,
bridge_name_generator: Callable[[structure.generic.Bridge], str],
node_id_generator: Callable[[structure.generic.Node], int],
) -> Tuple[List[structure.generic.Bridge], List[structure.generic.Node]]:
bridges: List[structure.generic.Bridge] = []
nodes: List[structure.generic.Node] = []
queue: List[structure.Bridge] = [first]
queue: List[structure.generic.Bridge] = [first]
while len(queue) > 0:
bridge = queue.pop()
if bridge.get_name() != '':
@ -38,7 +39,7 @@ def bridge_node_search(
bridge.set_name(bridge_name_generator(bridge))
# from this bridge, find all nodes (via all interfaces)
reachable_nodes: List[structure.Node] = []
reachable_nodes: List[structure.generic.Node] = []
for interface in bridge.get_interfaces():
node = interface.get_node()
if node.get_id() is not None:
@ -63,7 +64,7 @@ class PrintRunner:
self._last_bridge: int = 0
self._last_node_id = 0
def build(self, bridge: structure.Bridge):
def build(self, bridge: structure.generic.Bridge):
bridges, nodes = bridge_node_search(bridge, lambda _: self.name_bridge(), lambda _: self.id_node())
print(bridges)
@ -121,10 +122,10 @@ class ProxmoxRunner:
self._template_id = template_id
self._initial_vm_id = initial_vm_id
self._internet_bridge = structure.Bridge()
self._internet_bridge = structure.generic.Bridge()
self._internet_bridge.set_name(internet_bridge)
self._management_bridge = structure.Bridge()
self._management_bridge = structure.generic.Bridge()
self._management_bridge.set_name(management_bridge)
self._management_initial_ip = management_initial_ip
self._management_netmask = management_netmask
@ -133,7 +134,7 @@ class ProxmoxRunner:
self._private_key = paramiko.RSAKey.generate(3072)
self._client = paramiko.SSHClient()
def build(self, bridge: structure.Bridge):
def build(self, bridge: structure.generic.Bridge):
bridges, nodes = bridge_node_search(bridge, lambda x: self._create_bridge(x), lambda x: self._create_node(x))
self._build_bridges(bridges)
@ -141,13 +142,13 @@ class ProxmoxRunner:
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
build_futures = [executor.submit(self._build_node, node) for node in nodes]
for future in build_futures:
future.result()
future.result(300)
# guarantee that setup is not called until all of the nodes are built
# this means that all will have their final IPs by this point
setup_futures = [executor.submit(self._setup_node, node) for node in nodes]
for future in setup_futures:
future.result()
future.result(300)
def _await_task(self, upid, timeout=10):
t1 = datetime.now()
@ -156,7 +157,7 @@ class ProxmoxRunner:
return
raise TimeoutError
def _create_bridge(self, bridge: structure.Bridge) -> str:
def _create_bridge(self, bridge: structure.generic.Bridge) -> str:
while True:
try:
self._proxmox.nodes(self._proxmox_node).network.post(
@ -177,7 +178,7 @@ class ProxmoxRunner:
self._created_bridges.append(bridge_name)
return bridge_name
def _build_bridges(self, bridges: List[structure.Bridge]):
def _build_bridges(self, bridges: List[structure.generic.Bridge]):
network_task = self._proxmox.nodes(self._proxmox_node).network.put()
self._await_task(network_task)
@ -187,7 +188,7 @@ class ProxmoxRunner:
bridge.new_network()
existing.append(bridge.get_network())
def _create_node(self, node: structure.Node) -> int:
def _create_node(self, node: structure.generic.Node) -> int:
while True:
try:
clone_task = self._proxmox.nodes(self._proxmox_node).qemu(self._template_id).clone.post(
@ -207,10 +208,10 @@ class ProxmoxRunner:
self._created_nodes.append(new_id - 1)
return new_id - 1
def _open_ssh(self, node: structure.Node, interface: structure.Interface = None):
def _open_ssh(self, node: structure.generic.Node, interface: structure.generic.Interface = None):
if interface is None:
for iface in node.get_interfaces():
if iface.get_method() == structure.IpMethod.Management:
if iface.get_method() == structure.generic.IpMethod.Management:
interface = iface
break
if interface is None:
@ -235,13 +236,13 @@ class ProxmoxRunner:
node.ssh_client = client
def _close_ssh(self, node: structure.Node):
def _close_ssh(self, node: structure.generic.Node):
node.ssh_client.close()
del node.ssh_client
def ssh(
self,
node: structure.Node,
node: structure.generic.Node,
command: str,
error_stderr=False,
error_stdout=False,
@ -269,7 +270,7 @@ class ProxmoxRunner:
return exit_status
def _build_node(self, node: structure.Node):
def _build_node(self, node: structure.generic.Node):
# Step 1: Configure access
self._proxmox.nodes(self._proxmox_node).qemu(node.get_id()).config.put(
ciuser='python',
@ -282,7 +283,7 @@ class ProxmoxRunner:
# Step 2: connect to Internet bridge with DHCP to install packages
if node.get_internet_setup() is not None:
interfaces = node.get_interfaces()
internet_interface = structure.Interface(structure.IpMethod.Dhcp4)
internet_interface = structure.generic.Interface(structure.generic.IpMethod.Dhcp4)
internet_interface.set_bridge(self._internet_bridge)
temp_interfaces = [internet_interface, interfaces[len(interfaces) - 1]]
@ -307,7 +308,7 @@ class ProxmoxRunner:
self._open_ssh(node)
node.ssh = (lambda n: lambda *args, **kwargs: self.ssh(n, *args, **kwargs))(node)
def _setup_node_interfaces(self, node: structure.Node, interfaces: List[structure.Interface] = None):
def _setup_node_interfaces(self, node: structure.generic.Node, interfaces: List[structure.generic.Interface] = None):
if interfaces is None:
interfaces = node.get_interfaces()
@ -316,20 +317,20 @@ class ProxmoxRunner:
interface = interfaces[i]
method = interface.get_method()
if method == structure.IpMethod.Management:
if method == structure.generic.IpMethod.Management:
interface.set_bridge(self._management_bridge)
addr = self._management_initial_ip + node.get_id() - self._initial_vm_id
kwargs['ipconfig{}'.format(i)] = 'ip={}/{}'.format(addr, self._management_netmask)
interface.set_address(addr)
elif method == structure.IpMethod.Auto4 or method == structure.IpMethod.Manual:
elif method == structure.generic.IpMethod.Auto4 or method == structure.generic.IpMethod.Manual:
# handle manual the same as auto4 so it doesn't get stuck in DHCP
bridge = interface.get_bridge()
addr = bridge.get_ip_address()
kwargs['ipconfig{}'.format(i)] = 'ip={}/{}'.format(addr, bridge.netmask)
interface.set_address(addr)
elif method == structure.IpMethod.Dhcp4:
elif method == structure.generic.IpMethod.Dhcp4:
kwargs['ipconfig{}'.format(i)] = 'ip=dhcp'
else:
raise RuntimeError('not implemented')
@ -340,7 +341,7 @@ class ProxmoxRunner:
def interface_set_rate(iface):
def new_set_rate(rate: Optional[int]):
structure.Interface.set_rate(iface, rate)
structure.generic.Interface.set_rate(iface, rate)
self._update_node_interfaces(node)
return new_set_rate
@ -349,7 +350,7 @@ class ProxmoxRunner:
self._proxmox.nodes(self._proxmox_node).qemu(node.get_id()).config.put(**kwargs)
def _update_node_interfaces(self, node: structure.Node):
def _update_node_interfaces(self, node: structure.generic.Node):
interfaces = node.get_interfaces()
old_config = self._proxmox.nodes(self._proxmox_node).qemu(node.get_id()).config.get()
@ -368,7 +369,7 @@ class ProxmoxRunner:
self._proxmox.nodes(self._proxmox_node).qemu(node.get_id()).config.put(**new_config)
def _setup_node(self, node: structure.Node):
def _setup_node(self, node: structure.generic.Node):
if node.get_setup() is not None:
cmd = node.get_setup()
self.ssh(node, cmd, error_stdout=True, error_stderr=True)
@ -383,6 +384,7 @@ class ProxmoxRunner:
for bridge in self._created_bridges:
self._proxmox.nodes(self._proxmox_node).network(bridge).delete()
if len(self._created_bridges) > 0:
network_task = self._proxmox.nodes(self._proxmox_node).network.put()
self._await_task(network_task)

7
structure/__init__.py
View File

@ -1,6 +1,5 @@
from .structure import Node
from .structure import IpMethod, Interface, Bridge
from .generic import IpMethod, Interface, Bridge, Node
from .structure import SpeedTestServer, LocalPortal, RemotePortal
from .structure import StandardEnvironment, StandardTest, StandardIperfResult
from .tests import StandardTest, DirectTest, IperfResult
from .environments import StandardEnvironment, DirectEnvironment, BaseEnvironment

114
structure/environments.py Normal file
View File

@ -0,0 +1,114 @@
import threading
from typing import Tuple, Optional
from structure import Bridge, StandardTest, SpeedTestServer, Node, IperfResult, RemotePortal, Interface, \
IpMethod, LocalPortal
from structure.tests import repeat_until_satisfied
class BaseEnvironment:
def __init__(self, runner, top_level_bridge: Bridge):
self.top_level_bridge = top_level_bridge
self._runner = runner
def __enter__(self):
try:
self._runner.build(self.top_level_bridge)
except Exception as e:
self._runner.teardown()
raise e
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self._runner.teardown()
def _test(
self,
test: StandardTest,
inbound_server: SpeedTestServer,
inbound_client: SpeedTestServer,
rated_node: Node,
expected_interfaces: int,
) -> Tuple[IperfResult, IperfResult]:
if len(test.rates) != expected_interfaces:
raise RuntimeError('mismatched number of interfaces')
results = []
for server, client in [(inbound_server, inbound_client), (inbound_client, inbound_server)]:
def test_reducer(old: Optional[IperfResult]) -> IperfResult:
for i, r in enumerate(test.rates):
rated_node.get_interfaces()[i].set_rate(r)
server.server()
for delay, (index, rate) in test.events.items():
iface = rated_node.get_interfaces()[index]
threading.Timer(6 + delay, iface.set_rate, args=[rate]).start()
iperf = client.client(server, time=test.duration)
if old is None:
return IperfResult(test, iperf)
else:
old.add_results(iperf)
return old
def test_satisfier(val: IperfResult) -> bool:
if val.num_tests < 3:
return False
return val.bandwidth_coefficient_variance() < test.bandwidth_variation_target and False not in \
[x < test.interval_variation_target for x in val.interval_coefficient_variances().values()]
result = repeat_until_satisfied(
test_reducer,
test_satisfier,
max_failures=test.max_failures,
max_attempts=test.max_attempts,
)
results.append(result)
# Return a tuple of (inbound, outbound)
return results[0], results[1]
def test(self, test: StandardTest) -> Tuple[IperfResult, IperfResult]:
raise RuntimeError('not implemented')
class StandardEnvironment(BaseEnvironment):
def __init__(self, interfaces: int, runner, setup_params: dict):
self._interfaces = interfaces
self.rp = RemotePortal([Interface(IpMethod.Auto4)], setup_params=setup_params)
self.st = SpeedTestServer()
self.cl = SpeedTestServer(clone_interface=self.rp.get_interfaces()[0])
self.lp = LocalPortal(
[Interface(IpMethod.Auto4) for _ in range(interfaces)],
self.cl,
setup_params=setup_params,
)
self.rp.set_local_portal(self.lp)
self.lp.set_remote_portal(self.rp)
super().__init__(runner, Bridge(
self.st.get_interfaces()[0],
self.rp.get_interfaces()[0],
*self.lp.get_interfaces()[0:interfaces],
))
def test(self, test: StandardTest) -> Tuple[IperfResult, IperfResult]:
return self._test(test, self.st, self.cl, self.lp, self._interfaces)
class DirectEnvironment(BaseEnvironment):
def __init__(self, runner):
self.st1 = SpeedTestServer()
self.st2 = SpeedTestServer()
super().__init__(runner, Bridge(
self.st1.get_interfaces()[0],
self.st2.get_interfaces()[0],
))
def test(self, test: StandardTest) -> Tuple[IperfResult, IperfResult]:
return self._test(test, self.st2, self.st1, self.st2, 1)

132
structure/generic.py Normal file
View File

@ -0,0 +1,132 @@
import ipaddress
import random
from enum import Enum
from typing import Optional, List, Dict, Union
class IpMethod(Enum):
Manual = 0
Management = 1
Auto4 = 2
Auto6 = 3
Dhcp4 = 4
Dhcp6 = 5
class Interface:
def __init__(self, method: IpMethod, rate: Optional[int] = None):
self._method: IpMethod
self._node: Optional[Node] = None
self._rate: Optional[int] = None
self._bridge: Optional[Bridge] = None
self._method = method
self._rate = rate
self._address: ipaddress.ip_address = None
def get_method(self):
return self._method
def set_node(self, node):
self._node = node
def get_node(self):
return self._node
def set_bridge(self, bridge):
self._bridge = bridge
def get_bridge(self):
return self._bridge
def set_address(self, addr: ipaddress.ip_address):
self._address = addr
def get_address(self) -> ipaddress.ip_address:
return self._address
def get_rate(self) -> Optional[int]:
return self._rate
def set_rate(self, rate: Optional[int]):
self._rate = rate
class Bridge:
def __init__(self, *interfaces: Interface):
self._interfaces: List[Interface] = []
self._name: str = ''
for interface in interfaces:
self._interfaces.append(interface)
interface.set_bridge(self)
# Generate a random class c private range by default (10.0.0.0)
self._addr: ipaddress.ip_address = None
self._network_iterator: iter = None
self.netmask = 24
self.new_network()
def get_interfaces(self) -> List[Interface]:
return self._interfaces
def set_name(self, name: str):
self._name = name
def get_name(self) -> str:
return self._name
def set_netmask(self, mask: int):
self.netmask = mask
self._network_iterator = ipaddress.ip_network('{}/{}'.format(self._addr, self.netmask), False).hosts()
def get_ip_address(self) -> ipaddress.ip_address:
return next(self._network_iterator)
def get_network(self) -> ipaddress.ip_network:
return ipaddress.ip_network('{}/{}'.format(self._addr, self.netmask), False)
def get_network_string(self) -> str:
return str(ipaddress.ip_network('{}/{}'.format(self._addr, self.netmask), False))
def new_network(self):
self._addr: ipaddress.ip_address = ipaddress.ip_address('10.0.0.0') + random.randint(0, 16777216)
self._network_iterator = ipaddress.ip_network('{}/{}'.format(self._addr, self.netmask), False).hosts()
class Node:
def __init__(self, interfaces: List[Interface], setup_params: Dict = None):
self._id: Union[int, None] = None
self._interfaces: List[Interface] = interfaces
self._interfaces.append(Interface(IpMethod.Management))
for interface in self._interfaces:
interface.set_node(self)
self.setup_params = {} if setup_params is None else setup_params
def get_interfaces(self):
return self._interfaces
def set_id(self, new_id):
self._id = new_id
def get_id(self):
return self._id
def get_core_count(self) -> int:
return 2
def get_memory_mb(self) -> int:
return 2048
def get_internet_setup(self) -> Optional[str]:
return None
def get_setup(self) -> Optional[str]:
return None
def ssh(self, *args, **kwargs):
raise RuntimeError('ssh not implemented')

356
structure/structure.py
View File

@ -1,140 +1,8 @@
import ipaddress
import json
import textwrap
import threading
from enum import Enum
import random
import numpy as np
from typing import List, Optional, Union, Dict, Tuple
from typing import List, Optional
class IpMethod(Enum):
Manual = 0
Management = 1
Auto4 = 2
Auto6 = 3
Dhcp4 = 4
Dhcp6 = 5
class Interface:
def __init__(self, method: IpMethod, rate: Optional[int] = None):
self._method: IpMethod
self._node: Optional[Node] = None
self._rate: Optional[int] = None
self._bridge: Optional[Bridge] = None
self._method = method
self._rate = rate
self._address: ipaddress.ip_address = None
def get_method(self):
return self._method
def set_node(self, node):
self._node = node
def get_node(self):
return self._node
def set_bridge(self, bridge):
self._bridge = bridge
def get_bridge(self):
return self._bridge
def set_address(self, addr: ipaddress.ip_address):
self._address = addr
def get_address(self) -> ipaddress.ip_address:
return self._address
def get_rate(self) -> Optional[int]:
return self._rate
def set_rate(self, rate: Optional[int]):
self._rate = rate
class Bridge:
def __init__(self, *interfaces: Interface):
self._interfaces: List[Interface] = []
self._name: str = ''
for interface in interfaces:
self._interfaces.append(interface)
interface.set_bridge(self)
# Generate a random class c private range by default (10.0.0.0)
self._addr: ipaddress.ip_address = None
self._network_iterator: iter = None
self.netmask = 24
self.new_network()
def get_interfaces(self) -> List[Interface]:
return self._interfaces
def set_name(self, name: str):
self._name = name
def get_name(self) -> str:
return self._name
def set_netmask(self, mask: int):
self.netmask = mask
self._network_iterator = ipaddress.ip_network('{}/{}'.format(self._addr, self.netmask), False).hosts()
def get_ip_address(self) -> ipaddress.ip_address:
return next(self._network_iterator)
def get_network(self) -> ipaddress.ip_network:
return ipaddress.ip_network('{}/{}'.format(self._addr, self.netmask), False)
def get_network_string(self) -> str:
return str(ipaddress.ip_network('{}/{}'.format(self._addr, self.netmask), False))
def new_network(self):
self._addr: ipaddress.ip_address = ipaddress.ip_address('10.0.0.0') + random.randint(0, 16777216)
self._network_iterator = ipaddress.ip_network('{}/{}'.format(self._addr, self.netmask), False).hosts()
class Node:
def __init__(self, interfaces: List[Interface], setup_params: Dict = None):
self._id: Union[int, None] = None
self._interfaces: List[Interface] = interfaces
self._interfaces.append(Interface(IpMethod.Management))
for interface in self._interfaces:
interface.set_node(self)
self.setup_params = {} if setup_params is None else setup_params
def get_interfaces(self):
return self._interfaces
def set_id(self, new_id):
self._id = new_id
def get_id(self):
return self._id
def get_core_count(self) -> int:
return 2
def get_memory_mb(self) -> int:
return 2048
def get_internet_setup(self) -> Optional[str]:
return None
def get_setup(self) -> Optional[str]:
return None
def ssh(self, *args, **kwargs):
raise RuntimeError('ssh not implemented')
from structure import IpMethod, Interface, Bridge, Node
class SpeedTestServer(Node):
@ -343,223 +211,3 @@ class LocalPortal(Node):
remote_host=self.remote_portal.get_interfaces()[0].get_address(),
local_interface='eth{}'.format(len(self.get_interfaces()) - 2),
)
class StandardTest:
def __init__(
self,
rates: List[int],
events: Dict[float, Tuple[int, int]] = None,
duration: int = 10,
variation_target: float = 0.2,
max_failures: int = 3,
max_attempts: int = 60,
):
self.rates = rates
self.events = events if events is not None else dict()
self.duration = duration
self.variation_target = variation_target
self.max_failures = max_failures
self.max_attempts = max_attempts
def name(self) -> str:
name_builder = ['R{}-{}'.format(*y) for y in enumerate(self.rates)]
name_builder += ['E{}R{}-{}'.format(x, *y) for (x, y) in self.events.items()]
name_builder.append('T{}'.format(self.duration))
return ''.join(name_builder)
class DirectTest(StandardTest):
def __init__(self, rate: int, **kwargs):
super().__init__([rate], **kwargs)
def name(self) -> str:
return 'D{}'.format(super().name())
class StandardIperfResult:
def __init__(self, test: StandardTest, iperf: str, interval_size=2.0):
self.test = test
self.interval_size = interval_size
# list containing an exact time and a value
self.data: List[Tuple[float, float]] = []
self.num_tests = 0
self.add_results(iperf)
def add_results(self, iperf: str):
data = json.loads(iperf)
# grab the sum data of all non omitted intervals, excluding any that are smaller than expected
intervals = [
x['sum'] for x in data['intervals'] if
(not x['sum']['omitted']) and (x['sum']['end'] - x['sum']['start'] > self.interval_size / 2)
]
for (time, result) in zip(
[((x['start'] + x['end']) / 2) for x in intervals],
[x['bits_per_second'] for x in intervals],
):
self.data.append((time, result))
self.num_tests += 1
def bins(self) -> List[List[Tuple[float, float]]]:
bins: List[List[Tuple[float, float]]] = [[] for _ in np.arange(0, self.test.duration, self.interval_size)]
for time, result in self.data:
index = int(np.round((time - self.interval_size / 2) / self.interval_size))
bins[index].append((time, result))
return bins
def summarise(self) -> Dict[float, float]:
bins = self.bins()
means = [np.mean(x, axis=0)[1] for x in bins]
times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)]
return dict(zip(times, means))
def standard_deviation(self) -> Dict[float, float]:
bins = self.bins()
stds = [np.std(x, axis=0)[1] for x in bins]
times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)]
return dict(zip(times, stds))
def coefficient_variance(self) -> Dict[float, float]:
stds = self.standard_deviation()
means = self.summarise()
return {k: stds[k] / means[k] for k in stds.keys()}
def time_range(self) -> Dict[float, Tuple[float, float]]:
bins = self.bins()
times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)]
ranges = [(-np.min(x, axis=0)[0] + time, np.max(x, axis=0)[0] - time) for (x, time) in zip(bins, times)]
return dict(zip(times, ranges))
def repeat_until_satisfied(reducer, satisfied, initial=None, max_attempts=100, max_failures=3):
val = initial
for i in range(max_attempts):
for j in range(max_failures):
try:
val = reducer(val)
break
except Exception as e:
print('failed with {}'.format(e))
if j == max_failures - 1:
raise e
if satisfied(val):
return val
raise RuntimeError('too many attempts')
class BaseEnvironment:
def __init__(self, runner, top_level_bridge: Bridge):
self.top_level_bridge = top_level_bridge
self._runner = runner
def __enter__(self):
try:
self._runner.build(self.top_level_bridge)
except Exception as e:
self._runner.teardown()
raise e
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self._runner.teardown()
def _test(
self,
test: StandardTest,
inbound_server: SpeedTestServer,
inbound_client: SpeedTestServer,
rated_node: Node,
expected_interfaces: int,
) -> Tuple[StandardIperfResult, StandardIperfResult]:
if len(test.rates) != expected_interfaces:
raise RuntimeError('mismatched number of interfaces')
results = []
for server, client in [(inbound_server, inbound_client), (inbound_client, inbound_server)]:
def test_reducer(old: Optional[StandardIperfResult]) -> StandardIperfResult:
for i, r in enumerate(test.rates):
rated_node.get_interfaces()[i].set_rate(r)
server.server()
for t, (iface, rate) in test.events.items():
threading.Timer(
6 + t,
(lambda n: lambda: n.get_interfaces()[iface].set_rate(rate))(rated_node),
)
iperf = client.client(server, time=test.duration)
if old is None:
return StandardIperfResult(test, iperf)
else:
old.add_results(iperf)
return old
def test_satisfier(val: StandardIperfResult) -> bool:
if val.num_tests < 3:
return False
return False not in [x < test.variation_target for x in val.coefficient_variance().values()]
result = repeat_until_satisfied(
test_reducer,
test_satisfier,
max_failures=test.max_failures,
max_attempts=test.max_attempts,
)
results.append(result)
# Return a tuple of (inbound, outbound)
return results[0], results[1]
class StandardEnvironment(BaseEnvironment):
def __init__(self, interfaces: int, runner, setup_params: dict):
self._interfaces = interfaces
self.rp = RemotePortal([Interface(IpMethod.Auto4)], setup_params=setup_params)
self.st = SpeedTestServer()
self.cl = SpeedTestServer(clone_interface=self.rp.get_interfaces()[0])
self.lp = LocalPortal(
[Interface(IpMethod.Auto4) for _ in range(interfaces)],
self.cl,
setup_params=setup_params,
)
self.rp.set_local_portal(self.lp)
self.lp.set_remote_portal(self.rp)
super().__init__(runner, Bridge(
self.st.get_interfaces()[0],
self.rp.get_interfaces()[0],
*self.lp.get_interfaces()[0:interfaces],
))
def test(self, test: StandardTest) -> Tuple[StandardIperfResult, StandardIperfResult]:
return self._test(test, self.st, self.cl, self.lp, self._interfaces)
class DirectEnvironment(BaseEnvironment):
def __init__(self, runner):
self.st1 = SpeedTestServer()
self.st2 = SpeedTestServer()
super().__init__(runner, Bridge(
self.st1.get_interfaces()[0],
self.st2.get_interfaces()[0],
))
def test(self, test: StandardTest) -> Tuple[StandardIperfResult, StandardIperfResult]:
return self._test(test, self.st2, self.st1, self.st2, 1)

137
structure/tests.py Normal file
View File

@ -0,0 +1,137 @@
import json
from typing import List, Dict, Tuple
import numpy as np
def repeat_until_satisfied(reducer, satisfied, initial=None, max_attempts=100, max_failures=3):
val = initial
for i in range(max_attempts):
for j in range(max_failures):
try:
val = reducer(val)
break
except Exception as e:
print('failed with {}'.format(e))
if j == max_failures - 1:
raise e
if satisfied(val):
return val
raise RuntimeError('too many attempts')
class StandardTest:
def __init__(
self,
rates: List[int],
events: Dict[float, Tuple[int, int]] = None,
duration: int = 10,
interval_variation_target: float = np.inf,
bandwidth_variation_target: float = np.inf,
max_failures: int = 3,
max_attempts: int = 60,
):
self.rates = rates
self.events = events if events is not None else dict()
self.duration = duration
self.interval_variation_target = interval_variation_target
self.bandwidth_variation_target = bandwidth_variation_target
self.max_failures = max_failures
self.max_attempts = max_attempts
def name(self) -> str:
name_builder = ['R{}-{}'.format(*y) for y in enumerate(self.rates)]
name_builder += ['E{}R{}-{}'.format(x, *y) for (x, y) in self.events.items()]
name_builder.append('T{}'.format(self.duration))
return ''.join(name_builder)
class DirectTest(StandardTest):
def __init__(self, rate: int, **kwargs):
super().__init__([rate], **kwargs)
def name(self) -> str:
return 'D{}'.format(super().name())
class IperfResult:
def __init__(self, test: StandardTest, iperf: str, interval_size=2.0):
self.test = test
self.interval_size = interval_size
# list containing an exact time and a value
self.interval_data: List[Tuple[float, float]] = []
# list containing the overall data transferred and the time taken
self.bandwidth_data: List[float] = []
self.num_tests = 0
self.add_results(iperf)
def add_results(self, iperf: str):
data = json.loads(iperf)
# grab the overall bandwidth
self.bandwidth_data.append(data['end']['sum_sent']['bits_per_second'])
# grab the sum data of all non omitted intervals, excluding any that are smaller than expected
intervals = [
x['sum'] for x in data['intervals'] if
(not x['sum']['omitted']) and (x['sum']['end'] - x['sum']['start'] > self.interval_size / 2)
]
for (time, result) in zip(
[((x['start'] + x['end']) / 2) for x in intervals],
[x['bits_per_second'] for x in intervals],
):
self.interval_data.append((time, result))
self.num_tests += 1
def bins(self) -> List[List[Tuple[float, float]]]:
bins: List[List[Tuple[float, float]]] = [[] for _ in np.arange(0, self.test.duration, self.interval_size)]
for time, result in self.interval_data:
index = int(np.round((time - self.interval_size / 2) / self.interval_size))
bins[index].append((time, result))
return bins
def interval_means(self) -> Dict[float, float]:
bins = self.bins()
means = [np.mean(x, axis=0)[1] for x in bins]
times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)]
return dict(zip(times, means))
def interval_standard_deviations(self) -> Dict[float, float]:
bins = self.bins()
stds = [np.std(x, axis=0)[1] for x in bins]
times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)]
return dict(zip(times, stds))
def interval_coefficient_variances(self) -> Dict[float, float]:
stds = self.interval_standard_deviations()
means = self.interval_means()
return {k: stds[k] / means[k] for k in stds.keys()}
def interval_time_ranges(self) -> Dict[float, Tuple[float, float]]:
bins = self.bins()
times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)]
ranges = [(-np.min(x, axis=0)[0] + time, np.max(x, axis=0)[0] - time) for (x, time) in zip(bins, times)]
return dict(zip(times, ranges))
def bandwidth_mean(self):
return np.mean(self.bandwidth_data)
def bandwidth_standard_deviation(self):
return np.std(self.bandwidth_data)
def bandwidth_coefficient_variance(self):
return self.bandwidth_standard_deviation() / self.bandwidth_mean()