JakeHillion/dissertation-3-evaluation
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

improved testing

Browse Source
This commit is contained in:
Jake Hillion 2020-11-15 00:48:44 +00:00
parent 3e777bde92
commit 135d2e8cec
3 changed files with 189 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
graphs/
# Created by https://www.toptal.com/developers/gitignore/api/python,intellij+all,dotenv
# Edit at https://www.toptal.com/developers/gitignore?templates=python,intellij+all,dotenv

202
evaluation.ipynb
View File

@ -87,7 +87,21 @@
"directionOutbound: Dict[str, StandardIperfResult] = {}\n",
"\n",
"def run_and_save_test(env: StandardEnvironment, test: StandardTest):\n",
" (directionInbound[test.name()], directionOutbound[test.name()]) = env.test(test)"
" (directionInbound[test.name()], directionOutbound[test.name()]) = env.test(test)\n",
"\n",
"def attempt_n_times(foo, n=3):\n",
" i = 0\n",
" while True:\n",
" try:\n",
" return foo()\n",
" except Exception as e:\n",
" if i == n:\n",
" raise e\n",
" finally:\n",
" i += 1\n",
"\n",
"\n",
"fast_tests = True"
]
},
{
@ -124,11 +138,19 @@
},
"outputs": [],
"source": [
"with StandardEnvironment(2, runner, setup_params) as env:\n",
" run_and_save_test(env, StandardTest([1,1]))\n",
" run_and_save_test(env, StandardTest([1,2]))\n",
" run_and_save_test(env, StandardTest([2,2]))\n",
" run_and_save_test(env, StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30))\n"
"def two_interface_tests():\n",
" with StandardEnvironment(2, runner, setup_params) as env:\n",
" run_and_save_test(env, StandardTest([1,1], variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, StandardTest([1,2], variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, StandardTest([2,2], variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, StandardTest(\n",
" [2,2],\n",
" events={10: (0,1), 15: (0,2)},\n",
" duration=30,\n",
" variation_target=0.8 if fast_tests else 0.5,\n",
" ))\n",
"\n",
"attempt_n_times(two_interface_tests)"
]
},
{
@ -150,9 +172,12 @@
},
"outputs": [],
"source": [
"with StandardEnvironment(3, runner, setup_params) as env:\n",
" run_and_save_test(env, StandardTest([1,1,1]))\n",
" run_and_save_test(env, StandardTest([2,2,2]))\n"
"def three_interface_tests():\n",
" with StandardEnvironment(3, runner, setup_params) as env:\n",
" run_and_save_test(env, StandardTest([1,1,1], variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, StandardTest([2,2,2], variation_target=0.4 if fast_tests else 0.2))\n",
"\n",
"attempt_n_times(three_interface_tests)"
]
},
{
@ -174,9 +199,12 @@
},
"outputs": [],
"source": [
"with StandardEnvironment(4, runner, setup_params) as env:\n",
" run_and_save_test(env, StandardTest([1,1,1,1]))\n",
" run_and_save_test(env, StandardTest([2,2,2,2]))\n"
"def four_interface_tests():\n",
" with StandardEnvironment(4, runner, setup_params) as env:\n",
" run_and_save_test(env, StandardTest([1,1,1,1], variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, StandardTest([2,2,2,2], variation_target=0.4 if fast_tests else 0.2))\n",
"\n",
"attempt_n_times(four_interface_tests)"
]
},
{
@ -203,12 +231,26 @@
"def plot_iperf_results(\n",
" series: Dict[str, StandardTest],\n",
" title: str = None,\n",
" direction = 'inbound',\n",
" direction = 'both',\n",
" error_bars_x=False,\n",
" error_bars_y=False,\n",
" error_bars_y=True,\n",
" filename=None,\n",
" start_at_zero=True,\n",
"):\n",
" if direction == 'both':\n",
" plot_iperf_results(series, title, 'outbound', error_bars_x, error_bars_y, filename, start_at_zero)\n",
" plot_iperf_results(series, title, 'inbound', error_bars_x, error_bars_y, filename, start_at_zero)\n",
" return\n",
"\n",
" if filename in ['png', 'eps']:\n",
" filename = 'graphs/{}{}{}{}.{}'.format(\n",
" 'I' if direction == 'inbound' else 'O',\n",
" 'Ex' if error_bars_x else '',\n",
" 'Ey' if error_bars_y else '',\n",
" ''.join(['S{}-{}'.format(i,x.name()) for (i, x) in enumerate(series.values())]),\n",
" filename,\n",
" )\n",
"\n",
" series = {\n",
" k: (directionInbound if direction == 'inbound' else directionOutbound)[v.name()] for (k, v) in series.items()\n",
" }\n",
@ -249,20 +291,63 @@
" axes.axvline(k, linestyle='--', color='grey')\n",
" axes.annotate(v, (k, 1.02), xycoords=axes.get_xaxis_transform(), ha='center')\n",
"\n",
" if fast_tests:\n",
" fig.text(0.95, 0.05, 'Draft', fontsize=50, color='gray', ha='right', va='bottom', alpha=0.5)\n",
"\n",
" if filename is not None:\n",
" fig.savefig(filename, bbox_extra_artists=(legend,), bbox_inches='tight', pad_inches=0.3)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"if not os.path.exists('graphs/'):\n",
" os.makedirs('graphs/')\n",
"\n",
"for filename in os.listdir('graphs'):\n",
" file_path = os.path.join('graphs/', filename)\n",
" os.unlink(file_path)"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
},
"source": [
"### Equal Connection Scaling\n",
"This section shows equal connections scaling at various speeds and number of connections."
"### Error Bar Evaluation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n",
" '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" },\n",
" error_bars_x=False,\n",
" error_bars_y=False,\n",
" filename='png',\n",
")"
]
},
{
@ -281,9 +366,41 @@
" '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" },\n",
" 'Scaling of Equal Connections',\n",
" error_bars_x=True,\n",
" error_bars_y=True,\n",
" error_bars_y=False,\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
},
"source": [
"### More than 2 connections evaluation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n",
" '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" },\n",
" filename='png',\n",
")"
]
},
@ -303,12 +420,54 @@
" '3x2MB Connections (proxied)': StandardTest([2,2,2]),\n",
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
" },\n",
" 'Scaling of Equal Connections',\n",
" error_bars_x=True,\n",
" error_bars_y=True,\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
},
"source": [
"### Mixed Performance Evaluation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
" '1x1MB + 1x2MB Connections (proxied)': StandardTest([1,2]),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
},
"source": [
"### Eventful Evaluation"
]
},
{
"cell_type": "code",
"execution_count": null,
@ -324,6 +483,7 @@
" 'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n",
" },\n",
" error_bars_y=True,\n",
" filename='png',\n",
")"
]
},

23
structure/structure.py
View File

@ -68,6 +68,9 @@ class Bridge:
interface.set_bridge(self)
# Generate a random class c private range by default (10.0.0.0)
self._addr: ipaddress.ip_address = None
self._network_iterator: iter = None
self.netmask = 24
self.new_network()
@ -185,13 +188,7 @@ class RemotePortal(Node):
return textwrap.dedent('''
set -e
wget -q http://10.20.0.11/minio-client
chmod +x minio-client
./minio-client alias set s3 http://10.20.0.25:3900 {access_key} {secret_key} || \
./minio-client alias set s3 s3.us-west-001.backblazeb2.com {access_key} {secret_key}
./minio-client cp s3/dissertation/binaries/debian/{branch} mpbl3p
wget -q https://f001.backblazeb2.com/file/dissertation/binaries/debian/{branch} -O mpbl3p
chmod +x mpbl3p
cloud-init status --wait || cloud-init status --long
@ -258,14 +255,8 @@ class LocalPortal(Node):
def get_internet_setup(self) -> Optional[str]:
return textwrap.dedent('''
set -e
wget -q http://10.20.0.11/minio-client
chmod +x minio-client
./minio-client alias set s3 http://10.20.0.25:3900 {access_key} {secret_key} || \
./minio-client alias set s3 s3.us-west-001.backblazeb2.com {access_key} {secret_key}
./minio-client cp s3/dissertation/binaries/debian/{branch} mpbl3p
wget -q https://f001.backblazeb2.com/file/dissertation/binaries/debian/{branch} -O mpbl3p
chmod +x mpbl3p
cloud-init status --wait || cloud-init status --long
@ -354,7 +345,7 @@ class StandardTest:
rates: List[int],
events: Dict[float, Tuple[int, int]] = None,
duration: int = 10,
variation_target: float = 0.3,
variation_target: float = 0.2,
max_failures: int = 3,
max_attempts: int = 60,
):
@ -369,7 +360,7 @@ class StandardTest:
def name(self) -> str:
name_builder = ['R{}-{}'.format(*y) for y in enumerate(self.rates)]
name_builder += ['E{}R{}-{}'.format(x, *y) for (x, y) in self.events.items()]
name_builder.append('D{}'.format(self.duration))
name_builder.append('T{}'.format(self.duration))
return ''.join(name_builder)