diff --git a/.gitignore b/.gitignore index 00491ff..e64f433 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -graphs/ +output/ # Created by https://www.toptal.com/developers/gitignore/api/python,intellij+all,dotenv # Edit at https://www.toptal.com/developers/gitignore?templates=python,intellij+all,dotenv diff --git a/evaluation.ipynb b/evaluation.ipynb index 858fdb9..e08da75 100644 --- a/evaluation.ipynb +++ b/evaluation.ipynb @@ -2,7 +2,9 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "collapsed": false + }, "source": [ "# Project Evaluation\n", "\n", @@ -13,9 +15,7 @@ { "cell_type": "markdown", "metadata": { - "pycharm": { - "name": "#%% md\n" - } + "collapsed": false }, "source": [ "## Setup\n", @@ -25,16 +25,20 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [], "source": [ "import os\n", "import ipaddress\n", - "from typing import Dict\n", "\n", "import runners\n", - "from structure import StandardEnvironment, StandardTest, StandardIperfResult\n", - "from structure.structure import DirectEnvironment, DirectTest\n", + "from structure import StandardEnvironment, StandardTest\n", + "from structure import DirectEnvironment, DirectTest\n", + "from structure import BaseEnvironment\n", "\n", "%load_ext dotenv\n", "%dotenv" @@ -43,9 +47,7 @@ { "cell_type": "markdown", "metadata": { - "pycharm": { - "name": "#%% md\n" - } + "collapsed": false }, "source": [ "## Testing\n", @@ -82,13 +84,27 @@ " 'access_key': os.getenv('S3_ACCESS_KEY'),\n", " 'secret_key': os.getenv('S3_SECRET_KEY'),\n", " 'branch': os.getenv('TARGET_BRANCH'),\n", - "}\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "from graphs import TestStore\n", "\n", - "directionInbound: Dict[str, StandardIperfResult] = {}\n", - "directionOutbound: Dict[str, StandardIperfResult] = {}\n", + "test_store = TestStore()\n", "\n", - "def run_and_save_test(env: StandardEnvironment, test: StandardTest):\n", - " (directionInbound[test.name()], directionOutbound[test.name()]) = env.test(test)\n", + "def run_and_save_test(env: BaseEnvironment, test: StandardTest):\n", + " (inbound, outbound) = env.test(test)\n", + " test_store.save_inbound(test, inbound)\n", + " test_store.save_outbound(test, outbound)\n", "\n", "def attempt_n_times(foo, n=3):\n", " for i in range(n):\n", @@ -107,10 +123,7 @@ { "cell_type": "markdown", "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } + "collapsed": false }, "source": [ "### Direct Server to Server" @@ -128,10 +141,10 @@ "source": [ "def direct_tests():\n", " with DirectEnvironment(runner) as env:\n", - " run_and_save_test(env, DirectTest(1, variation_target=0.4 if fast_tests else 0.2))\n", - " run_and_save_test(env, DirectTest(2, variation_target=0.4 if fast_tests else 0.2))\n", - " run_and_save_test(env, DirectTest(3, variation_target=0.4 if fast_tests else 0.2))\n", - " run_and_save_test(env, DirectTest(4, variation_target=0.4 if fast_tests else 0.2))\n", + " run_and_save_test(env, DirectTest(1, bandwidth_variation_target=0.2 if fast_tests else 0.05))\n", + " run_and_save_test(env, DirectTest(2, bandwidth_variation_target=0.2 if fast_tests else 0.05))\n", + " run_and_save_test(env, DirectTest(3, bandwidth_variation_target=0.2 if fast_tests else 0.05))\n", + " run_and_save_test(env, DirectTest(4, bandwidth_variation_target=0.2 if fast_tests else 0.05))\n", "\n", "attempt_n_times(direct_tests)" ] @@ -139,10 +152,7 @@ { "cell_type": "markdown", "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } + "collapsed": false }, "source": [ "### Local Portal with 2 Interfaces" @@ -160,14 +170,14 @@ "source": [ "def two_interface_tests():\n", " with StandardEnvironment(2, runner, setup_params) as env:\n", - " run_and_save_test(env, StandardTest([1,1], variation_target=0.4 if fast_tests else 0.2))\n", - " run_and_save_test(env, StandardTest([1,2], variation_target=0.4 if fast_tests else 0.2))\n", - " run_and_save_test(env, StandardTest([2,2], variation_target=0.4 if fast_tests else 0.2))\n", + " run_and_save_test(env, StandardTest([1,1], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n", + " run_and_save_test(env, StandardTest([1,2], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n", + " run_and_save_test(env, StandardTest([2,2], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n", " run_and_save_test(env, StandardTest(\n", " [2,2],\n", - " events={10: (0,1), 15: (0,2)},\n", + " events={10: (0,1), 20: (0,2)},\n", " duration=30,\n", - " variation_target=0.8 if fast_tests else 0.5,\n", + " interval_variation_target=0.8 if fast_tests else 0.5,\n", " ))\n", "\n", "attempt_n_times(two_interface_tests)" @@ -194,8 +204,8 @@ "source": [ "def three_interface_tests():\n", " with StandardEnvironment(3, runner, setup_params) as env:\n", - " run_and_save_test(env, StandardTest([1,1,1], variation_target=0.4 if fast_tests else 0.2))\n", - " run_and_save_test(env, StandardTest([2,2,2], variation_target=0.4 if fast_tests else 0.2))\n", + " run_and_save_test(env, StandardTest([1,1,1], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n", + " run_and_save_test(env, StandardTest([2,2,2], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n", "\n", "attempt_n_times(three_interface_tests)" ] @@ -221,8 +231,8 @@ "source": [ "def four_interface_tests():\n", " with StandardEnvironment(4, runner, setup_params) as env:\n", - " run_and_save_test(env, StandardTest([1,1,1,1], variation_target=0.4 if fast_tests else 0.2))\n", - " run_and_save_test(env, StandardTest([2,2,2,2], variation_target=0.4 if fast_tests else 0.2))\n", + " run_and_save_test(env, StandardTest([1,1,1,1], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n", + " run_and_save_test(env, StandardTest([2,2,2,2], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n", "\n", "attempt_n_times(four_interface_tests)" ] @@ -230,9 +240,7 @@ { "cell_type": "markdown", "metadata": { - "pycharm": { - "name": "#%% md\n" - } + "collapsed": false }, "source": [ "## Graphs\n", @@ -249,107 +257,24 @@ }, "outputs": [], "source": [ - "from itertools import cycle\n", - "import matplotlib.pyplot as plt\n", + "from graphs import plot_iperf_results, plot_iperf_results_time\n", "\n", - "def plot_iperf_results(\n", - " series: Dict[str, StandardTest],\n", - " title: str = None,\n", - " direction = 'both',\n", - " error_bars_x=False,\n", - " error_bars_y=True,\n", - " filename=None,\n", - " start_at_zero=True,\n", - "):\n", - " if direction == 'both':\n", - " plot_iperf_results(series, title, 'outbound', error_bars_x, error_bars_y, filename, start_at_zero)\n", - " plot_iperf_results(series, title, 'inbound', error_bars_x, error_bars_y, filename, start_at_zero)\n", - " return\n", + "if not os.path.exists('output/'):\n", + " os.makedirs('output/')\n", "\n", - " if filename in ['png', 'eps']:\n", - " filename = 'graphs/{}{}{}{}.{}'.format(\n", - " 'I' if direction == 'inbound' else 'O',\n", - " 'Ex' if error_bars_x else '',\n", - " 'Ey' if error_bars_y else '',\n", - " ''.join(['S{}-{}'.format(i,x.name()) for (i, x) in enumerate(series.values())]),\n", - " filename,\n", - " )\n", - "\n", - " series = {\n", - " k: (directionInbound if direction == 'inbound' else directionOutbound)[v.name()] for (k, v) in series.items()\n", - " }\n", - "\n", - " cycol = cycle('brgy')\n", - "\n", - " fig = plt.figure()\n", - " axes = fig.add_axes([0,0,1,1])\n", - "\n", - " if title is not None:\n", - " axes.set_title(title, pad=20.0 if True in [len(x.test.events) > 0 for x in series.values()] else None)\n", - "\n", - " axes.set_xlabel('Time (s)')\n", - " axes.set_ylabel('Throughput (Mbps)')\n", - "\n", - " for k, v in series.items():\n", - " data = v.summarise()\n", - "\n", - " axes.errorbar(\n", - " data.keys(),\n", - " [x/1e6 for x in data.values()],\n", - " xerr=([x[0] for x in v.time_range().values()], [x[1] for x in v.time_range().values()]) if error_bars_x else None,\n", - " yerr=[x*1.5/1e6 for x in v.standard_deviation().values()] if error_bars_y else None,\n", - " capsize=3,\n", - " ecolor='grey',\n", - " color=next(cycol),\n", - " label=k,\n", - " )\n", - "\n", - " legend = axes.legend()\n", - "\n", - " if start_at_zero:\n", - " axes.set_ylim(bottom=0)\n", - " axes.set_xlim(left=0)\n", - "\n", - " if False:\n", - " for k, v in events.items():\n", - " axes.axvline(k, linestyle='--', color='grey')\n", - " axes.annotate(v, (k, 1.02), xycoords=axes.get_xaxis_transform(), ha='center')\n", - "\n", - " if fast_tests:\n", - " fig.text(0.95, 0.05, 'Draft', fontsize=50, color='gray', ha='right', va='bottom', alpha=0.5)\n", - "\n", - " if filename is not None:\n", - " fig.savefig(filename, bbox_extra_artists=(legend,), bbox_inches='tight', pad_inches=0.3)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "if not os.path.exists('graphs/'):\n", - " os.makedirs('graphs/')\n", - "\n", - "for filename in os.listdir('graphs'):\n", - " file_path = os.path.join('graphs/', filename)\n", + "for filename in os.listdir('output'):\n", + " file_path = os.path.join('output/', filename)\n", " os.unlink(file_path)" ] }, { "cell_type": "markdown", "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } + "collapsed": false }, "source": [ - "### Error Bar Evaluation" + "### Section 4.2: Graphs\n", + "#### Subsection 4.2.2 Line Graphs" ] }, { @@ -362,12 +287,13 @@ }, "outputs": [], "source": [ - "plot_iperf_results(\n", + "plot_iperf_results_time(test_store,\n", " {\n", " '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n", " '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n", " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", " },\n", + " fast_tests,\n", " error_bars_x=False,\n", " error_bars_y=False,\n", " filename='png',\n", @@ -384,27 +310,19 @@ }, "outputs": [], "source": [ - "plot_iperf_results(\n", + "plot_iperf_results_time(test_store,\n", " {\n", " '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n", " '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n", " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", " },\n", + " fast_tests,\n", " error_bars_x=True,\n", " error_bars_y=False,\n", " filename='png',\n", ")" ] }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": false - }, - "source": [ - "### More than 2 connections evaluation" - ] - }, { "cell_type": "code", "execution_count": null, @@ -415,16 +333,29 @@ }, "outputs": [], "source": [ - "plot_iperf_results(\n", + "plot_iperf_results_time(test_store,\n", " {\n", " '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n", " '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n", " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", " },\n", + " fast_tests,\n", + " error_bars_x=False,\n", + " error_bars_y=True,\n", " filename='png',\n", ")" ] }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false + }, + "source": [ + "### Section 4.3: Success Criteria\n", + "#### Subsection 4.3.2: Bidirectional Performance Gains" + ] + }, { "cell_type": "code", "execution_count": null, @@ -435,12 +366,13 @@ }, "outputs": [], "source": [ - "plot_iperf_results(\n", + "plot_iperf_results(test_store,\n", " {\n", - " '4x2MB Connections (proxied)': StandardTest([2,2,2,2]),\n", - " '3x2MB Connections (proxied)': StandardTest([2,2,2]),\n", + " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", + " '1x1MB + 1x2MB\\nConnections (proxied)': StandardTest([1,2]),\n", " '2x2MB Connections (proxied)': StandardTest([2,2]),\n", " },\n", + " fast_tests,\n", " filename='png',\n", ")" ] @@ -451,7 +383,7 @@ "collapsed": false }, "source": [ - "### Mixed Performance Evaluation" + "#### Subsection 4.3.5: More Bandwidth over Two Equal Connections" ] }, { @@ -464,137 +396,14 @@ }, "outputs": [], "source": [ - "plot_iperf_results(\n", + "plot_iperf_results(test_store,\n", " {\n", - " '2x2MB Connections (proxied)': StandardTest([2,2]),\n", - " '1x1MB + 1x2MB Connections (proxied)': StandardTest([1,2]),\n", - " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", - " },\n", - " filename='png',\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": false - }, - "source": [ - "### Eventful Evaluation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "plot_iperf_results(\n", - " {\n", - " 'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n", - " },\n", - " filename='png',\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": false - }, - "source": [ - "### Comparisons to a Direct Connection" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "plot_iperf_results(\n", - " {\n", - " '1x2MB Connection (not proxied)': DirectTest(2),\n", - " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", - " '1x1MB Connection (not proxied)': DirectTest(1),\n", + " '1x1MB Connection\\n(direct)': DirectTest(1),\n", + " '2x1MB Connections\\n(proxied)': StandardTest([1,1]),\n", + " '1x2MB Connection\\n(direct)': DirectTest(2),\n", "\n", " },\n", - " filename='png',\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "plot_iperf_results(\n", - " {\n", - " '1x4MB Connection (not proxied)': DirectTest(4),\n", - " '2x2MB Connections (proxied)': StandardTest([2,2]),\n", - " '1x2MB Connection (not proxied)': DirectTest(2),\n", - "\n", - " },\n", - " filename='png',\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": false - }, - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "plot_iperf_results(\n", - " {\n", - " '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n", - " '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n", - " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", - " },\n", - " filename='png',\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "plot_iperf_results(\n", - " {\n", - " '4x2MB Connections (proxied)': StandardTest([2,2,2,2]),\n", - " '3x2MB Connections (proxied)': StandardTest([2,2,2]),\n", - " '2x2MB Connections (proxied)': StandardTest([2,2]),\n", - " },\n", + " fast_tests,\n", " filename='png',\n", ")" ] @@ -605,7 +414,8 @@ "collapsed": false }, "source": [ - "### Mixed Performance Evaluation" + "### Section 4.4: Extended Goals\n", + "#### Subsection 4.4.1: More Bandwidth over Unequal Connections" ] }, { @@ -618,12 +428,13 @@ }, "outputs": [], "source": [ - "plot_iperf_results(\n", + "plot_iperf_results(test_store,\n", " {\n", - " '2x2MB Connections (proxied)': StandardTest([2,2]),\n", - " '1x1MB + 1x2MB Connections (proxied)': StandardTest([1,2]),\n", - " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", + " '2x2MB Connections\\n(proxied)': StandardTest([2,2]),\n", + " '1x1MB + 1x2MB\\nConnections (proxied)': StandardTest([1,2]),\n", + " '2x1MB Connections\\n(proxied)': StandardTest([1,1]),\n", " },\n", + " fast_tests,\n", " filename='png',\n", ")" ] @@ -634,7 +445,7 @@ "collapsed": false }, "source": [ - "### Eventful Evaluation" + "#### Subsection 4.4.2: More Bandwidth over Four Equal Connections" ] }, { @@ -647,10 +458,34 @@ }, "outputs": [], "source": [ - "plot_iperf_results(\n", + "plot_iperf_results(test_store,\n", " {\n", - " 'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n", + " '4x1MB Connections\\n(proxied)': StandardTest([1,1,1,1]),\n", + " '3x1MB Connections\\n(proxied)': StandardTest([1,1,1]),\n", + " '2x1MB Connections\\n(proxied)': StandardTest([1,1]),\n", " },\n", + " fast_tests,\n", + " filename='png',\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plot_iperf_results(test_store,\n", + " {\n", + " '4x2MB Connections\\n(proxied)': StandardTest([2,2,2,2]),\n", + " '3x2MB Connections\\n(proxied)': StandardTest([2,2,2]),\n", + " '2x2MB Connections\\n(proxied)': StandardTest([2,2]),\n", + " },\n", + " fast_tests,\n", " filename='png',\n", ")" ] @@ -661,7 +496,7 @@ "collapsed": false }, "source": [ - "### Comparisons to a Direct Connection" + "#### Subsection 4.4.3: Bandwidth Variation" ] }, { @@ -674,200 +509,11 @@ }, "outputs": [], "source": [ - "plot_iperf_results(\n", + "plot_iperf_results_time(test_store,\n", " {\n", - " '1x2MB Connection (not proxied)': DirectTest(2),\n", - " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", - " '1x1MB Connection (not proxied)': DirectTest(1),\n", - "\n", - " },\n", - " filename='png',\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "plot_iperf_results(\n", - " {\n", - " '1x4MB Connection (not proxied)': DirectTest(4),\n", - " '2x2MB Connections (proxied)': StandardTest([2,2]),\n", - " '1x2MB Connection (not proxied)': DirectTest(2),\n", - "\n", - " },\n", - " filename='png',\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "plot_iperf_results(\n", - " {\n", - " '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n", - " '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n", - " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", - " },\n", - " filename='png',\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "plot_iperf_results(\n", - " {\n", - " '4x2MB Connections (proxied)': StandardTest([2,2,2,2]),\n", - " '3x2MB Connections (proxied)': StandardTest([2,2,2]),\n", - " '2x2MB Connections (proxied)': StandardTest([2,2]),\n", - " },\n", - " filename='png',\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Mixed Performance Evaluation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "plot_iperf_results(\n", - " {\n", - " '2x2MB Connections (proxied)': StandardTest([2,2]),\n", - " '1x1MB + 1x2MB Connections (proxied)': StandardTest([1,2]),\n", - " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", - " },\n", - " filename='png',\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Eventful Evaluation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "plot_iperf_results(\n", - " {\n", - " 'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n", - " },\n", - " filename='png',\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Comparisons to a Direct Connection" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "plot_iperf_results(\n", - " {\n", - " '1x2MB Connection (not proxied)': DirectTest(2),\n", - " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", - " '1x1MB Connection (not proxied)': DirectTest(1),\n", - "\n", - " },\n", - " filename='png',\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "plot_iperf_results(\n", - " {\n", - " '1x4MB Connection (not proxied)': DirectTest(4),\n", - " '2x2MB Connections (proxied)': StandardTest([2,2]),\n", - " '1x2MB Connection (not proxied)': DirectTest(2),\n", - "\n", + " 'Varied Connection': StandardTest([2,2], events={10: (0,1), 20: (0,2)}, duration=30),\n", " },\n", + " fast_tests,\n", " filename='png',\n", ")" ] @@ -900,7 +546,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.6" + "version": "3.9.0" } }, "nbformat": 4, diff --git a/graphs/__init__.py b/graphs/__init__.py new file mode 100644 index 0000000..2432ed8 --- /dev/null +++ b/graphs/__init__.py @@ -0,0 +1,2 @@ +from .store import TestStore +from .graphs import plot_iperf_results, plot_iperf_results_time diff --git a/graphs/graphs.py b/graphs/graphs.py new file mode 100644 index 0000000..e9fe7af --- /dev/null +++ b/graphs/graphs.py @@ -0,0 +1,132 @@ +from itertools import cycle +from typing import Dict, Union + +from matplotlib import pyplot as plt + +from graphs import TestStore +from structure import StandardTest + + +def plot_iperf_results( + store: TestStore, + series: Dict[str, StandardTest], + fast_tests: bool, + title: str = None, + direction='both', + error_bars=True, + filename=None, +): + if direction == 'both': + plot_iperf_results(store, series, fast_tests, title, 'inbound', error_bars, filename) + plot_iperf_results(store, series, fast_tests, title, 'outbound', error_bars, filename) + return + + if filename in ['png', 'eps']: + filename = 'output/{}{}{}.{}'.format( + 'I' if direction == 'inbound' else 'O', + 'E' if error_bars else '', + ''.join(['S{}-{}'.format(i, x.name()) for (i, x) in enumerate(series.values())]), + filename, + ) + print(filename) + + series = { + k: (store.get_inbound(v) if direction == 'inbound' else store.get_outbound(v)) + for (k, v) in series.items() + } + + fig = plt.figure() + axes = fig.add_axes([0, 0, 1, 1]) + + if title is not None: + axes.set_title(title, pad=20.0 if True in [len(x.test.events) > 0 for x in series.values()] else None) + + axes.set_ylabel('Throughput (Mbps)') + + for k, v in series.items(): + axes.bar( + k, + v.bandwidth_mean() / 1e6, + yerr=1.5 * v.bandwidth_standard_deviation() / 1e6, + width=0.4, + capsize=15, + ) + + if fast_tests: + fig.text(0.95, 0.05, 'Draft', fontsize=50, color='gray', ha='right', va='bottom', alpha=0.5) + + if filename is not None: + fig.savefig(filename, bbox_inches='tight', pad_inches=0.3) + + +def plot_iperf_results_time( + store: TestStore, + series: Dict[str, StandardTest], + fast_tests: bool, + title: str = None, + direction='both', + error_bars_x=False, + error_bars_y=True, + filename=None, + start_at_zero=True, +): + if direction == 'both': + plot_iperf_results_time(store, series, fast_tests, title, 'outbound', error_bars_x, error_bars_y, filename, + start_at_zero) + plot_iperf_results_time(store, series, fast_tests, title, 'inbound', error_bars_x, error_bars_y, filename, + start_at_zero) + return + + if filename in ['png', 'eps']: + filename = 'output/T{}{}{}{}.{}'.format( + 'I' if direction == 'inbound' else 'O', + 'Ex' if error_bars_x else '', + 'Ey' if error_bars_y else '', + ''.join(['S{}-{}'.format(i, x.name()) for (i, x) in enumerate(series.values())]), + filename, + ) + print(filename) + + series = { + k: (store.get_inbound(v) if direction == 'inbound' else store.get_outbound(v)) + for (k, v) in series.items() + } + + cycol = cycle('brgy') + + fig = plt.figure() + axes = fig.add_axes([0, 0, 1, 1]) + + if title is not None: + axes.set_title(title, pad=20.0 if True in [len(x.test.events) > 0 for x in series.values()] else None) + + axes.set_xlabel('Time (s)') + axes.set_ylabel('Throughput (Mbps)') + + for k, v in series.items(): + data = v.interval_means() + + axes.errorbar( + data.keys(), + [x / 1e6 for x in data.values()], + xerr=( + [x[0] for x in v.interval_time_ranges().values()], + [x[1] for x in v.interval_time_ranges().values()]) if error_bars_x else None, + yerr=[x * 1.5 / 1e6 for x in v.interval_standard_deviations().values()] if error_bars_y else None, + capsize=3, + ecolor='grey', + color=next(cycol), + label=k, + ) + + legend = axes.legend() + + if start_at_zero: + axes.set_ylim(bottom=0) + axes.set_xlim(left=0) + + if fast_tests: + fig.text(0.95, 0.05, 'Draft', fontsize=50, color='gray', ha='right', va='bottom', alpha=0.5) + + if filename is not None: + fig.savefig(filename, bbox_extra_artists=(legend,), bbox_inches='tight', pad_inches=0.3) diff --git a/graphs/store.py b/graphs/store.py new file mode 100644 index 0000000..9e501bd --- /dev/null +++ b/graphs/store.py @@ -0,0 +1,21 @@ +from typing import Dict + +from structure import IperfResult, StandardTest + + +class TestStore: + def __init__(self): + self.inbound: Dict[str, IperfResult] = dict() + self.outbound: Dict[str, IperfResult] = dict() + + def save_inbound(self, test: StandardTest, result: IperfResult): + self.inbound[test.name()] = result + + def save_outbound(self, test: StandardTest, result: IperfResult): + self.outbound[test.name()] = result + + def get_inbound(self, test: StandardTest) -> IperfResult: + return self.inbound[test.name()] + + def get_outbound(self, test: StandardTest) -> IperfResult: + return self.outbound[test.name()] diff --git a/runners/runners.py b/runners/runners.py index 0ed4ecf..ea56808 100644 --- a/runners/runners.py +++ b/runners/runners.py @@ -11,6 +11,7 @@ import proxmoxer import paramiko import structure +import structure.generic def check_env(*names: str) -> bool: @@ -21,14 +22,14 @@ def check_env(*names: str) -> bool: def bridge_node_search( - first: structure.Bridge, - bridge_name_generator: Callable[[structure.Bridge], str], - node_id_generator: Callable[[structure.Node], int], -) -> Tuple[List[structure.Bridge], List[structure.Node]]: - bridges: List[structure.Bridge] = [] - nodes: List[structure.Node] = [] + first: structure.generic.Bridge, + bridge_name_generator: Callable[[structure.generic.Bridge], str], + node_id_generator: Callable[[structure.generic.Node], int], +) -> Tuple[List[structure.generic.Bridge], List[structure.generic.Node]]: + bridges: List[structure.generic.Bridge] = [] + nodes: List[structure.generic.Node] = [] - queue: List[structure.Bridge] = [first] + queue: List[structure.generic.Bridge] = [first] while len(queue) > 0: bridge = queue.pop() if bridge.get_name() != '': @@ -38,7 +39,7 @@ def bridge_node_search( bridge.set_name(bridge_name_generator(bridge)) # from this bridge, find all nodes (via all interfaces) - reachable_nodes: List[structure.Node] = [] + reachable_nodes: List[structure.generic.Node] = [] for interface in bridge.get_interfaces(): node = interface.get_node() if node.get_id() is not None: @@ -63,7 +64,7 @@ class PrintRunner: self._last_bridge: int = 0 self._last_node_id = 0 - def build(self, bridge: structure.Bridge): + def build(self, bridge: structure.generic.Bridge): bridges, nodes = bridge_node_search(bridge, lambda _: self.name_bridge(), lambda _: self.id_node()) print(bridges) @@ -121,10 +122,10 @@ class ProxmoxRunner: self._template_id = template_id self._initial_vm_id = initial_vm_id - self._internet_bridge = structure.Bridge() + self._internet_bridge = structure.generic.Bridge() self._internet_bridge.set_name(internet_bridge) - self._management_bridge = structure.Bridge() + self._management_bridge = structure.generic.Bridge() self._management_bridge.set_name(management_bridge) self._management_initial_ip = management_initial_ip self._management_netmask = management_netmask @@ -133,7 +134,7 @@ class ProxmoxRunner: self._private_key = paramiko.RSAKey.generate(3072) self._client = paramiko.SSHClient() - def build(self, bridge: structure.Bridge): + def build(self, bridge: structure.generic.Bridge): bridges, nodes = bridge_node_search(bridge, lambda x: self._create_bridge(x), lambda x: self._create_node(x)) self._build_bridges(bridges) @@ -141,13 +142,13 @@ class ProxmoxRunner: with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: build_futures = [executor.submit(self._build_node, node) for node in nodes] for future in build_futures: - future.result() + future.result(300) # guarantee that setup is not called until all of the nodes are built # this means that all will have their final IPs by this point setup_futures = [executor.submit(self._setup_node, node) for node in nodes] for future in setup_futures: - future.result() + future.result(300) def _await_task(self, upid, timeout=10): t1 = datetime.now() @@ -156,7 +157,7 @@ class ProxmoxRunner: return raise TimeoutError - def _create_bridge(self, bridge: structure.Bridge) -> str: + def _create_bridge(self, bridge: structure.generic.Bridge) -> str: while True: try: self._proxmox.nodes(self._proxmox_node).network.post( @@ -177,7 +178,7 @@ class ProxmoxRunner: self._created_bridges.append(bridge_name) return bridge_name - def _build_bridges(self, bridges: List[structure.Bridge]): + def _build_bridges(self, bridges: List[structure.generic.Bridge]): network_task = self._proxmox.nodes(self._proxmox_node).network.put() self._await_task(network_task) @@ -187,7 +188,7 @@ class ProxmoxRunner: bridge.new_network() existing.append(bridge.get_network()) - def _create_node(self, node: structure.Node) -> int: + def _create_node(self, node: structure.generic.Node) -> int: while True: try: clone_task = self._proxmox.nodes(self._proxmox_node).qemu(self._template_id).clone.post( @@ -207,10 +208,10 @@ class ProxmoxRunner: self._created_nodes.append(new_id - 1) return new_id - 1 - def _open_ssh(self, node: structure.Node, interface: structure.Interface = None): + def _open_ssh(self, node: structure.generic.Node, interface: structure.generic.Interface = None): if interface is None: for iface in node.get_interfaces(): - if iface.get_method() == structure.IpMethod.Management: + if iface.get_method() == structure.generic.IpMethod.Management: interface = iface break if interface is None: @@ -235,13 +236,13 @@ class ProxmoxRunner: node.ssh_client = client - def _close_ssh(self, node: structure.Node): + def _close_ssh(self, node: structure.generic.Node): node.ssh_client.close() del node.ssh_client def ssh( self, - node: structure.Node, + node: structure.generic.Node, command: str, error_stderr=False, error_stdout=False, @@ -269,7 +270,7 @@ class ProxmoxRunner: return exit_status - def _build_node(self, node: structure.Node): + def _build_node(self, node: structure.generic.Node): # Step 1: Configure access self._proxmox.nodes(self._proxmox_node).qemu(node.get_id()).config.put( ciuser='python', @@ -282,7 +283,7 @@ class ProxmoxRunner: # Step 2: connect to Internet bridge with DHCP to install packages if node.get_internet_setup() is not None: interfaces = node.get_interfaces() - internet_interface = structure.Interface(structure.IpMethod.Dhcp4) + internet_interface = structure.generic.Interface(structure.generic.IpMethod.Dhcp4) internet_interface.set_bridge(self._internet_bridge) temp_interfaces = [internet_interface, interfaces[len(interfaces) - 1]] @@ -307,7 +308,7 @@ class ProxmoxRunner: self._open_ssh(node) node.ssh = (lambda n: lambda *args, **kwargs: self.ssh(n, *args, **kwargs))(node) - def _setup_node_interfaces(self, node: structure.Node, interfaces: List[structure.Interface] = None): + def _setup_node_interfaces(self, node: structure.generic.Node, interfaces: List[structure.generic.Interface] = None): if interfaces is None: interfaces = node.get_interfaces() @@ -316,20 +317,20 @@ class ProxmoxRunner: interface = interfaces[i] method = interface.get_method() - if method == structure.IpMethod.Management: + if method == structure.generic.IpMethod.Management: interface.set_bridge(self._management_bridge) addr = self._management_initial_ip + node.get_id() - self._initial_vm_id kwargs['ipconfig{}'.format(i)] = 'ip={}/{}'.format(addr, self._management_netmask) interface.set_address(addr) - elif method == structure.IpMethod.Auto4 or method == structure.IpMethod.Manual: + elif method == structure.generic.IpMethod.Auto4 or method == structure.generic.IpMethod.Manual: # handle manual the same as auto4 so it doesn't get stuck in DHCP bridge = interface.get_bridge() addr = bridge.get_ip_address() kwargs['ipconfig{}'.format(i)] = 'ip={}/{}'.format(addr, bridge.netmask) interface.set_address(addr) - elif method == structure.IpMethod.Dhcp4: + elif method == structure.generic.IpMethod.Dhcp4: kwargs['ipconfig{}'.format(i)] = 'ip=dhcp' else: raise RuntimeError('not implemented') @@ -340,7 +341,7 @@ class ProxmoxRunner: def interface_set_rate(iface): def new_set_rate(rate: Optional[int]): - structure.Interface.set_rate(iface, rate) + structure.generic.Interface.set_rate(iface, rate) self._update_node_interfaces(node) return new_set_rate @@ -349,7 +350,7 @@ class ProxmoxRunner: self._proxmox.nodes(self._proxmox_node).qemu(node.get_id()).config.put(**kwargs) - def _update_node_interfaces(self, node: structure.Node): + def _update_node_interfaces(self, node: structure.generic.Node): interfaces = node.get_interfaces() old_config = self._proxmox.nodes(self._proxmox_node).qemu(node.get_id()).config.get() @@ -368,7 +369,7 @@ class ProxmoxRunner: self._proxmox.nodes(self._proxmox_node).qemu(node.get_id()).config.put(**new_config) - def _setup_node(self, node: structure.Node): + def _setup_node(self, node: structure.generic.Node): if node.get_setup() is not None: cmd = node.get_setup() self.ssh(node, cmd, error_stdout=True, error_stderr=True) @@ -383,8 +384,9 @@ class ProxmoxRunner: for bridge in self._created_bridges: self._proxmox.nodes(self._proxmox_node).network(bridge).delete() - network_task = self._proxmox.nodes(self._proxmox_node).network.put() - self._await_task(network_task) + if len(self._created_bridges) > 0: + network_task = self._proxmox.nodes(self._proxmox_node).network.put() + self._await_task(network_task) self._created_nodes = [] self._last_node_id = 0 diff --git a/structure/__init__.py b/structure/__init__.py index d47d649..04cbf48 100644 --- a/structure/__init__.py +++ b/structure/__init__.py @@ -1,6 +1,5 @@ -from .structure import Node - -from .structure import IpMethod, Interface, Bridge +from .generic import IpMethod, Interface, Bridge, Node from .structure import SpeedTestServer, LocalPortal, RemotePortal -from .structure import StandardEnvironment, StandardTest, StandardIperfResult +from .tests import StandardTest, DirectTest, IperfResult +from .environments import StandardEnvironment, DirectEnvironment, BaseEnvironment diff --git a/structure/environments.py b/structure/environments.py new file mode 100644 index 0000000..bd940d5 --- /dev/null +++ b/structure/environments.py @@ -0,0 +1,114 @@ +import threading +from typing import Tuple, Optional + +from structure import Bridge, StandardTest, SpeedTestServer, Node, IperfResult, RemotePortal, Interface, \ + IpMethod, LocalPortal +from structure.tests import repeat_until_satisfied + + +class BaseEnvironment: + def __init__(self, runner, top_level_bridge: Bridge): + self.top_level_bridge = top_level_bridge + self._runner = runner + + def __enter__(self): + try: + self._runner.build(self.top_level_bridge) + except Exception as e: + self._runner.teardown() + raise e + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self._runner.teardown() + + def _test( + self, + test: StandardTest, + inbound_server: SpeedTestServer, + inbound_client: SpeedTestServer, + rated_node: Node, + expected_interfaces: int, + ) -> Tuple[IperfResult, IperfResult]: + if len(test.rates) != expected_interfaces: + raise RuntimeError('mismatched number of interfaces') + + results = [] + for server, client in [(inbound_server, inbound_client), (inbound_client, inbound_server)]: + def test_reducer(old: Optional[IperfResult]) -> IperfResult: + for i, r in enumerate(test.rates): + rated_node.get_interfaces()[i].set_rate(r) + server.server() + + for delay, (index, rate) in test.events.items(): + iface = rated_node.get_interfaces()[index] + threading.Timer(6 + delay, iface.set_rate, args=[rate]).start() + + iperf = client.client(server, time=test.duration) + if old is None: + return IperfResult(test, iperf) + else: + old.add_results(iperf) + return old + + def test_satisfier(val: IperfResult) -> bool: + if val.num_tests < 3: + return False + return val.bandwidth_coefficient_variance() < test.bandwidth_variation_target and False not in \ + [x < test.interval_variation_target for x in val.interval_coefficient_variances().values()] + + result = repeat_until_satisfied( + test_reducer, + test_satisfier, + max_failures=test.max_failures, + max_attempts=test.max_attempts, + ) + results.append(result) + + # Return a tuple of (inbound, outbound) + return results[0], results[1] + + def test(self, test: StandardTest) -> Tuple[IperfResult, IperfResult]: + raise RuntimeError('not implemented') + + +class StandardEnvironment(BaseEnvironment): + def __init__(self, interfaces: int, runner, setup_params: dict): + self._interfaces = interfaces + + self.rp = RemotePortal([Interface(IpMethod.Auto4)], setup_params=setup_params) + + self.st = SpeedTestServer() + self.cl = SpeedTestServer(clone_interface=self.rp.get_interfaces()[0]) + + self.lp = LocalPortal( + [Interface(IpMethod.Auto4) for _ in range(interfaces)], + self.cl, + setup_params=setup_params, + ) + + self.rp.set_local_portal(self.lp) + self.lp.set_remote_portal(self.rp) + + super().__init__(runner, Bridge( + self.st.get_interfaces()[0], + self.rp.get_interfaces()[0], + *self.lp.get_interfaces()[0:interfaces], + )) + + def test(self, test: StandardTest) -> Tuple[IperfResult, IperfResult]: + return self._test(test, self.st, self.cl, self.lp, self._interfaces) + + +class DirectEnvironment(BaseEnvironment): + def __init__(self, runner): + self.st1 = SpeedTestServer() + self.st2 = SpeedTestServer() + + super().__init__(runner, Bridge( + self.st1.get_interfaces()[0], + self.st2.get_interfaces()[0], + )) + + def test(self, test: StandardTest) -> Tuple[IperfResult, IperfResult]: + return self._test(test, self.st2, self.st1, self.st2, 1) diff --git a/structure/generic.py b/structure/generic.py new file mode 100644 index 0000000..49dd515 --- /dev/null +++ b/structure/generic.py @@ -0,0 +1,132 @@ +import ipaddress +import random +from enum import Enum +from typing import Optional, List, Dict, Union + + +class IpMethod(Enum): + Manual = 0 + Management = 1 + Auto4 = 2 + Auto6 = 3 + Dhcp4 = 4 + Dhcp6 = 5 + + +class Interface: + def __init__(self, method: IpMethod, rate: Optional[int] = None): + self._method: IpMethod + + self._node: Optional[Node] = None + self._rate: Optional[int] = None + self._bridge: Optional[Bridge] = None + + self._method = method + self._rate = rate + self._address: ipaddress.ip_address = None + + def get_method(self): + return self._method + + def set_node(self, node): + self._node = node + + def get_node(self): + return self._node + + def set_bridge(self, bridge): + self._bridge = bridge + + def get_bridge(self): + return self._bridge + + def set_address(self, addr: ipaddress.ip_address): + self._address = addr + + def get_address(self) -> ipaddress.ip_address: + return self._address + + def get_rate(self) -> Optional[int]: + return self._rate + + def set_rate(self, rate: Optional[int]): + self._rate = rate + + +class Bridge: + def __init__(self, *interfaces: Interface): + self._interfaces: List[Interface] = [] + self._name: str = '' + + for interface in interfaces: + self._interfaces.append(interface) + interface.set_bridge(self) + + # Generate a random class c private range by default (10.0.0.0) + self._addr: ipaddress.ip_address = None + self._network_iterator: iter = None + + self.netmask = 24 + self.new_network() + + def get_interfaces(self) -> List[Interface]: + return self._interfaces + + def set_name(self, name: str): + self._name = name + + def get_name(self) -> str: + return self._name + + def set_netmask(self, mask: int): + self.netmask = mask + self._network_iterator = ipaddress.ip_network('{}/{}'.format(self._addr, self.netmask), False).hosts() + + def get_ip_address(self) -> ipaddress.ip_address: + return next(self._network_iterator) + + def get_network(self) -> ipaddress.ip_network: + return ipaddress.ip_network('{}/{}'.format(self._addr, self.netmask), False) + + def get_network_string(self) -> str: + return str(ipaddress.ip_network('{}/{}'.format(self._addr, self.netmask), False)) + + def new_network(self): + self._addr: ipaddress.ip_address = ipaddress.ip_address('10.0.0.0') + random.randint(0, 16777216) + self._network_iterator = ipaddress.ip_network('{}/{}'.format(self._addr, self.netmask), False).hosts() + + +class Node: + def __init__(self, interfaces: List[Interface], setup_params: Dict = None): + self._id: Union[int, None] = None + self._interfaces: List[Interface] = interfaces + self._interfaces.append(Interface(IpMethod.Management)) + + for interface in self._interfaces: + interface.set_node(self) + + self.setup_params = {} if setup_params is None else setup_params + + def get_interfaces(self): + return self._interfaces + + def set_id(self, new_id): + self._id = new_id + + def get_id(self): + return self._id + + def get_core_count(self) -> int: + return 2 + + def get_memory_mb(self) -> int: + return 2048 + + def get_internet_setup(self) -> Optional[str]: + return None + + def get_setup(self) -> Optional[str]: + return None + + def ssh(self, *args, **kwargs): + raise RuntimeError('ssh not implemented') \ No newline at end of file diff --git a/structure/structure.py b/structure/structure.py index 445f657..3555048 100644 --- a/structure/structure.py +++ b/structure/structure.py @@ -1,140 +1,8 @@ -import ipaddress -import json import textwrap -import threading -from enum import Enum -import random -import numpy as np -from typing import List, Optional, Union, Dict, Tuple +from typing import List, Optional - -class IpMethod(Enum): - Manual = 0 - Management = 1 - Auto4 = 2 - Auto6 = 3 - Dhcp4 = 4 - Dhcp6 = 5 - - -class Interface: - def __init__(self, method: IpMethod, rate: Optional[int] = None): - self._method: IpMethod - - self._node: Optional[Node] = None - self._rate: Optional[int] = None - self._bridge: Optional[Bridge] = None - - self._method = method - self._rate = rate - self._address: ipaddress.ip_address = None - - def get_method(self): - return self._method - - def set_node(self, node): - self._node = node - - def get_node(self): - return self._node - - def set_bridge(self, bridge): - self._bridge = bridge - - def get_bridge(self): - return self._bridge - - def set_address(self, addr: ipaddress.ip_address): - self._address = addr - - def get_address(self) -> ipaddress.ip_address: - return self._address - - def get_rate(self) -> Optional[int]: - return self._rate - - def set_rate(self, rate: Optional[int]): - self._rate = rate - - -class Bridge: - def __init__(self, *interfaces: Interface): - self._interfaces: List[Interface] = [] - self._name: str = '' - - for interface in interfaces: - self._interfaces.append(interface) - interface.set_bridge(self) - - # Generate a random class c private range by default (10.0.0.0) - self._addr: ipaddress.ip_address = None - self._network_iterator: iter = None - - self.netmask = 24 - self.new_network() - - def get_interfaces(self) -> List[Interface]: - return self._interfaces - - def set_name(self, name: str): - self._name = name - - def get_name(self) -> str: - return self._name - - def set_netmask(self, mask: int): - self.netmask = mask - self._network_iterator = ipaddress.ip_network('{}/{}'.format(self._addr, self.netmask), False).hosts() - - def get_ip_address(self) -> ipaddress.ip_address: - return next(self._network_iterator) - - def get_network(self) -> ipaddress.ip_network: - return ipaddress.ip_network('{}/{}'.format(self._addr, self.netmask), False) - - def get_network_string(self) -> str: - return str(ipaddress.ip_network('{}/{}'.format(self._addr, self.netmask), False)) - - def new_network(self): - self._addr: ipaddress.ip_address = ipaddress.ip_address('10.0.0.0') + random.randint(0, 16777216) - self._network_iterator = ipaddress.ip_network('{}/{}'.format(self._addr, self.netmask), False).hosts() - - -class Node: - def __init__(self, interfaces: List[Interface], setup_params: Dict = None): - self._id: Union[int, None] = None - self._interfaces: List[Interface] = interfaces - self._interfaces.append(Interface(IpMethod.Management)) - - for interface in self._interfaces: - interface.set_node(self) - - self.setup_params = {} if setup_params is None else setup_params - - def get_interfaces(self): - return self._interfaces - - def set_id(self, new_id): - self._id = new_id - - def get_id(self): - return self._id - - def get_core_count(self) -> int: - return 2 - - def get_memory_mb(self) -> int: - return 2048 - - def get_internet_setup(self) -> Optional[str]: - return None - - def get_setup(self) -> Optional[str]: - return None - - def ssh(self, *args, **kwargs): - raise RuntimeError('ssh not implemented') +from structure import IpMethod, Interface, Bridge, Node class SpeedTestServer(Node): @@ -343,223 +211,3 @@ class LocalPortal(Node): remote_host=self.remote_portal.get_interfaces()[0].get_address(), local_interface='eth{}'.format(len(self.get_interfaces()) - 2), ) - - -class StandardTest: - def __init__( - self, - rates: List[int], - events: Dict[float, Tuple[int, int]] = None, - duration: int = 10, - variation_target: float = 0.2, - max_failures: int = 3, - max_attempts: int = 60, - ): - self.rates = rates - self.events = events if events is not None else dict() - self.duration = duration - - self.variation_target = variation_target - self.max_failures = max_failures - self.max_attempts = max_attempts - - def name(self) -> str: - name_builder = ['R{}-{}'.format(*y) for y in enumerate(self.rates)] - name_builder += ['E{}R{}-{}'.format(x, *y) for (x, y) in self.events.items()] - name_builder.append('T{}'.format(self.duration)) - return ''.join(name_builder) - - -class DirectTest(StandardTest): - def __init__(self, rate: int, **kwargs): - super().__init__([rate], **kwargs) - - def name(self) -> str: - return 'D{}'.format(super().name()) - - -class StandardIperfResult: - def __init__(self, test: StandardTest, iperf: str, interval_size=2.0): - self.test = test - - self.interval_size = interval_size - - # list containing an exact time and a value - self.data: List[Tuple[float, float]] = [] - - self.num_tests = 0 - self.add_results(iperf) - - def add_results(self, iperf: str): - data = json.loads(iperf) - # grab the sum data of all non omitted intervals, excluding any that are smaller than expected - intervals = [ - x['sum'] for x in data['intervals'] if - (not x['sum']['omitted']) and (x['sum']['end'] - x['sum']['start'] > self.interval_size / 2) - ] - - for (time, result) in zip( - [((x['start'] + x['end']) / 2) for x in intervals], - [x['bits_per_second'] for x in intervals], - ): - self.data.append((time, result)) - - self.num_tests += 1 - - def bins(self) -> List[List[Tuple[float, float]]]: - bins: List[List[Tuple[float, float]]] = [[] for _ in np.arange(0, self.test.duration, self.interval_size)] - - for time, result in self.data: - index = int(np.round((time - self.interval_size / 2) / self.interval_size)) - bins[index].append((time, result)) - - return bins - - def summarise(self) -> Dict[float, float]: - bins = self.bins() - means = [np.mean(x, axis=0)[1] for x in bins] - times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)] - return dict(zip(times, means)) - - def standard_deviation(self) -> Dict[float, float]: - bins = self.bins() - stds = [np.std(x, axis=0)[1] for x in bins] - times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)] - return dict(zip(times, stds)) - - def coefficient_variance(self) -> Dict[float, float]: - stds = self.standard_deviation() - means = self.summarise() - - return {k: stds[k] / means[k] for k in stds.keys()} - - def time_range(self) -> Dict[float, Tuple[float, float]]: - bins = self.bins() - times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)] - ranges = [(-np.min(x, axis=0)[0] + time, np.max(x, axis=0)[0] - time) for (x, time) in zip(bins, times)] - return dict(zip(times, ranges)) - - -def repeat_until_satisfied(reducer, satisfied, initial=None, max_attempts=100, max_failures=3): - val = initial - - for i in range(max_attempts): - for j in range(max_failures): - try: - val = reducer(val) - break - except Exception as e: - print('failed with {}'.format(e)) - if j == max_failures - 1: - raise e - - if satisfied(val): - return val - - raise RuntimeError('too many attempts') - - -class BaseEnvironment: - def __init__(self, runner, top_level_bridge: Bridge): - self.top_level_bridge = top_level_bridge - self._runner = runner - - def __enter__(self): - try: - self._runner.build(self.top_level_bridge) - except Exception as e: - self._runner.teardown() - raise e - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self._runner.teardown() - - def _test( - self, - test: StandardTest, - inbound_server: SpeedTestServer, - inbound_client: SpeedTestServer, - rated_node: Node, - expected_interfaces: int, - ) -> Tuple[StandardIperfResult, StandardIperfResult]: - if len(test.rates) != expected_interfaces: - raise RuntimeError('mismatched number of interfaces') - - results = [] - for server, client in [(inbound_server, inbound_client), (inbound_client, inbound_server)]: - def test_reducer(old: Optional[StandardIperfResult]) -> StandardIperfResult: - for i, r in enumerate(test.rates): - rated_node.get_interfaces()[i].set_rate(r) - server.server() - - for t, (iface, rate) in test.events.items(): - threading.Timer( - 6 + t, - (lambda n: lambda: n.get_interfaces()[iface].set_rate(rate))(rated_node), - ) - - iperf = client.client(server, time=test.duration) - if old is None: - return StandardIperfResult(test, iperf) - else: - old.add_results(iperf) - return old - - def test_satisfier(val: StandardIperfResult) -> bool: - if val.num_tests < 3: - return False - return False not in [x < test.variation_target for x in val.coefficient_variance().values()] - - result = repeat_until_satisfied( - test_reducer, - test_satisfier, - max_failures=test.max_failures, - max_attempts=test.max_attempts, - ) - results.append(result) - - # Return a tuple of (inbound, outbound) - return results[0], results[1] - - -class StandardEnvironment(BaseEnvironment): - def __init__(self, interfaces: int, runner, setup_params: dict): - self._interfaces = interfaces - - self.rp = RemotePortal([Interface(IpMethod.Auto4)], setup_params=setup_params) - - self.st = SpeedTestServer() - self.cl = SpeedTestServer(clone_interface=self.rp.get_interfaces()[0]) - - self.lp = LocalPortal( - [Interface(IpMethod.Auto4) for _ in range(interfaces)], - self.cl, - setup_params=setup_params, - ) - - self.rp.set_local_portal(self.lp) - self.lp.set_remote_portal(self.rp) - - super().__init__(runner, Bridge( - self.st.get_interfaces()[0], - self.rp.get_interfaces()[0], - *self.lp.get_interfaces()[0:interfaces], - )) - - def test(self, test: StandardTest) -> Tuple[StandardIperfResult, StandardIperfResult]: - return self._test(test, self.st, self.cl, self.lp, self._interfaces) - - -class DirectEnvironment(BaseEnvironment): - def __init__(self, runner): - self.st1 = SpeedTestServer() - self.st2 = SpeedTestServer() - - super().__init__(runner, Bridge( - self.st1.get_interfaces()[0], - self.st2.get_interfaces()[0], - )) - - def test(self, test: StandardTest) -> Tuple[StandardIperfResult, StandardIperfResult]: - return self._test(test, self.st2, self.st1, self.st2, 1) diff --git a/structure/tests.py b/structure/tests.py new file mode 100644 index 0000000..5b6dac3 --- /dev/null +++ b/structure/tests.py @@ -0,0 +1,137 @@ +import json +from typing import List, Dict, Tuple + +import numpy as np + + +def repeat_until_satisfied(reducer, satisfied, initial=None, max_attempts=100, max_failures=3): + val = initial + + for i in range(max_attempts): + for j in range(max_failures): + try: + val = reducer(val) + break + except Exception as e: + print('failed with {}'.format(e)) + if j == max_failures - 1: + raise e + + if satisfied(val): + return val + + raise RuntimeError('too many attempts') + + +class StandardTest: + def __init__( + self, + rates: List[int], + events: Dict[float, Tuple[int, int]] = None, + duration: int = 10, + + interval_variation_target: float = np.inf, + bandwidth_variation_target: float = np.inf, + max_failures: int = 3, + max_attempts: int = 60, + ): + self.rates = rates + self.events = events if events is not None else dict() + self.duration = duration + + self.interval_variation_target = interval_variation_target + self.bandwidth_variation_target = bandwidth_variation_target + self.max_failures = max_failures + self.max_attempts = max_attempts + + def name(self) -> str: + name_builder = ['R{}-{}'.format(*y) for y in enumerate(self.rates)] + name_builder += ['E{}R{}-{}'.format(x, *y) for (x, y) in self.events.items()] + name_builder.append('T{}'.format(self.duration)) + return ''.join(name_builder) + + +class DirectTest(StandardTest): + def __init__(self, rate: int, **kwargs): + super().__init__([rate], **kwargs) + + def name(self) -> str: + return 'D{}'.format(super().name()) + + +class IperfResult: + def __init__(self, test: StandardTest, iperf: str, interval_size=2.0): + self.test = test + + self.interval_size = interval_size + + # list containing an exact time and a value + self.interval_data: List[Tuple[float, float]] = [] + + # list containing the overall data transferred and the time taken + self.bandwidth_data: List[float] = [] + + self.num_tests = 0 + self.add_results(iperf) + + def add_results(self, iperf: str): + data = json.loads(iperf) + + # grab the overall bandwidth + self.bandwidth_data.append(data['end']['sum_sent']['bits_per_second']) + + # grab the sum data of all non omitted intervals, excluding any that are smaller than expected + intervals = [ + x['sum'] for x in data['intervals'] if + (not x['sum']['omitted']) and (x['sum']['end'] - x['sum']['start'] > self.interval_size / 2) + ] + + for (time, result) in zip( + [((x['start'] + x['end']) / 2) for x in intervals], + [x['bits_per_second'] for x in intervals], + ): + self.interval_data.append((time, result)) + + self.num_tests += 1 + + def bins(self) -> List[List[Tuple[float, float]]]: + bins: List[List[Tuple[float, float]]] = [[] for _ in np.arange(0, self.test.duration, self.interval_size)] + + for time, result in self.interval_data: + index = int(np.round((time - self.interval_size / 2) / self.interval_size)) + bins[index].append((time, result)) + + return bins + + def interval_means(self) -> Dict[float, float]: + bins = self.bins() + means = [np.mean(x, axis=0)[1] for x in bins] + times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)] + return dict(zip(times, means)) + + def interval_standard_deviations(self) -> Dict[float, float]: + bins = self.bins() + stds = [np.std(x, axis=0)[1] for x in bins] + times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)] + return dict(zip(times, stds)) + + def interval_coefficient_variances(self) -> Dict[float, float]: + stds = self.interval_standard_deviations() + means = self.interval_means() + + return {k: stds[k] / means[k] for k in stds.keys()} + + def interval_time_ranges(self) -> Dict[float, Tuple[float, float]]: + bins = self.bins() + times = [i + self.interval_size / 2 for i in np.arange(0, self.test.duration, self.interval_size)] + ranges = [(-np.min(x, axis=0)[0] + time, np.max(x, axis=0)[0] - time) for (x, time) in zip(bins, times)] + return dict(zip(times, ranges)) + + def bandwidth_mean(self): + return np.mean(self.bandwidth_data) + + def bandwidth_standard_deviation(self): + return np.std(self.bandwidth_data) + + def bandwidth_coefficient_variance(self): + return self.bandwidth_standard_deviation() / self.bandwidth_mean()