diff --git a/.gitignore b/.gitignore index 777ad6a..00491ff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +graphs/ # Created by https://www.toptal.com/developers/gitignore/api/python,intellij+all,dotenv # Edit at https://www.toptal.com/developers/gitignore?templates=python,intellij+all,dotenv diff --git a/evaluation.ipynb b/evaluation.ipynb index 2583999..19370e1 100644 --- a/evaluation.ipynb +++ b/evaluation.ipynb @@ -87,7 +87,21 @@ "directionOutbound: Dict[str, StandardIperfResult] = {}\n", "\n", "def run_and_save_test(env: StandardEnvironment, test: StandardTest):\n", - " (directionInbound[test.name()], directionOutbound[test.name()]) = env.test(test)" + " (directionInbound[test.name()], directionOutbound[test.name()]) = env.test(test)\n", + "\n", + "def attempt_n_times(foo, n=3):\n", + " i = 0\n", + " while True:\n", + " try:\n", + " return foo()\n", + " except Exception as e:\n", + " if i == n:\n", + " raise e\n", + " finally:\n", + " i += 1\n", + "\n", + "\n", + "fast_tests = True" ] }, { @@ -124,11 +138,19 @@ }, "outputs": [], "source": [ - "with StandardEnvironment(2, runner, setup_params) as env:\n", - " run_and_save_test(env, StandardTest([1,1]))\n", - " run_and_save_test(env, StandardTest([1,2]))\n", - " run_and_save_test(env, StandardTest([2,2]))\n", - " run_and_save_test(env, StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30))\n" + "def two_interface_tests():\n", + " with StandardEnvironment(2, runner, setup_params) as env:\n", + " run_and_save_test(env, StandardTest([1,1], variation_target=0.4 if fast_tests else 0.2))\n", + " run_and_save_test(env, StandardTest([1,2], variation_target=0.4 if fast_tests else 0.2))\n", + " run_and_save_test(env, StandardTest([2,2], variation_target=0.4 if fast_tests else 0.2))\n", + " run_and_save_test(env, StandardTest(\n", + " [2,2],\n", + " events={10: (0,1), 15: (0,2)},\n", + " duration=30,\n", + " variation_target=0.8 if fast_tests else 0.5,\n", + " ))\n", + "\n", + "attempt_n_times(two_interface_tests)" ] }, { @@ -150,9 +172,12 @@ }, "outputs": [], "source": [ - "with StandardEnvironment(3, runner, setup_params) as env:\n", - " run_and_save_test(env, StandardTest([1,1,1]))\n", - " run_and_save_test(env, StandardTest([2,2,2]))\n" + "def three_interface_tests():\n", + " with StandardEnvironment(3, runner, setup_params) as env:\n", + " run_and_save_test(env, StandardTest([1,1,1], variation_target=0.4 if fast_tests else 0.2))\n", + " run_and_save_test(env, StandardTest([2,2,2], variation_target=0.4 if fast_tests else 0.2))\n", + "\n", + "attempt_n_times(three_interface_tests)" ] }, { @@ -174,9 +199,12 @@ }, "outputs": [], "source": [ - "with StandardEnvironment(4, runner, setup_params) as env:\n", - " run_and_save_test(env, StandardTest([1,1,1,1]))\n", - " run_and_save_test(env, StandardTest([2,2,2,2]))\n" + "def four_interface_tests():\n", + " with StandardEnvironment(4, runner, setup_params) as env:\n", + " run_and_save_test(env, StandardTest([1,1,1,1], variation_target=0.4 if fast_tests else 0.2))\n", + " run_and_save_test(env, StandardTest([2,2,2,2], variation_target=0.4 if fast_tests else 0.2))\n", + "\n", + "attempt_n_times(four_interface_tests)" ] }, { @@ -203,12 +231,26 @@ "def plot_iperf_results(\n", " series: Dict[str, StandardTest],\n", " title: str = None,\n", - " direction = 'inbound',\n", + " direction = 'both',\n", " error_bars_x=False,\n", - " error_bars_y=False,\n", + " error_bars_y=True,\n", " filename=None,\n", " start_at_zero=True,\n", "):\n", + " if direction == 'both':\n", + " plot_iperf_results(series, title, 'outbound', error_bars_x, error_bars_y, filename, start_at_zero)\n", + " plot_iperf_results(series, title, 'inbound', error_bars_x, error_bars_y, filename, start_at_zero)\n", + " return\n", + "\n", + " if filename in ['png', 'eps']:\n", + " filename = 'graphs/{}{}{}{}.{}'.format(\n", + " 'I' if direction == 'inbound' else 'O',\n", + " 'Ex' if error_bars_x else '',\n", + " 'Ey' if error_bars_y else '',\n", + " ''.join(['S{}-{}'.format(i,x.name()) for (i, x) in enumerate(series.values())]),\n", + " filename,\n", + " )\n", + "\n", " series = {\n", " k: (directionInbound if direction == 'inbound' else directionOutbound)[v.name()] for (k, v) in series.items()\n", " }\n", @@ -249,20 +291,63 @@ " axes.axvline(k, linestyle='--', color='grey')\n", " axes.annotate(v, (k, 1.02), xycoords=axes.get_xaxis_transform(), ha='center')\n", "\n", + " if fast_tests:\n", + " fig.text(0.95, 0.05, 'Draft', fontsize=50, color='gray', ha='right', va='bottom', alpha=0.5)\n", + "\n", " if filename is not None:\n", " fig.savefig(filename, bbox_extra_artists=(legend,), bbox_inches='tight', pad_inches=0.3)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "if not os.path.exists('graphs/'):\n", + " os.makedirs('graphs/')\n", + "\n", + "for filename in os.listdir('graphs'):\n", + " file_path = os.path.join('graphs/', filename)\n", + " os.unlink(file_path)" + ] + }, { "cell_type": "markdown", "metadata": { + "collapsed": false, "pycharm": { "name": "#%% md\n" } }, "source": [ - "### Equal Connection Scaling\n", - "This section shows equal connections scaling at various speeds and number of connections." + "### Error Bar Evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plot_iperf_results(\n", + " {\n", + " '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n", + " '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n", + " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", + " },\n", + " error_bars_x=False,\n", + " error_bars_y=False,\n", + " filename='png',\n", + ")" ] }, { @@ -281,9 +366,41 @@ " '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n", " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", " },\n", - " 'Scaling of Equal Connections',\n", " error_bars_x=True,\n", - " error_bars_y=True,\n", + " error_bars_y=False,\n", + " filename='png',\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### More than 2 connections evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plot_iperf_results(\n", + " {\n", + " '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n", + " '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n", + " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", + " },\n", + " filename='png',\n", ")" ] }, @@ -303,12 +420,54 @@ " '3x2MB Connections (proxied)': StandardTest([2,2,2]),\n", " '2x2MB Connections (proxied)': StandardTest([2,2]),\n", " },\n", - " 'Scaling of Equal Connections',\n", - " error_bars_x=True,\n", - " error_bars_y=True,\n", + " filename='png',\n", ")" ] }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Mixed Performance Evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plot_iperf_results(\n", + " {\n", + " '2x2MB Connections (proxied)': StandardTest([2,2]),\n", + " '1x1MB + 1x2MB Connections (proxied)': StandardTest([1,2]),\n", + " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", + " },\n", + " filename='png',\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Eventful Evaluation" + ] + }, { "cell_type": "code", "execution_count": null, @@ -324,6 +483,7 @@ " 'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n", " },\n", " error_bars_y=True,\n", + " filename='png',\n", ")" ] }, diff --git a/structure/structure.py b/structure/structure.py index ff11a73..4e6c01b 100644 --- a/structure/structure.py +++ b/structure/structure.py @@ -68,6 +68,9 @@ class Bridge: interface.set_bridge(self) # Generate a random class c private range by default (10.0.0.0) + self._addr: ipaddress.ip_address = None + self._network_iterator: iter = None + self.netmask = 24 self.new_network() @@ -185,13 +188,7 @@ class RemotePortal(Node): return textwrap.dedent(''' set -e - wget -q http://10.20.0.11/minio-client - chmod +x minio-client - - ./minio-client alias set s3 http://10.20.0.25:3900 {access_key} {secret_key} || \ - ./minio-client alias set s3 s3.us-west-001.backblazeb2.com {access_key} {secret_key} - ./minio-client cp s3/dissertation/binaries/debian/{branch} mpbl3p - + wget -q https://f001.backblazeb2.com/file/dissertation/binaries/debian/{branch} -O mpbl3p chmod +x mpbl3p cloud-init status --wait || cloud-init status --long @@ -258,14 +255,8 @@ class LocalPortal(Node): def get_internet_setup(self) -> Optional[str]: return textwrap.dedent(''' set -e - - wget -q http://10.20.0.11/minio-client - chmod +x minio-client - - ./minio-client alias set s3 http://10.20.0.25:3900 {access_key} {secret_key} || \ - ./minio-client alias set s3 s3.us-west-001.backblazeb2.com {access_key} {secret_key} - ./minio-client cp s3/dissertation/binaries/debian/{branch} mpbl3p + wget -q https://f001.backblazeb2.com/file/dissertation/binaries/debian/{branch} -O mpbl3p chmod +x mpbl3p cloud-init status --wait || cloud-init status --long @@ -354,7 +345,7 @@ class StandardTest: rates: List[int], events: Dict[float, Tuple[int, int]] = None, duration: int = 10, - variation_target: float = 0.3, + variation_target: float = 0.2, max_failures: int = 3, max_attempts: int = 60, ): @@ -369,7 +360,7 @@ class StandardTest: def name(self) -> str: name_builder = ['R{}-{}'.format(*y) for y in enumerate(self.rates)] name_builder += ['E{}R{}-{}'.format(x, *y) for (x, y) in self.events.items()] - name_builder.append('D{}'.format(self.duration)) + name_builder.append('T{}'.format(self.duration)) return ''.join(name_builder)