diff --git a/evaluation.ipynb b/evaluation.ipynb index 36cd786..858fdb9 100644 --- a/evaluation.ipynb +++ b/evaluation.ipynb @@ -91,15 +91,14 @@ " (directionInbound[test.name()], directionOutbound[test.name()]) = env.test(test)\n", "\n", "def attempt_n_times(foo, n=3):\n", - " i = 0\n", - " while True:\n", + " for i in range(n):\n", " try:\n", " return foo()\n", + " except KeyboardInterrupt as e:\n", + " raise e\n", " except Exception as e:\n", - " if i == n:\n", + " if i == n - 1:\n", " raise e\n", - " finally:\n", - " i += 1\n", "\n", "\n", "fast_tests = True" @@ -114,7 +113,7 @@ } }, "source": [ - "### Direct Server to Server Testing" + "### Direct Server to Server" ] }, { @@ -132,7 +131,9 @@ " run_and_save_test(env, DirectTest(1, variation_target=0.4 if fast_tests else 0.2))\n", " run_and_save_test(env, DirectTest(2, variation_target=0.4 if fast_tests else 0.2))\n", " run_and_save_test(env, DirectTest(3, variation_target=0.4 if fast_tests else 0.2))\n", - " run_and_save_test(env, DirectTest(4, variation_target=0.4 if fast_tests else 0.2))" + " run_and_save_test(env, DirectTest(4, variation_target=0.4 if fast_tests else 0.2))\n", + "\n", + "attempt_n_times(direct_tests)" ] }, { @@ -228,7 +229,11 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "## Graphs\n", "This section produces graphs from the collected data." @@ -391,6 +396,316 @@ ")" ] }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false + }, + "source": [ + "### More than 2 connections evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plot_iperf_results(\n", + " {\n", + " '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n", + " '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n", + " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", + " },\n", + " filename='png',\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plot_iperf_results(\n", + " {\n", + " '4x2MB Connections (proxied)': StandardTest([2,2,2,2]),\n", + " '3x2MB Connections (proxied)': StandardTest([2,2,2]),\n", + " '2x2MB Connections (proxied)': StandardTest([2,2]),\n", + " },\n", + " filename='png',\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false + }, + "source": [ + "### Mixed Performance Evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plot_iperf_results(\n", + " {\n", + " '2x2MB Connections (proxied)': StandardTest([2,2]),\n", + " '1x1MB + 1x2MB Connections (proxied)': StandardTest([1,2]),\n", + " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", + " },\n", + " filename='png',\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false + }, + "source": [ + "### Eventful Evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plot_iperf_results(\n", + " {\n", + " 'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n", + " },\n", + " filename='png',\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false + }, + "source": [ + "### Comparisons to a Direct Connection" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plot_iperf_results(\n", + " {\n", + " '1x2MB Connection (not proxied)': DirectTest(2),\n", + " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", + " '1x1MB Connection (not proxied)': DirectTest(1),\n", + "\n", + " },\n", + " filename='png',\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plot_iperf_results(\n", + " {\n", + " '1x4MB Connection (not proxied)': DirectTest(4),\n", + " '2x2MB Connections (proxied)': StandardTest([2,2]),\n", + " '1x2MB Connection (not proxied)': DirectTest(2),\n", + "\n", + " },\n", + " filename='png',\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false + }, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plot_iperf_results(\n", + " {\n", + " '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n", + " '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n", + " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", + " },\n", + " filename='png',\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plot_iperf_results(\n", + " {\n", + " '4x2MB Connections (proxied)': StandardTest([2,2,2,2]),\n", + " '3x2MB Connections (proxied)': StandardTest([2,2,2]),\n", + " '2x2MB Connections (proxied)': StandardTest([2,2]),\n", + " },\n", + " filename='png',\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false + }, + "source": [ + "### Mixed Performance Evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plot_iperf_results(\n", + " {\n", + " '2x2MB Connections (proxied)': StandardTest([2,2]),\n", + " '1x1MB + 1x2MB Connections (proxied)': StandardTest([1,2]),\n", + " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", + " },\n", + " filename='png',\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false + }, + "source": [ + "### Eventful Evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plot_iperf_results(\n", + " {\n", + " 'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n", + " },\n", + " filename='png',\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false + }, + "source": [ + "### Comparisons to a Direct Connection" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plot_iperf_results(\n", + " {\n", + " '1x2MB Connection (not proxied)': DirectTest(2),\n", + " '2x1MB Connections (proxied)': StandardTest([1,1]),\n", + " '1x1MB Connection (not proxied)': DirectTest(1),\n", + "\n", + " },\n", + " filename='png',\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plot_iperf_results(\n", + " {\n", + " '1x4MB Connection (not proxied)': DirectTest(4),\n", + " '2x2MB Connections (proxied)': StandardTest([2,2]),\n", + " '1x2MB Connection (not proxied)': DirectTest(2),\n", + "\n", + " },\n", + " filename='png',\n", + ")" + ] + }, { "cell_type": "markdown", "metadata": { @@ -399,9 +714,7 @@ "name": "#%% md\n" } }, - "source": [ - "### More than 2 connections evaluation" - ] + "source": [] }, { "cell_type": "code", diff --git a/structure/structure.py b/structure/structure.py index c27bf78..445f657 100644 --- a/structure/structure.py +++ b/structure/structure.py @@ -167,11 +167,15 @@ class SpeedTestServer(Node): def server(self): self.ssh('iperf3 -s -1 -D', error_stdout=True, error_stderr=True) - def client(self, target, time=30): + def client(self, target, time=30, interval_size=2.0): if isinstance(target, SpeedTestServer): target = target.get_interfaces()[0].get_address() - command = 'iperf3 -c {target} -t {time} -O 5 -J'.format(target=target, time=time) + command = 'iperf3 -c {target} -t {time} -O 6 -i {interval_size} -J'.format( + target=target, + time=time, + interval_size=interval_size, + ) return self.ssh(command, error_stdout=True, error_stderr=True, return_stdout=True) @@ -230,7 +234,8 @@ class RemotePortal(Node): sudo ip route flush 10 sudo ip route add table 10 to {local_host} via 172.19.152.3 dev nc0 sudo ip rule add to {local_host} table 10 priority 10 - + + ping -W 0.1 -c 1 172.19.152.3 ps $(cat mpbl3p.pid) || cat mpbl3p.log ''').format( local_host=self.get_interfaces()[0].get_address(), @@ -328,7 +333,8 @@ class LocalPortal(Node): sudo ip route flush 19 sudo ip route add to {remote_host} dev {local_interface} table 19 sudo ip rule add to {remote_host} table 19 priority 19 - + + ping -W 0.1 -c 1 172.19.152.2 ps $(cat mpbl3p.pid) || cat mpbl3p.log ''').format( **self.setup_params, @@ -373,7 +379,7 @@ class DirectTest(StandardTest): class StandardIperfResult: - def __init__(self, test: StandardTest, iperf: str, interval_size=1.0): + def __init__(self, test: StandardTest, iperf: str, interval_size=2.0): self.test = test self.interval_size = interval_size @@ -381,6 +387,7 @@ class StandardIperfResult: # list containing an exact time and a value self.data: List[Tuple[float, float]] = [] + self.num_tests = 0 self.add_results(iperf) def add_results(self, iperf: str): @@ -397,6 +404,8 @@ class StandardIperfResult: ): self.data.append((time, result)) + self.num_tests += 1 + def bins(self) -> List[List[Tuple[float, float]]]: bins: List[List[Tuple[float, float]]] = [[] for _ in np.arange(0, self.test.duration, self.interval_size)] @@ -432,12 +441,13 @@ class StandardIperfResult: def repeat_until_satisfied(reducer, satisfied, initial=None, max_attempts=100, max_failures=3): - val = initial() - i = 0 + val = initial + for i in range(max_attempts): for j in range(max_failures): try: val = reducer(val) + break except Exception as e: print('failed with {}'.format(e)) if j == max_failures - 1: @@ -485,8 +495,8 @@ class BaseEnvironment: for t, (iface, rate) in test.events.items(): threading.Timer( - 5 + t, - (lambda s: lambda: s.lp.get_interfaces()[iface].set_rate(rate))(self), + 6 + t, + (lambda n: lambda: n.get_interfaces()[iface].set_rate(rate))(rated_node), ) iperf = client.client(server, time=test.duration) @@ -496,9 +506,14 @@ class BaseEnvironment: old.add_results(iperf) return old + def test_satisfier(val: StandardIperfResult) -> bool: + if val.num_tests < 3: + return False + return False not in [x < test.variation_target for x in val.coefficient_variance().values()] + result = repeat_until_satisfied( test_reducer, - lambda x: max(x.coefficient_variance().values()) < test.variation_target, + test_satisfier, max_failures=test.max_failures, max_attempts=test.max_attempts, )