improved testing

evaluation.ipynb

@@ -132,7 +132,9 @@
     "        run_and_save_test(env, DirectTest(1, variation_target=0.4 if fast_tests else 0.2))\n",
     "        run_and_save_test(env, DirectTest(2, variation_target=0.4 if fast_tests else 0.2))\n",
     "        run_and_save_test(env, DirectTest(3, variation_target=0.4 if fast_tests else 0.2))\n",
-    "        run_and_save_test(env, DirectTest(4, variation_target=0.4 if fast_tests else 0.2))"
+    "        run_and_save_test(env, DirectTest(4, variation_target=0.4 if fast_tests else 0.2))\n",
+    "\n",
+    "attempt_n_times(direct_tests)"
    ]
   },
   {
@@ -228,7 +230,11 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   },
    "source": [
     "## Graphs\n",
     "This section produces graphs from the collected data."
@@ -391,6 +397,162 @@
     ")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": false
+   },
+   "source": [
+    "### More than 2 connections evaluation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "plot_iperf_results(\n",
+    "    {\n",
+    "        '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n",
+    "        '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
+    "        '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
+    "    },\n",
+    "    filename='png',\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "plot_iperf_results(\n",
+    "    {\n",
+    "        '4x2MB Connections (proxied)': StandardTest([2,2,2,2]),\n",
+    "        '3x2MB Connections (proxied)': StandardTest([2,2,2]),\n",
+    "        '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
+    "    },\n",
+    "    filename='png',\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": false
+   },
+   "source": [
+    "### Mixed Performance Evaluation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "plot_iperf_results(\n",
+    "    {\n",
+    "        '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
+    "        '1x1MB + 1x2MB Connections (proxied)': StandardTest([1,2]),\n",
+    "        '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
+    "    },\n",
+    "    filename='png',\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": false
+   },
+   "source": [
+    "### Eventful Evaluation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "plot_iperf_results(\n",
+    "    {\n",
+    "        'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n",
+    "    },\n",
+    "    filename='png',\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": false
+   },
+   "source": [
+    "### Comparisons to a Direct Connection"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "plot_iperf_results(\n",
+    "    {\n",
+    "        '1x2MB Connection (not proxied)': DirectTest(2),\n",
+    "        '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
+    "        '1x1MB Connection (not proxied)': DirectTest(1),\n",
+    "\n",
+    "    },\n",
+    "    filename='png',\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "plot_iperf_results(\n",
+    "    {\n",
+    "        '1x4MB Connection (not proxied)': DirectTest(4),\n",
+    "        '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
+    "        '1x2MB Connection (not proxied)': DirectTest(2),\n",
+    "\n",
+    "    },\n",
+    "    filename='png',\n",
+    ")"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {
@@ -399,9 +561,7 @@
      "name": "#%% md\n"
     }
    },
-   "source": [
+   "source": []
-    "### More than 2 connections evaluation"
-   ]
   },
   {
    "cell_type": "code",

structure/structure.py

@@ -381,6 +381,7 @@ class StandardIperfResult:
         # list containing an exact time and a value
         self.data: List[Tuple[float, float]] = []
 
+        self.num_tests = 0
         self.add_results(iperf)
 
     def add_results(self, iperf: str):
@@ -397,6 +398,8 @@ class StandardIperfResult:
         ):
             self.data.append((time, result))
 
+        self.num_tests += 1
+
     def bins(self) -> List[List[Tuple[float, float]]]:
         bins: List[List[Tuple[float, float]]] = [[] for _ in np.arange(0, self.test.duration, self.interval_size)]
 
@@ -432,12 +435,13 @@ class StandardIperfResult:
 
 
 def repeat_until_satisfied(reducer, satisfied, initial=None, max_attempts=100, max_failures=3):
-    val = initial()
+    val = initial
-    i = 0
+
     for i in range(max_attempts):
         for j in range(max_failures):
             try:
                 val = reducer(val)
+                break
             except Exception as e:
                 print('failed with {}'.format(e))
                 if j == max_failures - 1:
@@ -496,9 +500,14 @@ class BaseEnvironment:
                     old.add_results(iperf)
                     return old
 
+            def test_satisfier(val: StandardIperfResult) -> bool:
+                if val.num_tests < 3:
+                    return False
+                return False not in [x < test.variation_target for x in val.coefficient_variance().values()]
+
             result = repeat_until_satisfied(
                 test_reducer,
-                lambda x: max(x.coefficient_variance().values()) < test.variation_target,
+                test_satisfier,
                 max_failures=test.max_failures,
                 max_attempts=test.max_attempts,
             )