JakeHillion/dissertation-3-evaluation
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

improved testing

Browse Source
This commit is contained in:
Jake Hillion 2020-11-16 14:55:05 +00:00
parent 907093a925
commit 433f8785af
2 changed files with 177 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

170
evaluation.ipynb
View File

@ -132,7 +132,9 @@
" run_and_save_test(env, DirectTest(1, variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, DirectTest(2, variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, DirectTest(3, variation_target=0.4 if fast_tests else 0.2))\n",
" run_and_save_test(env, DirectTest(4, variation_target=0.4 if fast_tests else 0.2))"
" run_and_save_test(env, DirectTest(4, variation_target=0.4 if fast_tests else 0.2))\n",
"\n",
"attempt_n_times(direct_tests)"
]
},
{
@ -228,7 +230,11 @@
},
{
"cell_type": "markdown",
"metadata": {},
"metadata": {
"pycharm": {
"name": "#%% md\n"
}
},
"source": [
"## Graphs\n",
"This section produces graphs from the collected data."
@ -391,6 +397,162 @@
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false
},
"source": [
"### More than 2 connections evaluation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n",
" '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '4x2MB Connections (proxied)': StandardTest([2,2,2,2]),\n",
" '3x2MB Connections (proxied)': StandardTest([2,2,2]),\n",
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false
},
"source": [
"### Mixed Performance Evaluation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
" '1x1MB + 1x2MB Connections (proxied)': StandardTest([1,2]),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false
},
"source": [
"### Eventful Evaluation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" 'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false
},
"source": [
"### Comparisons to a Direct Connection"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '1x2MB Connection (not proxied)': DirectTest(2),\n",
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
" '1x1MB Connection (not proxied)': DirectTest(1),\n",
"\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"plot_iperf_results(\n",
" {\n",
" '1x4MB Connection (not proxied)': DirectTest(4),\n",
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
" '1x2MB Connection (not proxied)': DirectTest(2),\n",
"\n",
" },\n",
" filename='png',\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
@ -399,9 +561,7 @@
"name": "#%% md\n"
}
},
"source": [
"### More than 2 connections evaluation"
]
"source": []
},
{
"cell_type": "code",

15
structure/structure.py
View File

@ -381,6 +381,7 @@ class StandardIperfResult:
# list containing an exact time and a value
self.data: List[Tuple[float, float]] = []
self.num_tests = 0
self.add_results(iperf)
def add_results(self, iperf: str):
@ -397,6 +398,8 @@ class StandardIperfResult:
):
self.data.append((time, result))
self.num_tests += 1
def bins(self) -> List[List[Tuple[float, float]]]:
bins: List[List[Tuple[float, float]]] = [[] for _ in np.arange(0, self.test.duration, self.interval_size)]
@ -432,12 +435,13 @@ class StandardIperfResult:
def repeat_until_satisfied(reducer, satisfied, initial=None, max_attempts=100, max_failures=3):
val = initial()
i = 0
val = initial
for i in range(max_attempts):
for j in range(max_failures):
try:
val = reducer(val)
break
except Exception as e:
print('failed with {}'.format(e))
if j == max_failures - 1:
@ -496,9 +500,14 @@ class BaseEnvironment:
old.add_results(iperf)
return old
def test_satisfier(val: StandardIperfResult) -> bool:
if val.num_tests < 3:
return False
return False not in [x < test.variation_target for x in val.coefficient_variance().values()]
result = repeat_until_satisfied(
test_reducer,
lambda x: max(x.coefficient_variance().values()) < test.variation_target,
test_satisfier,
max_failures=test.max_failures,
max_attempts=test.max_attempts,
)