Merge pull request 'direct-fixes' (#6) from direct-fixes into develop
Reviewed-on: #6
This commit is contained in:
commit
ccff05c8d2
335
evaluation.ipynb
335
evaluation.ipynb
@ -91,15 +91,14 @@
|
|||||||
" (directionInbound[test.name()], directionOutbound[test.name()]) = env.test(test)\n",
|
" (directionInbound[test.name()], directionOutbound[test.name()]) = env.test(test)\n",
|
||||||
"\n",
|
"\n",
|
||||||
"def attempt_n_times(foo, n=3):\n",
|
"def attempt_n_times(foo, n=3):\n",
|
||||||
" i = 0\n",
|
" for i in range(n):\n",
|
||||||
" while True:\n",
|
|
||||||
" try:\n",
|
" try:\n",
|
||||||
" return foo()\n",
|
" return foo()\n",
|
||||||
|
" except KeyboardInterrupt as e:\n",
|
||||||
|
" raise e\n",
|
||||||
" except Exception as e:\n",
|
" except Exception as e:\n",
|
||||||
" if i == n:\n",
|
" if i == n - 1:\n",
|
||||||
" raise e\n",
|
" raise e\n",
|
||||||
" finally:\n",
|
|
||||||
" i += 1\n",
|
|
||||||
"\n",
|
"\n",
|
||||||
"\n",
|
"\n",
|
||||||
"fast_tests = True"
|
"fast_tests = True"
|
||||||
@ -114,7 +113,7 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"source": [
|
"source": [
|
||||||
"### Direct Server to Server Testing"
|
"### Direct Server to Server"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -132,7 +131,9 @@
|
|||||||
" run_and_save_test(env, DirectTest(1, variation_target=0.4 if fast_tests else 0.2))\n",
|
" run_and_save_test(env, DirectTest(1, variation_target=0.4 if fast_tests else 0.2))\n",
|
||||||
" run_and_save_test(env, DirectTest(2, variation_target=0.4 if fast_tests else 0.2))\n",
|
" run_and_save_test(env, DirectTest(2, variation_target=0.4 if fast_tests else 0.2))\n",
|
||||||
" run_and_save_test(env, DirectTest(3, variation_target=0.4 if fast_tests else 0.2))\n",
|
" run_and_save_test(env, DirectTest(3, variation_target=0.4 if fast_tests else 0.2))\n",
|
||||||
" run_and_save_test(env, DirectTest(4, variation_target=0.4 if fast_tests else 0.2))"
|
" run_and_save_test(env, DirectTest(4, variation_target=0.4 if fast_tests else 0.2))\n",
|
||||||
|
"\n",
|
||||||
|
"attempt_n_times(direct_tests)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -228,7 +229,11 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {
|
||||||
|
"pycharm": {
|
||||||
|
"name": "#%% md\n"
|
||||||
|
}
|
||||||
|
},
|
||||||
"source": [
|
"source": [
|
||||||
"## Graphs\n",
|
"## Graphs\n",
|
||||||
"This section produces graphs from the collected data."
|
"This section produces graphs from the collected data."
|
||||||
@ -391,6 +396,316 @@
|
|||||||
")"
|
")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"collapsed": false
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"### More than 2 connections evaluation"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"pycharm": {
|
||||||
|
"name": "#%%\n"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"plot_iperf_results(\n",
|
||||||
|
" {\n",
|
||||||
|
" '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n",
|
||||||
|
" '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
|
||||||
|
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
|
||||||
|
" },\n",
|
||||||
|
" filename='png',\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"pycharm": {
|
||||||
|
"name": "#%%\n"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"plot_iperf_results(\n",
|
||||||
|
" {\n",
|
||||||
|
" '4x2MB Connections (proxied)': StandardTest([2,2,2,2]),\n",
|
||||||
|
" '3x2MB Connections (proxied)': StandardTest([2,2,2]),\n",
|
||||||
|
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
|
||||||
|
" },\n",
|
||||||
|
" filename='png',\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"collapsed": false
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"### Mixed Performance Evaluation"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"pycharm": {
|
||||||
|
"name": "#%%\n"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"plot_iperf_results(\n",
|
||||||
|
" {\n",
|
||||||
|
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
|
||||||
|
" '1x1MB + 1x2MB Connections (proxied)': StandardTest([1,2]),\n",
|
||||||
|
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
|
||||||
|
" },\n",
|
||||||
|
" filename='png',\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"collapsed": false
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"### Eventful Evaluation"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"pycharm": {
|
||||||
|
"name": "#%%\n"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"plot_iperf_results(\n",
|
||||||
|
" {\n",
|
||||||
|
" 'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n",
|
||||||
|
" },\n",
|
||||||
|
" filename='png',\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"collapsed": false
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"### Comparisons to a Direct Connection"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"pycharm": {
|
||||||
|
"name": "#%%\n"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"plot_iperf_results(\n",
|
||||||
|
" {\n",
|
||||||
|
" '1x2MB Connection (not proxied)': DirectTest(2),\n",
|
||||||
|
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
|
||||||
|
" '1x1MB Connection (not proxied)': DirectTest(1),\n",
|
||||||
|
"\n",
|
||||||
|
" },\n",
|
||||||
|
" filename='png',\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"pycharm": {
|
||||||
|
"name": "#%%\n"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"plot_iperf_results(\n",
|
||||||
|
" {\n",
|
||||||
|
" '1x4MB Connection (not proxied)': DirectTest(4),\n",
|
||||||
|
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
|
||||||
|
" '1x2MB Connection (not proxied)': DirectTest(2),\n",
|
||||||
|
"\n",
|
||||||
|
" },\n",
|
||||||
|
" filename='png',\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"collapsed": false
|
||||||
|
},
|
||||||
|
"source": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"pycharm": {
|
||||||
|
"name": "#%%\n"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"plot_iperf_results(\n",
|
||||||
|
" {\n",
|
||||||
|
" '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n",
|
||||||
|
" '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
|
||||||
|
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
|
||||||
|
" },\n",
|
||||||
|
" filename='png',\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"pycharm": {
|
||||||
|
"name": "#%%\n"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"plot_iperf_results(\n",
|
||||||
|
" {\n",
|
||||||
|
" '4x2MB Connections (proxied)': StandardTest([2,2,2,2]),\n",
|
||||||
|
" '3x2MB Connections (proxied)': StandardTest([2,2,2]),\n",
|
||||||
|
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
|
||||||
|
" },\n",
|
||||||
|
" filename='png',\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"collapsed": false
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"### Mixed Performance Evaluation"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"pycharm": {
|
||||||
|
"name": "#%%\n"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"plot_iperf_results(\n",
|
||||||
|
" {\n",
|
||||||
|
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
|
||||||
|
" '1x1MB + 1x2MB Connections (proxied)': StandardTest([1,2]),\n",
|
||||||
|
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
|
||||||
|
" },\n",
|
||||||
|
" filename='png',\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"collapsed": false
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"### Eventful Evaluation"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"pycharm": {
|
||||||
|
"name": "#%%\n"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"plot_iperf_results(\n",
|
||||||
|
" {\n",
|
||||||
|
" 'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n",
|
||||||
|
" },\n",
|
||||||
|
" filename='png',\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"collapsed": false
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"### Comparisons to a Direct Connection"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"pycharm": {
|
||||||
|
"name": "#%%\n"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"plot_iperf_results(\n",
|
||||||
|
" {\n",
|
||||||
|
" '1x2MB Connection (not proxied)': DirectTest(2),\n",
|
||||||
|
" '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
|
||||||
|
" '1x1MB Connection (not proxied)': DirectTest(1),\n",
|
||||||
|
"\n",
|
||||||
|
" },\n",
|
||||||
|
" filename='png',\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"pycharm": {
|
||||||
|
"name": "#%%\n"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"plot_iperf_results(\n",
|
||||||
|
" {\n",
|
||||||
|
" '1x4MB Connection (not proxied)': DirectTest(4),\n",
|
||||||
|
" '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
|
||||||
|
" '1x2MB Connection (not proxied)': DirectTest(2),\n",
|
||||||
|
"\n",
|
||||||
|
" },\n",
|
||||||
|
" filename='png',\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {
|
"metadata": {
|
||||||
@ -399,9 +714,7 @@
|
|||||||
"name": "#%% md\n"
|
"name": "#%% md\n"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"source": [
|
"source": []
|
||||||
"### More than 2 connections evaluation"
|
|
||||||
]
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
|
@ -167,11 +167,15 @@ class SpeedTestServer(Node):
|
|||||||
def server(self):
|
def server(self):
|
||||||
self.ssh('iperf3 -s -1 -D', error_stdout=True, error_stderr=True)
|
self.ssh('iperf3 -s -1 -D', error_stdout=True, error_stderr=True)
|
||||||
|
|
||||||
def client(self, target, time=30):
|
def client(self, target, time=30, interval_size=2.0):
|
||||||
if isinstance(target, SpeedTestServer):
|
if isinstance(target, SpeedTestServer):
|
||||||
target = target.get_interfaces()[0].get_address()
|
target = target.get_interfaces()[0].get_address()
|
||||||
|
|
||||||
command = 'iperf3 -c {target} -t {time} -O 5 -J'.format(target=target, time=time)
|
command = 'iperf3 -c {target} -t {time} -O 6 -i {interval_size} -J'.format(
|
||||||
|
target=target,
|
||||||
|
time=time,
|
||||||
|
interval_size=interval_size,
|
||||||
|
)
|
||||||
return self.ssh(command, error_stdout=True, error_stderr=True, return_stdout=True)
|
return self.ssh(command, error_stdout=True, error_stderr=True, return_stdout=True)
|
||||||
|
|
||||||
|
|
||||||
@ -231,6 +235,7 @@ class RemotePortal(Node):
|
|||||||
sudo ip route add table 10 to {local_host} via 172.19.152.3 dev nc0
|
sudo ip route add table 10 to {local_host} via 172.19.152.3 dev nc0
|
||||||
sudo ip rule add to {local_host} table 10 priority 10
|
sudo ip rule add to {local_host} table 10 priority 10
|
||||||
|
|
||||||
|
ping -W 0.1 -c 1 172.19.152.3
|
||||||
ps $(cat mpbl3p.pid) || cat mpbl3p.log
|
ps $(cat mpbl3p.pid) || cat mpbl3p.log
|
||||||
''').format(
|
''').format(
|
||||||
local_host=self.get_interfaces()[0].get_address(),
|
local_host=self.get_interfaces()[0].get_address(),
|
||||||
@ -329,6 +334,7 @@ class LocalPortal(Node):
|
|||||||
sudo ip route add to {remote_host} dev {local_interface} table 19
|
sudo ip route add to {remote_host} dev {local_interface} table 19
|
||||||
sudo ip rule add to {remote_host} table 19 priority 19
|
sudo ip rule add to {remote_host} table 19 priority 19
|
||||||
|
|
||||||
|
ping -W 0.1 -c 1 172.19.152.2
|
||||||
ps $(cat mpbl3p.pid) || cat mpbl3p.log
|
ps $(cat mpbl3p.pid) || cat mpbl3p.log
|
||||||
''').format(
|
''').format(
|
||||||
**self.setup_params,
|
**self.setup_params,
|
||||||
@ -373,7 +379,7 @@ class DirectTest(StandardTest):
|
|||||||
|
|
||||||
|
|
||||||
class StandardIperfResult:
|
class StandardIperfResult:
|
||||||
def __init__(self, test: StandardTest, iperf: str, interval_size=1.0):
|
def __init__(self, test: StandardTest, iperf: str, interval_size=2.0):
|
||||||
self.test = test
|
self.test = test
|
||||||
|
|
||||||
self.interval_size = interval_size
|
self.interval_size = interval_size
|
||||||
@ -381,6 +387,7 @@ class StandardIperfResult:
|
|||||||
# list containing an exact time and a value
|
# list containing an exact time and a value
|
||||||
self.data: List[Tuple[float, float]] = []
|
self.data: List[Tuple[float, float]] = []
|
||||||
|
|
||||||
|
self.num_tests = 0
|
||||||
self.add_results(iperf)
|
self.add_results(iperf)
|
||||||
|
|
||||||
def add_results(self, iperf: str):
|
def add_results(self, iperf: str):
|
||||||
@ -397,6 +404,8 @@ class StandardIperfResult:
|
|||||||
):
|
):
|
||||||
self.data.append((time, result))
|
self.data.append((time, result))
|
||||||
|
|
||||||
|
self.num_tests += 1
|
||||||
|
|
||||||
def bins(self) -> List[List[Tuple[float, float]]]:
|
def bins(self) -> List[List[Tuple[float, float]]]:
|
||||||
bins: List[List[Tuple[float, float]]] = [[] for _ in np.arange(0, self.test.duration, self.interval_size)]
|
bins: List[List[Tuple[float, float]]] = [[] for _ in np.arange(0, self.test.duration, self.interval_size)]
|
||||||
|
|
||||||
@ -432,12 +441,13 @@ class StandardIperfResult:
|
|||||||
|
|
||||||
|
|
||||||
def repeat_until_satisfied(reducer, satisfied, initial=None, max_attempts=100, max_failures=3):
|
def repeat_until_satisfied(reducer, satisfied, initial=None, max_attempts=100, max_failures=3):
|
||||||
val = initial()
|
val = initial
|
||||||
i = 0
|
|
||||||
for i in range(max_attempts):
|
for i in range(max_attempts):
|
||||||
for j in range(max_failures):
|
for j in range(max_failures):
|
||||||
try:
|
try:
|
||||||
val = reducer(val)
|
val = reducer(val)
|
||||||
|
break
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print('failed with {}'.format(e))
|
print('failed with {}'.format(e))
|
||||||
if j == max_failures - 1:
|
if j == max_failures - 1:
|
||||||
@ -485,8 +495,8 @@ class BaseEnvironment:
|
|||||||
|
|
||||||
for t, (iface, rate) in test.events.items():
|
for t, (iface, rate) in test.events.items():
|
||||||
threading.Timer(
|
threading.Timer(
|
||||||
5 + t,
|
6 + t,
|
||||||
(lambda s: lambda: s.lp.get_interfaces()[iface].set_rate(rate))(self),
|
(lambda n: lambda: n.get_interfaces()[iface].set_rate(rate))(rated_node),
|
||||||
)
|
)
|
||||||
|
|
||||||
iperf = client.client(server, time=test.duration)
|
iperf = client.client(server, time=test.duration)
|
||||||
@ -496,9 +506,14 @@ class BaseEnvironment:
|
|||||||
old.add_results(iperf)
|
old.add_results(iperf)
|
||||||
return old
|
return old
|
||||||
|
|
||||||
|
def test_satisfier(val: StandardIperfResult) -> bool:
|
||||||
|
if val.num_tests < 3:
|
||||||
|
return False
|
||||||
|
return False not in [x < test.variation_target for x in val.coefficient_variance().values()]
|
||||||
|
|
||||||
result = repeat_until_satisfied(
|
result = repeat_until_satisfied(
|
||||||
test_reducer,
|
test_reducer,
|
||||||
lambda x: max(x.coefficient_variance().values()) < test.variation_target,
|
test_satisfier,
|
||||||
max_failures=test.max_failures,
|
max_failures=test.max_failures,
|
||||||
max_attempts=test.max_attempts,
|
max_attempts=test.max_attempts,
|
||||||
)
|
)
|
||||||
|
Loading…
Reference in New Issue
Block a user