functioning automatic testing
This commit is contained in:
parent
c5513d4aba
commit
165bb130d6
324
evaluation.ipynb
324
evaluation.ipynb
@ -1,5 +1,27 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Project Evaluation\n",
|
||||
"\n",
|
||||
"This file interfaces with a Proxmox server to automatically generate VM structures and graphs for testing the\n",
|
||||
"success criteria of my project."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"pycharm": {
|
||||
"name": "#%% md\n"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"## Setup\n",
|
||||
"This section sets up the required variables for the Proxmox server."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
@ -8,6 +30,7 @@
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import ipaddress\n",
|
||||
"import threading\n",
|
||||
"\n",
|
||||
"import runners\n",
|
||||
"from structure import Bridge\n",
|
||||
@ -18,6 +41,18 @@
|
||||
"%dotenv"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"pycharm": {
|
||||
"name": "#%% md\n"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"## Testing\n",
|
||||
"This section gathers the required data from the different structures for later graphs."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
@ -41,7 +76,6 @@
|
||||
" internet_bridge=os.getenv('INTERNET_BRIDGE'),\n",
|
||||
"\n",
|
||||
" management_bridge=os.getenv('MANAGEMENT_BRIDGE'),\n",
|
||||
" management_gateway=ipaddress.ip_address(os.getenv('MANAGEMENT_GATEWAY')),\n",
|
||||
" management_initial_ip=ipaddress.ip_address(os.getenv('MANAGEMENT_INITIAL_IP')),\n",
|
||||
")\n",
|
||||
"\n",
|
||||
@ -49,7 +83,10 @@
|
||||
" 'access_key': os.getenv('S3_ACCESS_KEY'),\n",
|
||||
" 'secret_key': os.getenv('S3_SECRET_KEY'),\n",
|
||||
" 'branch': os.getenv('TARGET_BRANCH'),\n",
|
||||
"}"
|
||||
"}\n",
|
||||
"\n",
|
||||
"directionInbound = {}\n",
|
||||
"directionOutbound = {}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -64,8 +101,49 @@
|
||||
"source": [
|
||||
"rp = RemotePortal([Interface(IpMethod.Auto4)], setup_params=setup_params)\n",
|
||||
"lp = LocalPortal([\n",
|
||||
" Interface(IpMethod.Auto4, limit=1),\n",
|
||||
" Interface(IpMethod.Auto4, limit=1),\n",
|
||||
" Interface(IpMethod.Auto4),\n",
|
||||
"], None, setup_params=setup_params)\n",
|
||||
"\n",
|
||||
"rp.set_local_portal(lp)\n",
|
||||
"lp.set_remote_portal(rp)\n",
|
||||
"\n",
|
||||
"top_level_bridge = Bridge(*[\n",
|
||||
" rp.get_interfaces()[0],\n",
|
||||
" lp.get_interfaces()[0],\n",
|
||||
"])\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" runner.build(top_level_bridge)\n",
|
||||
"\n",
|
||||
" lp.get_interfaces()[0].set_rate(1)\n",
|
||||
" lp.speedtest_server()\n",
|
||||
" directionInbound['One1MBNotProxied'] = rp.speedtest_client(lp.get_interfaces()[0].get_address())\n",
|
||||
" rp.speedtest_server()\n",
|
||||
" directionOutbound['One1MBNotProxied'] = lp.speedtest_client(rp.get_interfaces()[0].get_address())\n",
|
||||
"\n",
|
||||
" lp.get_interfaces()[0].set_rate(2)\n",
|
||||
" lp.speedtest_server()\n",
|
||||
" directionInbound['One2MBNotProxied'] = rp.speedtest_client(lp.get_interfaces()[0].get_address())\n",
|
||||
" rp.speedtest_server()\n",
|
||||
" directionOutbound['One2MBNotProxied'] = lp.speedtest_client(rp.get_interfaces()[0].get_address())\n",
|
||||
"finally:\n",
|
||||
" runner.teardown()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"rp = RemotePortal([Interface(IpMethod.Auto4)], setup_params=setup_params)\n",
|
||||
"lp = LocalPortal([\n",
|
||||
" Interface(IpMethod.Auto4),\n",
|
||||
" Interface(IpMethod.Auto4),\n",
|
||||
"], None, setup_params=setup_params)\n",
|
||||
"\n",
|
||||
"rp.set_local_portal(lp)\n",
|
||||
@ -75,10 +153,136 @@
|
||||
" rp.get_interfaces()[0],\n",
|
||||
" *lp.get_interfaces()[0:2],\n",
|
||||
"])\n",
|
||||
"runner.build(top_level_bridge)\n",
|
||||
"\n",
|
||||
"# Clean up\n",
|
||||
"runner.teardown()"
|
||||
"try:\n",
|
||||
" runner.build(top_level_bridge)\n",
|
||||
"\n",
|
||||
" lp.get_interfaces()[0].set_rate(1)\n",
|
||||
" lp.get_interfaces()[1].set_rate(1)\n",
|
||||
"\n",
|
||||
" lp.speedtest_server()\n",
|
||||
" directionInbound['Two1MBProxied'] = rp.speedtest_client('172.19.152.3')\n",
|
||||
" rp.speedtest_server()\n",
|
||||
" directionOutbound['Two1MBProxied'] = lp.speedtest_client('172.19.152.2')\n",
|
||||
"\n",
|
||||
" lp.get_interfaces()[0].set_rate(2)\n",
|
||||
" lp.get_interfaces()[1].set_rate(2)\n",
|
||||
"\n",
|
||||
" lp.speedtest_server()\n",
|
||||
" directionInbound['Two2MBProxied'] = rp.speedtest_client('172.19.152.3')\n",
|
||||
" rp.speedtest_server()\n",
|
||||
" directionOutbound['Two2MBProxied'] = lp.speedtest_client('172.19.152.2')\n",
|
||||
"\n",
|
||||
" lp.get_interfaces()[0].set_rate(1)\n",
|
||||
" lp.get_interfaces()[1].set_rate(2)\n",
|
||||
"\n",
|
||||
" lp.speedtest_server()\n",
|
||||
" directionInbound['One1MBOne2MBProxied'] = rp.speedtest_client('172.19.152.3')\n",
|
||||
" rp.speedtest_server()\n",
|
||||
" directionOutbound['One1MBOne2MBProxied'] = lp.speedtest_client('172.19.152.2')\n",
|
||||
"\n",
|
||||
" lp.get_interfaces()[0].set_rate(2)\n",
|
||||
" lp.get_interfaces()[1].set_rate(2)\n",
|
||||
"\n",
|
||||
" lp.speedtest_server()\n",
|
||||
" threading.Timer(5+15, lambda: lp.get_interfaces()[1].set_rate(1)).start()\n",
|
||||
" threading.Timer(5+30, lambda: lp.get_interfaces()[1].set_rate(2)).start()\n",
|
||||
"\n",
|
||||
" directionInbound['One2MBOneYMBProxiedSlow15Return30'] = rp.speedtest_client('172.19.152.3', time=60)\n",
|
||||
" rp.speedtest_server()\n",
|
||||
" directionOutbound['One2MBOneYMBProxiedSlow15Return30'] = lp.speedtest_client('172.19.152.2', time=60)\n",
|
||||
"finally:\n",
|
||||
" runner.teardown()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"rp = RemotePortal([Interface(IpMethod.Auto4)], setup_params=setup_params)\n",
|
||||
"lp = LocalPortal([\n",
|
||||
" Interface(IpMethod.Auto4),\n",
|
||||
" Interface(IpMethod.Auto4),\n",
|
||||
" Interface(IpMethod.Auto4),\n",
|
||||
"], None, setup_params=setup_params)\n",
|
||||
"\n",
|
||||
"rp.set_local_portal(lp)\n",
|
||||
"lp.set_remote_portal(rp)\n",
|
||||
"\n",
|
||||
"top_level_bridge = Bridge(*[\n",
|
||||
" rp.get_interfaces()[0],\n",
|
||||
" *lp.get_interfaces()[0:3],\n",
|
||||
"])\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" runner.build(top_level_bridge)\n",
|
||||
"\n",
|
||||
" lp.get_interfaces()[0].set_rate(1)\n",
|
||||
" lp.get_interfaces()[1].set_rate(1)\n",
|
||||
" lp.get_interfaces()[2].set_rate(1)\n",
|
||||
"\n",
|
||||
" lp.speedtest_server()\n",
|
||||
" directionInbound['Three1MBProxied'] = rp.speedtest_client('172.19.152.3')\n",
|
||||
" rp.speedtest_server()\n",
|
||||
" directionOutbound['Three1MBProxied'] = lp.speedtest_client('172.19.152.2')\n",
|
||||
"finally:\n",
|
||||
" runner.teardown()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"rp = RemotePortal([Interface(IpMethod.Auto4)], setup_params=setup_params)\n",
|
||||
"lp = LocalPortal([\n",
|
||||
" Interface(IpMethod.Auto4),\n",
|
||||
" Interface(IpMethod.Auto4),\n",
|
||||
" Interface(IpMethod.Auto4),\n",
|
||||
" Interface(IpMethod.Auto4),\n",
|
||||
"], None, setup_params=setup_params)\n",
|
||||
"\n",
|
||||
"rp.set_local_portal(lp)\n",
|
||||
"lp.set_remote_portal(rp)\n",
|
||||
"\n",
|
||||
"top_level_bridge = Bridge(*[\n",
|
||||
" rp.get_interfaces()[0],\n",
|
||||
" *lp.get_interfaces()[0:4],\n",
|
||||
"])\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" runner.build(top_level_bridge)\n",
|
||||
"\n",
|
||||
" lp.get_interfaces()[0].set_rate(1)\n",
|
||||
" lp.get_interfaces()[1].set_rate(1)\n",
|
||||
" lp.get_interfaces()[2].set_rate(1)\n",
|
||||
" lp.get_interfaces()[3].set_rate(1)\n",
|
||||
"\n",
|
||||
" lp.speedtest_server()\n",
|
||||
" directionInbound['Four1MBProxied'] = rp.speedtest_client('172.19.152.3')\n",
|
||||
" rp.speedtest_server()\n",
|
||||
" directionOutbound['Four1MBProxied'] = lp.speedtest_client('172.19.152.2')\n",
|
||||
"finally:\n",
|
||||
" runner.teardown()\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Graphs\n",
|
||||
"This section produces graphs from the collected data."
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -127,34 +331,15 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
"name": "#%% md\n"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Manual results\n",
|
||||
"# sudo iperf3 -c X.X.X.X -t 30 -O 5 -J\n",
|
||||
"\n",
|
||||
"import json\n",
|
||||
"\n",
|
||||
"def load_json_from_file(path):\n",
|
||||
" with open(path, 'r') as f:\n",
|
||||
" return json.loads(f.read())\n",
|
||||
"\n",
|
||||
"One1MBStraight = load_json_from_file('manual/One1MBStraight.json')\n",
|
||||
"One2MBStraight = load_json_from_file('manual/One2MBStraight.json')\n",
|
||||
"\n",
|
||||
"Two1MBAggregate = load_json_from_file('manual/Two1MBAggregate.json')\n",
|
||||
"Two2MBAggregate = load_json_from_file('manual/Two2MBAggregate.json')\n",
|
||||
"\n",
|
||||
"One1MBOne2MBAggregate = load_json_from_file('manual/One1MBOne2MBAggregate.json')\n",
|
||||
"\n",
|
||||
"# A 60 second long test\n",
|
||||
"Two2MBAggregateKillOneRecoverOne = load_json_from_file('manual/Two2MBAggregateKillOneRecoverOne.json')\n"
|
||||
"### Equal Connection Scaling\n",
|
||||
"This section shows equal connections scaling at various speeds and number of connections."
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -169,10 +354,10 @@
|
||||
"source": [
|
||||
"plot_iperf_results(\n",
|
||||
" {\n",
|
||||
" '1x1MBps connection (not proxied)': One1MBStraight,\n",
|
||||
" '2x1MBps connections (proxied)': Two1MBAggregate,\n",
|
||||
" '2x1MBps Connections (proxied)': directionInbound['Two1MBProxied'],\n",
|
||||
" '1x1MBps Connection (not proxied)': directionInbound['One1MBNotProxied'],\n",
|
||||
" },\n",
|
||||
" 'Proxying adds additional bandwidth',\n",
|
||||
" 'Two Equal 1MB Connections',\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
@ -188,12 +373,10 @@
|
||||
"source": [
|
||||
"plot_iperf_results(\n",
|
||||
" {\n",
|
||||
" '1x1MBps connection (not proxied)': One1MBStraight,\n",
|
||||
" '2x1MBps connections (proxied)': Two1MBAggregate,\n",
|
||||
" '1x2MBps connection (not proxied)': One2MBStraight,\n",
|
||||
" '2x2MBps connections (proxied)': Two2MBAggregate,\n",
|
||||
" '2x2MBps Connections (proxied)': directionInbound['Two2MBProxied'],\n",
|
||||
" '1x2MBps Connection (not proxied)': directionInbound['One2MBNotProxied'],\n",
|
||||
" },\n",
|
||||
" 'Proxing bandwidth scaling/overhead',\n",
|
||||
" 'Two Equal 2MB Connections',\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
@ -209,11 +392,43 @@
|
||||
"source": [
|
||||
"plot_iperf_results(\n",
|
||||
" {\n",
|
||||
" '2x1MBps connections (proxied)': Two1MBAggregate,\n",
|
||||
" '1x1MBps+1x2MBps connections (proxied)': One1MBOne2MBAggregate,\n",
|
||||
" '2x2MBps connections (proxied)': Two2MBAggregate,\n",
|
||||
" '4x1MBps Connections (proxied)': directionInbound['Four1MBProxied'],\n",
|
||||
" '3x1MBps Connections (proxied)': directionInbound['Three1MBProxied'],\n",
|
||||
" '2x1MBps Connections (proxied)': directionInbound['Two1MBProxied'],\n",
|
||||
" },\n",
|
||||
" 'Imbalanced connections add',\n",
|
||||
" 'More Equal Connections',\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"pycharm": {
|
||||
"name": "#%% md\n"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"### Mixed Connections Scaling\n",
|
||||
"This section shows mixed connections at various speeds with various events."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"plot_iperf_results(\n",
|
||||
" {\n",
|
||||
" '2x2MBps Connections (proxied)': directionInbound['Two2MBProxied'],\n",
|
||||
" '1x1MBps + 1x2MBps Connections (proxied)': directionInbound['One1MBOne2MBProxied'],\n",
|
||||
" '2x1MBps Connections (proxied)': directionInbound['Two1MBProxied'],\n",
|
||||
" },\n",
|
||||
" 'Mixed Speed Connections',\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
@ -229,22 +444,29 @@
|
||||
"source": [
|
||||
"plot_iperf_results(\n",
|
||||
" {\n",
|
||||
" '1x2MBps+1xYMBps connections (proxied)': Two2MBAggregateKillOneRecoverOne,\n",
|
||||
" '1x2MBps + 1xYMBps Connections (proxied)': directionInbound['One2MBOneYMBProxiedSlow15Return30'],\n",
|
||||
" },\n",
|
||||
" 'Killed connection',\n",
|
||||
" events={15: 'Y = 0', 40: 'Y = 2'},\n",
|
||||
" filename='graph4.png',\n",
|
||||
" 'Network Slow',\n",
|
||||
" events={0: 'Y=2', 15: 'Y=1', 30: 'Y=2'}\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"pycharm": {
|
||||
"name": "#%% md\n"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"## Criteria\n",
|
||||
"This section automatically verifies some criteria with assertions."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
},
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
|
||||
@ -1,8 +1,10 @@
|
||||
import concurrent.futures
|
||||
import ipaddress
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Callable, List, Tuple
|
||||
from typing import Callable, List, Tuple, Optional, Union
|
||||
from urllib.parse import quote
|
||||
|
||||
import proxmoxer
|
||||
@ -97,7 +99,7 @@ class ProxmoxRunner:
|
||||
|
||||
management_bridge: str,
|
||||
management_initial_ip: ipaddress,
|
||||
management_gateway: ipaddress,
|
||||
management_netmask: int = 24,
|
||||
|
||||
verify_ssl: bool = False,
|
||||
):
|
||||
@ -125,7 +127,7 @@ class ProxmoxRunner:
|
||||
self._management_bridge = structure.Bridge()
|
||||
self._management_bridge.set_name(management_bridge)
|
||||
self._management_initial_ip = management_initial_ip
|
||||
self._management_gateway = management_gateway
|
||||
self._management_netmask = management_netmask
|
||||
|
||||
# generate a single use SSH key (we can use any with Proxmox)
|
||||
self._private_key = paramiko.RSAKey.generate(3072)
|
||||
@ -136,13 +138,16 @@ class ProxmoxRunner:
|
||||
|
||||
self._build_bridges()
|
||||
|
||||
for node in nodes:
|
||||
self._build_node(node)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
|
||||
build_futures = [executor.submit(self._build_node, node) for node in nodes]
|
||||
for future in build_futures:
|
||||
future.result()
|
||||
|
||||
# guarantee that setup is not called until all of the nodes are built
|
||||
# this means that all will have their final IPs by this point
|
||||
for node in nodes:
|
||||
self._setup_node(node)
|
||||
# guarantee that setup is not called until all of the nodes are built
|
||||
# this means that all will have their final IPs by this point
|
||||
setup_futures = [executor.submit(self._setup_node, node) for node in nodes]
|
||||
for future in setup_futures:
|
||||
future.result()
|
||||
|
||||
def _await_task(self, upid, timeout=10):
|
||||
t1 = datetime.now()
|
||||
@ -228,7 +233,14 @@ class ProxmoxRunner:
|
||||
node.client.close()
|
||||
del node.client
|
||||
|
||||
def ssh(self, node: structure.Node, command: str, error_stderr=False, error_stdout=False) -> int:
|
||||
def ssh(
|
||||
self,
|
||||
node: structure.Node,
|
||||
command: str,
|
||||
error_stderr=False,
|
||||
error_stdout=False,
|
||||
return_stdout=False,
|
||||
) -> Union[int, str]:
|
||||
chan = node.client.get_transport().open_session()
|
||||
|
||||
chan.exec_command(command)
|
||||
@ -243,6 +255,12 @@ class ProxmoxRunner:
|
||||
if error_stdout:
|
||||
raise Exception(chan.recv(2048).decode())
|
||||
|
||||
if return_stdout is not False:
|
||||
if return_stdout is True:
|
||||
return chan.makefile().read()
|
||||
else:
|
||||
return chan.recv(return_stdout).decode()
|
||||
|
||||
return exit_status
|
||||
|
||||
def _build_node(self, node: structure.Node):
|
||||
@ -260,7 +278,7 @@ class ProxmoxRunner:
|
||||
interfaces = node.get_interfaces()
|
||||
internet_interface = structure.Interface(structure.IpMethod.Dhcp4)
|
||||
internet_interface.set_bridge(self._internet_bridge)
|
||||
temp_interfaces = [internet_interface, interfaces[len(interfaces)-1]]
|
||||
temp_interfaces = [internet_interface, interfaces[len(interfaces) - 1]]
|
||||
|
||||
self._setup_node_interfaces(node, temp_interfaces)
|
||||
|
||||
@ -272,7 +290,7 @@ class ProxmoxRunner:
|
||||
self._close_ssh(node)
|
||||
|
||||
stop_task = self._proxmox.nodes(self._proxmox_node).qemu(node.get_id()).status.shutdown.post()
|
||||
self._await_task(stop_task)
|
||||
self._await_task(stop_task, timeout=20)
|
||||
|
||||
# Step 3: connect to management bridge for final setup
|
||||
self._setup_node_interfaces(node)
|
||||
@ -281,6 +299,7 @@ class ProxmoxRunner:
|
||||
self._await_task(start_task)
|
||||
|
||||
self._open_ssh(node)
|
||||
node.ssh = (lambda n: lambda *args, **kwargs: self.ssh(n, *args, **kwargs))(node)
|
||||
|
||||
def _setup_node_interfaces(self, node: structure.Node, interfaces: List[structure.Interface] = None):
|
||||
if interfaces is None:
|
||||
@ -297,7 +316,7 @@ class ProxmoxRunner:
|
||||
interface.set_bridge(self._management_bridge)
|
||||
addr = self._management_initial_ip + node.get_id() - self._initial_vm_id
|
||||
|
||||
kwargs['ipconfig{}'.format(i)] = 'ip={}/24,gw={}'.format(addr, self._management_gateway)
|
||||
kwargs['ipconfig{}'.format(i)] = 'ip={}/{}'.format(addr, self._management_netmask)
|
||||
interface.set_address(addr)
|
||||
elif method == structure.IpMethod.Auto4:
|
||||
bridge = interface.get_bridge()
|
||||
@ -311,9 +330,39 @@ class ProxmoxRunner:
|
||||
raise RuntimeError('not implemented')
|
||||
|
||||
kwargs['net{}'.format(i)] = 'model=virtio,bridge={}'.format(interface.get_bridge().get_name())
|
||||
if interface.get_rate() is not None:
|
||||
kwargs['net{}'.format(i)] += ',rate={}'.format(interface.get_rate())
|
||||
|
||||
def interface_set_rate(iface):
|
||||
def new_set_rate(rate: Optional[int]):
|
||||
structure.Interface.set_rate(iface, rate)
|
||||
self._update_node_interfaces(node)
|
||||
|
||||
return new_set_rate
|
||||
|
||||
interface.set_rate = interface_set_rate(interface)
|
||||
|
||||
self._proxmox.nodes(self._proxmox_node).qemu(node.get_id()).config.put(**kwargs)
|
||||
|
||||
def _update_node_interfaces(self, node: structure.Node):
|
||||
interfaces = node.get_interfaces()
|
||||
|
||||
old_config = self._proxmox.nodes(self._proxmox_node).qemu(node.get_id()).config.get()
|
||||
old_digest = old_config['digest']
|
||||
old_config = {k: v for (k, v) in old_config.items() if k[:3] == 'net'}
|
||||
|
||||
rate_regex = re.compile(r',rate=(\d+(?:\.\d+)?)')
|
||||
|
||||
new_config = {'digest': old_digest}
|
||||
for k, v in old_config.items():
|
||||
index = int(k[3:])
|
||||
iface = interfaces[index]
|
||||
new_config[k] = rate_regex.sub('', v)
|
||||
if iface.get_rate() is not None:
|
||||
new_config[k] += ',rate={}'.format(iface.get_rate())
|
||||
|
||||
self._proxmox.nodes(self._proxmox_node).qemu(node.get_id()).config.put(**new_config)
|
||||
|
||||
def _setup_node(self, node: structure.Node):
|
||||
if node.get_setup() is not None:
|
||||
self.ssh(node, node.get_setup(), error_stdout=True, error_stderr=True)
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import ipaddress
|
||||
import json
|
||||
import textwrap
|
||||
from enum import Enum
|
||||
import random
|
||||
@ -15,15 +16,15 @@ class IpMethod(Enum):
|
||||
|
||||
|
||||
class Interface:
|
||||
def __init__(self, method: IpMethod, limit: Optional[int] = None):
|
||||
def __init__(self, method: IpMethod, rate: Optional[int] = None):
|
||||
self._method: IpMethod
|
||||
|
||||
self._node: Optional[Node] = None
|
||||
self._limit: Optional[int] = None
|
||||
self._rate: Optional[int] = None
|
||||
self._bridge: Optional[Bridge] = None
|
||||
|
||||
self._method = method
|
||||
self._limit = limit
|
||||
self._rate = rate
|
||||
self._address: ipaddress.ip_address = None
|
||||
|
||||
def get_method(self):
|
||||
@ -47,6 +48,12 @@ class Interface:
|
||||
def get_address(self) -> ipaddress.ip_address:
|
||||
return self._address
|
||||
|
||||
def get_rate(self) -> Optional[int]:
|
||||
return self._rate
|
||||
|
||||
def set_rate(self, rate: Optional[int]):
|
||||
self._rate = rate
|
||||
|
||||
|
||||
class Bridge:
|
||||
def __init__(self, *interfaces: Interface):
|
||||
@ -78,6 +85,9 @@ class Bridge:
|
||||
def get_ip_address(self) -> ipaddress.ip_address:
|
||||
return next(self._network_iterator)
|
||||
|
||||
def get_network(self) -> str:
|
||||
return str(ipaddress.ip_network('{}/{}'.format(self._addr, self.netmask), False))
|
||||
|
||||
|
||||
class Node:
|
||||
def __init__(self, interfaces: List[Interface], setup_params: Dict = None):
|
||||
@ -111,6 +121,9 @@ class Node:
|
||||
def get_setup(self) -> Optional[str]:
|
||||
return None
|
||||
|
||||
def ssh(self, *args, **kwargs):
|
||||
raise RuntimeError('ssh not implemented')
|
||||
|
||||
|
||||
class SpeedTestServer(Node):
|
||||
def client(self, server: Interface):
|
||||
@ -144,17 +157,19 @@ class RemotePortal(Node):
|
||||
./minio-client alias set s3 s3.us-west-001.backblazeb2.com {access_key} {secret_key}
|
||||
./minio-client cp s3/dissertation/binaries/debian/{branch} mpbl3p
|
||||
|
||||
cloud-init status --wait
|
||||
sudo apt-get install -y iperf3
|
||||
|
||||
chmod +x mpbl3p
|
||||
|
||||
cloud-init status --wait || cloud-init status --long
|
||||
sudo apt-get install -y iperf3
|
||||
''').format(**self.setup_params)
|
||||
|
||||
def get_setup(self) -> Optional[str]:
|
||||
return textwrap.dedent('''
|
||||
set -e
|
||||
|
||||
sudo sysctl -w net.ipv4.conf.all.arp_announce=1
|
||||
sudo sysctl -w net.ipv4.conf.all.arp_ignore=2
|
||||
|
||||
cat << EOF > config.ini
|
||||
[Host]
|
||||
PrivateKey = INVALID
|
||||
@ -166,11 +181,27 @@ class RemotePortal(Node):
|
||||
LocalHost = {local_host}
|
||||
LocalPort = 1234
|
||||
EOF
|
||||
|
||||
(nohup sudo ./mpbl3p > mpbl3p.log 2>&1 & echo $! > mpbl3p.pid)
|
||||
|
||||
sleep 1
|
||||
sudo ip link set up nc0
|
||||
sudo ip addr add 172.19.152.2/31 dev nc0
|
||||
|
||||
ps $(cat mpbl3p.pid)
|
||||
''').format(
|
||||
local_host=self.get_interfaces()[0].get_address(),
|
||||
**self.setup_params,
|
||||
)
|
||||
|
||||
def speedtest_server(self):
|
||||
self.ssh('iperf3 -s -1 -D', error_stdout=True, error_stderr=True)
|
||||
|
||||
def speedtest_client(self, target, time=30):
|
||||
command = 'iperf3 -c {target} -t {time} -O 5 -J'.format(target=target, time=time)
|
||||
out = self.ssh(command, error_stdout=True, error_stderr=True, return_stdout=True)
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
class LocalPortal(Node):
|
||||
def __init__(self, wan_interfaces: List[Interface], child: Optional[Node], **kwargs):
|
||||
@ -197,11 +228,10 @@ class LocalPortal(Node):
|
||||
./minio-client alias set s3 s3.us-west-001.backblazeb2.com {access_key} {secret_key}
|
||||
./minio-client cp s3/dissertation/binaries/debian/{branch} mpbl3p
|
||||
|
||||
cloud-init status --wait
|
||||
sudo apt-get install -y iperf3
|
||||
|
||||
chmod +x mpbl3p
|
||||
|
||||
cloud-init status --wait || cloud-init status --long
|
||||
sudo apt-get install -y iperf3
|
||||
''').format(**self.setup_params)
|
||||
|
||||
def get_setup(self) -> str:
|
||||
@ -221,13 +251,47 @@ class LocalPortal(Node):
|
||||
remote_host=self.remote_portal.get_interfaces()[0].get_address(),
|
||||
) for x in self.get_interfaces()[:-1]])
|
||||
|
||||
policy_routing_string = textwrap.dedent('''
|
||||
sudo ip route flush {table_number}
|
||||
sudo ip route add table {table_number} to {network} dev {device}
|
||||
sudo ip rule add from {local_address} table {table_number} priority {table_number}
|
||||
''')
|
||||
|
||||
policy_routing = '\n\n'.join([policy_routing_string.format(
|
||||
table_number=i+10,
|
||||
device='eth{}'.format(i),
|
||||
network=iface.get_bridge().get_network(),
|
||||
local_address=iface.get_address(),
|
||||
) for i, iface in enumerate(self.get_interfaces()[:-1])])
|
||||
|
||||
return textwrap.dedent('''
|
||||
set -e
|
||||
|
||||
sudo sysctl -w net.ipv4.conf.all.arp_announce=1
|
||||
sudo sysctl -w net.ipv4.conf.all.arp_ignore=2
|
||||
|
||||
{policy_routing}
|
||||
|
||||
cat << EOF > config.ini
|
||||
[Host]
|
||||
PrivateKey = INVALID
|
||||
|
||||
{peers}
|
||||
EOF
|
||||
''').format(**self.setup_params, peers=peers)
|
||||
|
||||
(nohup sudo ./mpbl3p > mpbl3p.log 2>&1 & echo $! > mpbl3p.pid)
|
||||
|
||||
sleep 1
|
||||
sudo ip link set up nc0
|
||||
sudo ip addr add 172.19.152.3/31 dev nc0
|
||||
|
||||
ps $(cat mpbl3p.pid)
|
||||
''').format(**self.setup_params, peers=peers, policy_routing=policy_routing)
|
||||
|
||||
def speedtest_server(self):
|
||||
self.ssh('iperf3 -s -1 -D', error_stdout=True, error_stderr=True)
|
||||
|
||||
def speedtest_client(self, target, time=30):
|
||||
command = 'iperf3 -c {target} -t {time} -O 5 -J'.format(target=target, time=time)
|
||||
out = self.ssh(command, error_stdout=True, error_stderr=True, return_stdout=True)
|
||||
return json.loads(out)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user