commit 0200a21db45d9e1f98873053cbdf3e4cda8cf2b8 Author: Jake Hillion Date: Sun Oct 18 13:06:40 2020 +0100 Initial commit - Structure built - Proxmox bridge and VM creation Todo: - Proxmox VM networking - SSH diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..777ad6a --- /dev/null +++ b/.gitignore @@ -0,0 +1,236 @@ + +# Created by https://www.toptal.com/developers/gitignore/api/python,intellij+all,dotenv +# Edit at https://www.toptal.com/developers/gitignore?templates=python,intellij+all,dotenv + +### dotenv ### +.env + +### Intellij+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### Intellij+all Patch ### +# Ignores the whole .idea folder and all .iml files +# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 + +.idea/ + +# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 + +*.iml +modules.xml +.idea/misc.xml +*.ipr + +# Sonarlint plugin +.idea/sonarlint + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +pytestdebug.log + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ +doc/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pythonenv* + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# profiling data +.prof + +# End of https://www.toptal.com/developers/gitignore/api/python,intellij+all,dotenv diff --git a/README.md b/README.md new file mode 100644 index 0000000..7f269d0 --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +# Dissertation Evaluation + +A Python backing to partially automate producing the graphs for my dissertation using an iPython Notebook and a Proxmox server. + \ No newline at end of file diff --git a/evaluation.ipynb b/evaluation.ipynb new file mode 100644 index 0000000..7b306ce --- /dev/null +++ b/evaluation.ipynb @@ -0,0 +1,124 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import os\n", + "import ipaddress\n", + "\n", + "import runners\n", + "from structure import Bridge\n", + "from structure import SpeedTestServer, RemoteServer, LocalServer\n", + "from structure import Interface, IpMethod\n", + "\n", + "%load_ext dotenv\n", + "%dotenv" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "outputs": [], + "source": [ + "runner = runners.ProxmoxRunner(\n", + " host=os.getenv('PROXMOX_HOST'),\n", + " node=os.getenv('PROXMOX_NODE'),\n", + " user=os.getenv('PROXMOX_USER'),\n", + " token_name=os.getenv('PROXMOX_TOKEN_NAME'),\n", + " token_value=os.getenv('PROXMOX_TOKEN_VALUE'),\n", + "\n", + " template_id=9000,\n", + " initial_vm_id=21002,\n", + "\n", + " internet_bridge='vmbr2',\n", + "\n", + " management_bridge='vmbr4',\n", + " management_initial_ip=ipaddress.ip_address('10.21.12.2'),\n", + ")" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 5, + "outputs": [], + "source": [ + "g_st = SpeedTestServer([Interface(IpMethod.Auto4)])\n", + "l_st = SpeedTestServer([Interface(IpMethod.Dhcp4)])\n", + "\n", + "rs = RemoteServer([Interface(IpMethod.Auto4)])\n", + "ls = LocalServer([\n", + " Interface(IpMethod.Auto4, limit=1),\n", + " Interface(IpMethod.Auto4, limit=1),\n", + "], l_st)\n", + "\n", + "top_level_bridge = Bridge(*[\n", + " g_st.get_interfaces()[0],\n", + " rs.get_interfaces()[0],\n", + " *ls.get_interfaces()[0:2],\n", + "])\n", + "runner.build(top_level_bridge)\n", + "\n", + "# Test from the client to the global network via the proxy\n", + "g_st.server()\n", + "l_st.client(rs.get_interfaces()[0])\n", + "\n", + "# Test from the global network to the client via the proxy\n", + "g_st.server()\n", + "l_st.client(rs.get_interfaces()[0])\n", + "\n", + "# Clean up\n", + "runner.teardown()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/runners/__init__.py b/runners/__init__.py new file mode 100644 index 0000000..1a8747d --- /dev/null +++ b/runners/__init__.py @@ -0,0 +1 @@ +from .runners import PrintRunner, ProxmoxRunner diff --git a/runners/runners.py b/runners/runners.py new file mode 100644 index 0000000..bd19170 --- /dev/null +++ b/runners/runners.py @@ -0,0 +1,203 @@ +import ipaddress +import os +from datetime import datetime +from typing import Callable, List, Tuple + +import proxmoxer + +import structure + + +def check_env(*names: str) -> bool: + for name in names: + if name not in os.environ: + return False + return True + + +def bridge_node_dfs( + first: structure.Bridge, + bridge_name_generator: Callable[[structure.Bridge], str], + node_id_generator: Callable[[structure.Node], int], +) -> Tuple[List[structure.Bridge], List[structure.Node]]: + bridges: List[structure.Bridge] = [] + nodes: List[structure.Node] = [] + + queue: List[structure.Bridge] = [first] + while len(queue) > 0: + bridge = queue.pop() + if bridge.get_name() != '': + continue + + bridges.append(bridge) + bridge.set_name(bridge_name_generator(bridge)) + + # from this bridge, find all nodes (via all interfaces) + reachable_nodes: List[structure.Node] = [] + for interface in bridge.get_interfaces(): + node = interface.get_node() + if node.get_id() is not None: + continue + + node.set_id(node_id_generator(node)) + reachable_nodes.append(node) + nodes.append(node) + + # from each node, find all bridges (via all interfaces) + for node in reachable_nodes: + for interface in node.get_interfaces(): + bridge = interface.get_bridge() + if bridge is not None and bridge.get_name() == '': + queue.append(bridge) + + return bridges, nodes + + +class PrintRunner: + def __init__(self): + self._last_bridge: int = 0 + self._last_node_id = 0 + + def build(self, bridge: structure.Bridge): + bridges, nodes = bridge_node_dfs(bridge, lambda _: self.name_bridge(), lambda _: self.id_node()) + + print(bridges) + print(nodes) + + def teardown(self): + pass + + def name_bridge(self) -> str: + self._last_bridge += 1 + return 'fake{}'.format(self._last_bridge) + + def id_node(self) -> int: + self._last_node_id += 1 + return self._last_node_id + + +class ProxmoxRunner: + def __init__( + self, + host: str, + node: str, + user: str, + token_name: str, + token_value: str, + + template_id: int, + initial_vm_id: int, + + internet_bridge: str, + + management_bridge: str, + management_initial_ip: ipaddress, + + verify_ssl: bool = False, + ): + self._last_node_id = 0 + self._created_nodes: List[int] = [] + self._last_bridge = 0 + self._created_bridges: List[str] = [] + + self._proxmox = proxmoxer.ProxmoxAPI( + host, + user=user, + token_name=token_name, + token_value=token_value, + verify_ssl=verify_ssl, + ) + + self._proxmox_node = node + + self._template_id = template_id + self._initial_vm_id = initial_vm_id - 1 + + self._internet_bridge = internet_bridge + + self._management_bridge = management_bridge + self._management_initial_ip = management_initial_ip + + def build(self, bridge: structure.Bridge): + bridges, nodes = bridge_node_dfs(bridge, lambda x: self._create_bridge(x), lambda x: self._create_node(x)) + + self._build_bridges() + for node in nodes: + self._build_node(node) + + def _await_task(self, upid, timeout=10): + t1 = datetime.now() + while (datetime.now() - t1).seconds < timeout: + if self._proxmox.nodes(self._proxmox_node).tasks(upid).status.get()['status'] == 'stopped': + return + raise TimeoutError + + def _create_bridge(self, bridge: structure.Bridge) -> str: + self._last_bridge += 1 + + while True: + try: + self._proxmox.nodes(self._proxmox_node).network.post( + iface='vmbr{}'.format(self._last_bridge), + type='bridge', + autostart=1, + comments='Automatically created by Python evaluation', + ) + break + except proxmoxer.core.ResourceException as e: + if 'interface already exists' in str(e): + self._last_bridge += 1 + else: + raise e + + bridge_name = 'vmbr{}'.format(self._last_bridge) + self._created_bridges.append(bridge_name) + return bridge_name + + def _build_bridges(self): + network_task = self._proxmox.nodes(self._proxmox_node).network.put() + self._await_task(network_task) + + def _create_node(self, node: structure.Node) -> int: + self._last_node_id += 1 + + while True: + try: + clone_task = self._proxmox.nodes(self._proxmox_node).qemu(self._template_id).clone.post( + newid=self._initial_vm_id + self._last_node_id, + name='Diss-{}-Testing'.format(node.__class__.__name__), + ) + break + except proxmoxer.core.ResourceException as e: + if 'config file already exists' in str(e): + self._last_node_id += 1 + else: + raise e + self._await_task(clone_task) + + new_id = self._initial_vm_id + self._last_node_id + self._created_nodes.append(new_id) + return new_id + + def _build_node(self, node: structure.Node): + # Step 1: connect to Internet bridge with DHCP to install packages + # Step 2: connect to management bridge for correct setup + pass + + def teardown(self): + for node in self._created_nodes: + stop_task = self._proxmox.nodes(self._proxmox_node).qemu(node).status.stop.post() + self._await_task(stop_task) + delete_task = self._proxmox.nodes(self._proxmox_node).qemu(node).delete() + self._await_task(delete_task) + + for bridge in self._created_bridges: + self._proxmox.nodes(self._proxmox_node).network(bridge).delete() + network_task = self._proxmox.nodes(self._proxmox_node).network.put() + self._await_task(network_task) + + self._created_nodes = [] + self._last_node_id = 0 + + self._created_bridges = [] + self._last_bridge = 0 diff --git a/structure/__init__.py b/structure/__init__.py new file mode 100644 index 0000000..3f37154 --- /dev/null +++ b/structure/__init__.py @@ -0,0 +1,5 @@ +from .structure import Node + +from .structure import IpMethod, Interface, Bridge + +from .structure import SpeedTestServer, LocalServer, RemoteServer diff --git a/structure/structure.py b/structure/structure.py new file mode 100644 index 0000000..0ae04fb --- /dev/null +++ b/structure/structure.py @@ -0,0 +1,97 @@ +from enum import Enum +from typing import List, Optional, Union + + +# Enums +class IpMethod(Enum): + Manual = 0 + Management = 1 + Auto4 = 2 + Auto6 = 3 + Dhcp4 = 4 + Dhcp6 = 5 + + +class Interface: + def __init__(self, method: IpMethod, limit: Optional[int] = None): + self._method: IpMethod + + self._node: Optional[Node] = None + self._limit: Optional[int] = None + self._bridge: Optional[Bridge] = None + + self._method = method + self._limit = limit + + def get_method(self): + return self._method + + def set_node(self, node): + self._node = node + + def get_node(self): + return self._node + + def set_bridge(self, bridge): + self._bridge = bridge + + def get_bridge(self): + return self._bridge + + +class Bridge: + def __init__(self, *interfaces: Interface): + self._interfaces: List[Interface] = [] + self._name: str = '' + + for interface in interfaces: + self._interfaces.append(interface) + interface.set_bridge(self) + + def get_interfaces(self) -> List[Interface]: + return self._interfaces + + def set_name(self, name: str): + self._name = name + + def get_name(self) -> str: + return self._name + + +class Node: + def __init__(self, interfaces: List[Interface]): + self._id: Union[int, None] = None + self._interfaces: List[Interface] = interfaces + self._interfaces.append(Interface(IpMethod.Management)) + + for interface in self._interfaces: + interface.set_node(self) + + def get_interfaces(self): + return self._interfaces + + def set_id(self, new_id): + self._id = new_id + + def get_id(self): + return self._id + + +class SpeedTestServer(Node): + def server(self): + pass + + def client(self, server: Interface): + pass + + +class RemoteServer(Node): + pass + + +class LocalServer(Node): + def __init__(self, wan_interfaces: List[Interface], child: Node): + lan_interface = Interface(IpMethod.Manual) + super().__init__([*wan_interfaces, lan_interface]) + + Bridge(lan_interface, child.get_interfaces()[0])