Initial commit

- Structure built
- Proxmox bridge and VM creation

Todo:
- Proxmox VM networking
- SSH

.gitignore

@@ -0,0 +1,236 @@
+
+# Created by https://www.toptal.com/developers/gitignore/api/python,intellij+all,dotenv
+# Edit at https://www.toptal.com/developers/gitignore?templates=python,intellij+all,dotenv
+
+### dotenv ###
+.env
+
+### Intellij+all ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### Intellij+all Patch ###
+# Ignores the whole .idea folder and all .iml files
+# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360
+
+.idea/
+
+# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023
+
+*.iml
+modules.xml
+.idea/misc.xml
+*.ipr
+
+# Sonarlint plugin
+.idea/sonarlint
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+pytestdebug.log
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+doc/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+pythonenv*
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# profiling data
+.prof
+
+# End of https://www.toptal.com/developers/gitignore/api/python,intellij+all,dotenv

README.md

@@ -0,0 +1,4 @@
+# Dissertation Evaluation
+
+A Python backing to partially automate producing the graphs for my dissertation using an iPython Notebook and a Proxmox server.
+ 

evaluation.ipynb

@@ -0,0 +1,124 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import ipaddress\n",
+    "\n",
+    "import runners\n",
+    "from structure import Bridge\n",
+    "from structure import SpeedTestServer, RemoteServer, LocalServer\n",
+    "from structure import Interface, IpMethod\n",
+    "\n",
+    "%load_ext dotenv\n",
+    "%dotenv"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "outputs": [],
+   "source": [
+    "runner = runners.ProxmoxRunner(\n",
+    "    host=os.getenv('PROXMOX_HOST'),\n",
+    "    node=os.getenv('PROXMOX_NODE'),\n",
+    "    user=os.getenv('PROXMOX_USER'),\n",
+    "    token_name=os.getenv('PROXMOX_TOKEN_NAME'),\n",
+    "    token_value=os.getenv('PROXMOX_TOKEN_VALUE'),\n",
+    "\n",
+    "    template_id=9000,\n",
+    "    initial_vm_id=21002,\n",
+    "\n",
+    "    internet_bridge='vmbr2',\n",
+    "\n",
+    "    management_bridge='vmbr4',\n",
+    "    management_initial_ip=ipaddress.ip_address('10.21.12.2'),\n",
+    ")"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "outputs": [],
+   "source": [
+    "g_st = SpeedTestServer([Interface(IpMethod.Auto4)])\n",
+    "l_st = SpeedTestServer([Interface(IpMethod.Dhcp4)])\n",
+    "\n",
+    "rs = RemoteServer([Interface(IpMethod.Auto4)])\n",
+    "ls = LocalServer([\n",
+    "    Interface(IpMethod.Auto4, limit=1),\n",
+    "    Interface(IpMethod.Auto4, limit=1),\n",
+    "], l_st)\n",
+    "\n",
+    "top_level_bridge = Bridge(*[\n",
+    "    g_st.get_interfaces()[0],\n",
+    "    rs.get_interfaces()[0],\n",
+    "    *ls.get_interfaces()[0:2],\n",
+    "])\n",
+    "runner.build(top_level_bridge)\n",
+    "\n",
+    "# Test from the client to the global network via the proxy\n",
+    "g_st.server()\n",
+    "l_st.client(rs.get_interfaces()[0])\n",
+    "\n",
+    "# Test from the global network to the client via the proxy\n",
+    "g_st.server()\n",
+    "l_st.client(rs.get_interfaces()[0])\n",
+    "\n",
+    "# Clean up\n",
+    "runner.teardown()"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}

runners/__init__.py

@@ -0,0 +1 @@
+from .runners import PrintRunner, ProxmoxRunner

runners/runners.py

@@ -0,0 +1,203 @@
+import ipaddress
+import os
+from datetime import datetime
+from typing import Callable, List, Tuple
+
+import proxmoxer
+
+import structure
+
+
+def check_env(*names: str) -> bool:
+    for name in names:
+        if name not in os.environ:
+            return False
+    return True
+
+
+def bridge_node_dfs(
+        first: structure.Bridge,
+        bridge_name_generator: Callable[[structure.Bridge], str],
+        node_id_generator: Callable[[structure.Node], int],
+) -> Tuple[List[structure.Bridge], List[structure.Node]]:
+    bridges: List[structure.Bridge] = []
+    nodes: List[structure.Node] = []
+
+    queue: List[structure.Bridge] = [first]
+    while len(queue) > 0:
+        bridge = queue.pop()
+        if bridge.get_name() != '':
+            continue
+
+        bridges.append(bridge)
+        bridge.set_name(bridge_name_generator(bridge))
+
+        # from this bridge, find all nodes (via all interfaces)
+        reachable_nodes: List[structure.Node] = []
+        for interface in bridge.get_interfaces():
+            node = interface.get_node()
+            if node.get_id() is not None:
+                continue
+
+            node.set_id(node_id_generator(node))
+            reachable_nodes.append(node)
+            nodes.append(node)
+
+        # from each node, find all bridges (via all interfaces)
+        for node in reachable_nodes:
+            for interface in node.get_interfaces():
+                bridge = interface.get_bridge()
+                if bridge is not None and bridge.get_name() == '':
+                    queue.append(bridge)
+
+    return bridges, nodes
+
+
+class PrintRunner:
+    def __init__(self):
+        self._last_bridge: int = 0
+        self._last_node_id = 0
+
+    def build(self, bridge: structure.Bridge):
+        bridges, nodes = bridge_node_dfs(bridge, lambda _: self.name_bridge(), lambda _: self.id_node())
+
+        print(bridges)
+        print(nodes)
+
+    def teardown(self):
+        pass
+
+    def name_bridge(self) -> str:
+        self._last_bridge += 1
+        return 'fake{}'.format(self._last_bridge)
+
+    def id_node(self) -> int:
+        self._last_node_id += 1
+        return self._last_node_id
+
+
+class ProxmoxRunner:
+    def __init__(
+            self,
+            host: str,
+            node: str,
+            user: str,
+            token_name: str,
+            token_value: str,
+
+            template_id: int,
+            initial_vm_id: int,
+
+            internet_bridge: str,
+
+            management_bridge: str,
+            management_initial_ip: ipaddress,
+
+            verify_ssl: bool = False,
+    ):
+        self._last_node_id = 0
+        self._created_nodes: List[int] = []
+        self._last_bridge = 0
+        self._created_bridges: List[str] = []
+
+        self._proxmox = proxmoxer.ProxmoxAPI(
+            host,
+            user=user,
+            token_name=token_name,
+            token_value=token_value,
+            verify_ssl=verify_ssl,
+        )
+
+        self._proxmox_node = node
+
+        self._template_id = template_id
+        self._initial_vm_id = initial_vm_id - 1
+
+        self._internet_bridge = internet_bridge
+
+        self._management_bridge = management_bridge
+        self._management_initial_ip = management_initial_ip
+
+    def build(self, bridge: structure.Bridge):
+        bridges, nodes = bridge_node_dfs(bridge, lambda x: self._create_bridge(x), lambda x: self._create_node(x))
+
+        self._build_bridges()
+        for node in nodes:
+            self._build_node(node)
+
+    def _await_task(self, upid, timeout=10):
+        t1 = datetime.now()
+        while (datetime.now() - t1).seconds < timeout:
+            if self._proxmox.nodes(self._proxmox_node).tasks(upid).status.get()['status'] == 'stopped':
+                return
+        raise TimeoutError
+
+    def _create_bridge(self, bridge: structure.Bridge) -> str:
+        self._last_bridge += 1
+
+        while True:
+            try:
+                self._proxmox.nodes(self._proxmox_node).network.post(
+                    iface='vmbr{}'.format(self._last_bridge),
+                    type='bridge',
+                    autostart=1,
+                    comments='Automatically created by Python evaluation',
+                )
+                break
+            except proxmoxer.core.ResourceException as e:
+                if 'interface already exists' in str(e):
+                    self._last_bridge += 1
+                else:
+                    raise e
+
+        bridge_name = 'vmbr{}'.format(self._last_bridge)
+        self._created_bridges.append(bridge_name)
+        return bridge_name
+
+    def _build_bridges(self):
+        network_task = self._proxmox.nodes(self._proxmox_node).network.put()
+        self._await_task(network_task)
+
+    def _create_node(self, node: structure.Node) -> int:
+        self._last_node_id += 1
+
+        while True:
+            try:
+                clone_task = self._proxmox.nodes(self._proxmox_node).qemu(self._template_id).clone.post(
+                    newid=self._initial_vm_id + self._last_node_id,
+                    name='Diss-{}-Testing'.format(node.__class__.__name__),
+                )
+                break
+            except proxmoxer.core.ResourceException as e:
+                if 'config file already exists' in str(e):
+                    self._last_node_id += 1
+                else:
+                    raise e
+        self._await_task(clone_task)
+
+        new_id = self._initial_vm_id + self._last_node_id
+        self._created_nodes.append(new_id)
+        return new_id
+
+    def _build_node(self, node: structure.Node):
+        # Step 1: connect to Internet bridge with DHCP to install packages
+        # Step 2: connect to management bridge for correct setup
+        pass
+
+    def teardown(self):
+        for node in self._created_nodes:
+            stop_task = self._proxmox.nodes(self._proxmox_node).qemu(node).status.stop.post()
+            self._await_task(stop_task)
+            delete_task = self._proxmox.nodes(self._proxmox_node).qemu(node).delete()
+            self._await_task(delete_task)
+
+        for bridge in self._created_bridges:
+            self._proxmox.nodes(self._proxmox_node).network(bridge).delete()
+        network_task = self._proxmox.nodes(self._proxmox_node).network.put()
+        self._await_task(network_task)
+
+        self._created_nodes = []
+        self._last_node_id = 0
+
+        self._created_bridges = []
+        self._last_bridge = 0

structure/__init__.py

@@ -0,0 +1,5 @@
+from .structure import Node
+
+from .structure import IpMethod, Interface, Bridge
+
+from .structure import SpeedTestServer, LocalServer, RemoteServer

structure/structure.py

@@ -0,0 +1,97 @@
+from enum import Enum
+from typing import List, Optional, Union
+
+
+# Enums
+class IpMethod(Enum):
+    Manual = 0
+    Management = 1
+    Auto4 = 2
+    Auto6 = 3
+    Dhcp4 = 4
+    Dhcp6 = 5
+
+
+class Interface:
+    def __init__(self, method: IpMethod, limit: Optional[int] = None):
+        self._method: IpMethod
+
+        self._node: Optional[Node] = None
+        self._limit: Optional[int] = None
+        self._bridge: Optional[Bridge] = None
+
+        self._method = method
+        self._limit = limit
+
+    def get_method(self):
+        return self._method
+
+    def set_node(self, node):
+        self._node = node
+
+    def get_node(self):
+        return self._node
+
+    def set_bridge(self, bridge):
+        self._bridge = bridge
+
+    def get_bridge(self):
+        return self._bridge
+
+
+class Bridge:
+    def __init__(self, *interfaces: Interface):
+        self._interfaces: List[Interface] = []
+        self._name: str = ''
+
+        for interface in interfaces:
+            self._interfaces.append(interface)
+            interface.set_bridge(self)
+
+    def get_interfaces(self) -> List[Interface]:
+        return self._interfaces
+
+    def set_name(self, name: str):
+        self._name = name
+
+    def get_name(self) -> str:
+        return self._name
+
+
+class Node:
+    def __init__(self, interfaces: List[Interface]):
+        self._id: Union[int, None] = None
+        self._interfaces: List[Interface] = interfaces
+        self._interfaces.append(Interface(IpMethod.Management))
+
+        for interface in self._interfaces:
+            interface.set_node(self)
+
+    def get_interfaces(self):
+        return self._interfaces
+
+    def set_id(self, new_id):
+        self._id = new_id
+
+    def get_id(self):
+        return self._id
+
+
+class SpeedTestServer(Node):
+    def server(self):
+        pass
+
+    def client(self, server: Interface):
+        pass
+
+
+class RemoteServer(Node):
+    pass
+
+
+class LocalServer(Node):
+    def __init__(self, wan_interfaces: List[Interface], child: Node):
+        lan_interface = Interface(IpMethod.Manual)
+        super().__init__([*wan_interfaces, lan_interface])
+
+        Bridge(lan_interface, child.get_interfaces()[0])