dissertation-3-evaluation/evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Project Evaluation\n",
    "\n",
    "This file interfaces with a Proxmox server to automatically generate VM structures and graphs for testing the\n",
    "success criteria of my project."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup\n",
    "This section sets up the required variables for the Proxmox server."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import ipaddress\n",
    "\n",
    "import runners\n",
    "from structure import StandardEnvironment, StandardTest\n",
    "from structure import DirectEnvironment, DirectTest\n",
    "from structure import BaseEnvironment\n",
    "\n",
    "%load_ext dotenv\n",
    "%dotenv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing\n",
    "This section gathers the required data from the different structures for later graphs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "runner = runners.ProxmoxRunner(\n",
    "    host=os.getenv('PROXMOX_HOST'),\n",
    "    node=os.getenv('PROXMOX_NODE'),\n",
    "    user=os.getenv('PROXMOX_USER'),\n",
    "    token_name=os.getenv('PROXMOX_TOKEN_NAME'),\n",
    "    token_value=os.getenv('PROXMOX_TOKEN_VALUE'),\n",
    "\n",
    "    template_id=9000,\n",
    "    initial_vm_id=21002,\n",
    "\n",
    "    internet_bridge=os.getenv('INTERNET_BRIDGE'),\n",
    "\n",
    "    management_bridge=os.getenv('MANAGEMENT_BRIDGE'),\n",
    "    management_initial_ip=ipaddress.ip_address(os.getenv('MANAGEMENT_INITIAL_IP')),\n",
    ")\n",
    "\n",
    "setup_params = {\n",
    "    'access_key': os.getenv('S3_ACCESS_KEY'),\n",
    "    'secret_key': os.getenv('S3_SECRET_KEY'),\n",
    "    'branch': os.getenv('TARGET_BRANCH'),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from graphs import TestStore\n",
    "\n",
    "test_store = TestStore()\n",
    "\n",
    "def run_and_save_test(env: BaseEnvironment, test: StandardTest):\n",
    "    (inbound, outbound) = env.test(test)\n",
    "    test_store.save_inbound(test, inbound)\n",
    "    test_store.save_outbound(test, outbound)\n",
    "\n",
    "def attempt_n_times(foo, n=3):\n",
    "    for i in range(n):\n",
    "        try:\n",
    "            return foo()\n",
    "        except KeyboardInterrupt as e:\n",
    "            raise e\n",
    "        except Exception as e:\n",
    "            if i == n - 1:\n",
    "                raise e\n",
    "\n",
    "\n",
    "fast_tests = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Direct Server to Server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def direct_tests():\n",
    "    with DirectEnvironment(runner) as env:\n",
    "        run_and_save_test(env, DirectTest(1, bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
    "        run_and_save_test(env, DirectTest(2, bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
    "        run_and_save_test(env, DirectTest(3, bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
    "        run_and_save_test(env, DirectTest(4, bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
    "\n",
    "attempt_n_times(direct_tests)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Local Portal with 2 Interfaces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def two_interface_tests():\n",
    "    with StandardEnvironment(2, runner, setup_params) as env:\n",
    "        run_and_save_test(env, StandardTest([1,1], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
    "        run_and_save_test(env, StandardTest([1,2], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
    "        run_and_save_test(env, StandardTest([2,2], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
    "        run_and_save_test(env, StandardTest(\n",
    "            [2,2],\n",
    "            events={10: (0,1), 20: (0,2)},\n",
    "            duration=30,\n",
    "            interval_variation_target=0.8 if fast_tests else 0.5,\n",
    "        ))\n",
    "\n",
    "attempt_n_times(two_interface_tests)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Local Portal with 3 Interfaces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def three_interface_tests():\n",
    "    with StandardEnvironment(3, runner, setup_params) as env:\n",
    "        run_and_save_test(env, StandardTest([1,1,1], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
    "        run_and_save_test(env, StandardTest([2,2,2], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
    "\n",
    "attempt_n_times(three_interface_tests)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Local Portal with 4 Interfaces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def four_interface_tests():\n",
    "    with StandardEnvironment(4, runner, setup_params) as env:\n",
    "        run_and_save_test(env, StandardTest([1,1,1,1], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
    "        run_and_save_test(env, StandardTest([2,2,2,2], bandwidth_variation_target=0.2 if fast_tests else 0.05))\n",
    "\n",
    "attempt_n_times(four_interface_tests)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Graphs\n",
    "This section produces graphs from the collected data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from graphs import plot_iperf_results, plot_iperf_results_time\n",
    "\n",
    "if not os.path.exists('output/'):\n",
    "    os.makedirs('output/')\n",
    "\n",
    "for filename in os.listdir('output'):\n",
    "    file_path = os.path.join('output/', filename)\n",
    "    os.unlink(file_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Section 4.2: Graphs\n",
    "#### Subsection 4.2.2 Line Graphs\n",
    "\n",
    "Figure 4.1 Line graphs produced with varying error bars enabled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "plot_iperf_results_time(test_store,\n",
    "    {\n",
    "        '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n",
    "        '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
    "        '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
    "    },\n",
    "    fast_tests,\n",
    "    error_bars_x=False,\n",
    "    error_bars_y=False,\n",
    "    filename='png',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "plot_iperf_results_time(test_store,\n",
    "    {\n",
    "        '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n",
    "        '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
    "        '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
    "    },\n",
    "    fast_tests,\n",
    "    error_bars_x=True,\n",
    "    error_bars_y=False,\n",
    "    filename='png',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "plot_iperf_results_time(test_store,\n",
    "    {\n",
    "        '4x1MB Connections (proxied)': StandardTest([1,1,1,1]),\n",
    "        '3x1MB Connections (proxied)': StandardTest([1,1,1]),\n",
    "        '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
    "    },\n",
    "    fast_tests,\n",
    "    error_bars_x=False,\n",
    "    error_bars_y=True,\n",
    "    filename='png',\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### More than 2 connections evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "plot_iperf_results(test_store,\n",
    "    {\n",
    "        '4x1MB Connections\\n(proxied)': StandardTest([1,1,1,1]),\n",
    "        '3x1MB Connections\\n(proxied)': StandardTest([1,1,1]),\n",
    "        '2x1MB Connections\\n(proxied)': StandardTest([1,1]),\n",
    "    },\n",
    "    fast_tests,\n",
    "    filename='png',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "plot_iperf_results(test_store,\n",
    "    {\n",
    "        '4x2MB Connections\\n(proxied)': StandardTest([2,2,2,2]),\n",
    "        '3x2MB Connections\\n(proxied)': StandardTest([2,2,2]),\n",
    "        '2x2MB Connections\\n(proxied)': StandardTest([2,2]),\n",
    "    },\n",
    "    fast_tests,\n",
    "    filename='png',\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mixed Performance Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "plot_iperf_results(test_store,\n",
    "    {\n",
    "        '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
    "        '1x1MB + 1x2MB\\nConnections (proxied)': StandardTest([1,2]),\n",
    "        '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
    "    },\n",
    "    fast_tests,\n",
    "    filename='png',\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Eventful Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "plot_iperf_results_time(test_store,\n",
    "    {\n",
    "        'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n",
    "    },\n",
    "    fast_tests,\n",
    "    filename='png',\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparisons to a Direct Connection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "plot_iperf_results(test_store,\n",
    "    {\n",
    "        '1x2MB Connection\\n(direct)': DirectTest(2),\n",
    "        '2x1MB Connections\\n(proxied)': StandardTest([1,1]),\n",
    "        '1x1MB Connection\\n(direct)': DirectTest(1),\n",
    "\n",
    "    },\n",
    "    fast_tests,\n",
    "    filename='png',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "plot_iperf_results(test_store,\n",
    "    {\n",
    "        '1x4MB Connection\\n(direct)': DirectTest(4),\n",
    "        '2x2MB Connections\\n(proxied)': StandardTest([2,2]),\n",
    "        '1x2MB Connection\\n(direct)': DirectTest(2),\n",
    "\n",
    "    },\n",
    "    fast_tests,\n",
    "    filename='png',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "plot_iperf_results(test_store,\n",
    "    {\n",
    "        '4x1MB Connections\\n(proxied)': StandardTest([1,1,1,1]),\n",
    "        '3x1MB Connections\\n(proxied)': StandardTest([1,1,1]),\n",
    "        '2x1MB Connections\\n(proxied)': StandardTest([1,1]),\n",
    "    },\n",
    "    fast_tests,\n",
    "    filename='png',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "plot_iperf_results(test_store,\n",
    "    {\n",
    "        '4x2MB Connections\\n(proxied)': StandardTest([2,2,2,2]),\n",
    "        '3x2MB Connections\\n(proxied)': StandardTest([2,2,2]),\n",
    "        '2x2MB Connections\\n(proxied)': StandardTest([2,2]),\n",
    "    },\n",
    "    fast_tests,\n",
    "    filename='png',\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mixed Performance Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "plot_iperf_results_time(test_store,\n",
    "    {\n",
    "        '2x2MB Connections\\n(proxied)': StandardTest([2,2]),\n",
    "        '1x1MB + 1x2MB\\nConnections (proxied)': StandardTest([1,2]),\n",
    "        '2x1MB Connections\\n(proxied)': StandardTest([1,1]),\n",
    "    },\n",
    "    fast_tests,\n",
    "    filename='png',\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Eventful Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "plot_iperf_results_time(test_store,\n",
    "    {\n",
    "        'Varied Connection': StandardTest([2,2], events={10: (0,1), 15: (0,2)}, duration=30),\n",
    "    },\n",
    "    fast_tests,\n",
    "    filename='png',\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparisons to a Direct Connection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "plot_iperf_results_time(test_store,\n",
    "    {\n",
    "        '1x2MB Connection (not proxied)': DirectTest(2),\n",
    "        '2x1MB Connections (proxied)': StandardTest([1,1]),\n",
    "        '1x1MB Connection (not proxied)': DirectTest(1),\n",
    "\n",
    "    },\n",
    "    fast_tests,\n",
    "    filename='png',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "plot_iperf_results_time(test_store,\n",
    "    {\n",
    "        '1x4MB Connection (not proxied)': DirectTest(4),\n",
    "        '2x2MB Connections (proxied)': StandardTest([2,2]),\n",
    "        '1x2MB Connection (not proxied)': DirectTest(2),\n",
    "\n",
    "    },\n",
    "    fast_tests,\n",
    "    filename='png',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}