object-introspection/test/integration/runner_common.cpp

#include "runner_common.h"

#include <algorithm>
#include <boost/algorithm/string.hpp>
#include <boost/asio.hpp>
#include <boost/process.hpp>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <ios>
#include <iostream>
#include <string>
#include <utility>

#include "oi/OIOpts.h"
#include "oi/support/Toml.h"

using namespace std::literals;

namespace bp = boost::process;
namespace bpt = boost::property_tree;
namespace fs = std::filesystem;

bool run_skipped_tests = false;

namespace {

std::string oidExe = OID_EXE_PATH;
std::string configFile = CONFIG_FILE_PATH;

bool verbose = false;
bool preserve = false;
bool preserve_on_failure = false;
std::vector<std::string> global_oid_args{};

constexpr static OIOpts cliOpts{
    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit"},
    OIOpt{'p',
          "preserve",
          no_argument,
          nullptr,
          "Do not clean up files generated by OID after tests are finished"},
    OIOpt{'P',
          "preserve-on-failure",
          no_argument,
          nullptr,
          "Do not clean up files generated by OID for failed tests"},
    OIOpt{'v',
          "verbose",
          no_argument,
          nullptr,
          "Verbose output. Show OID's stdout and stderr on test failure"},
    OIOpt{'f',
          "force",
          no_argument,
          nullptr,
          "Force running tests, even if they are marked as skipped"},
    OIOpt{'x',
          "oid",
          required_argument,
          nullptr,
          "Path to OID executable to test"},
    OIOpt{'\0',
          "enable-feature",
          required_argument,
          nullptr,
          "Enable extra OID feature."},
};

void usage(std::string_view progname) {
  std::cout << "usage: " << progname << " ...\n";
  std::cout << cliOpts;
}

}  // namespace

int main(int argc, char* argv[]) {
  ::testing::InitGoogleTest(&argc, argv);

  if (const char* envArgs = std::getenv("OID_TEST_ARGS")) {
    boost::split(global_oid_args, envArgs, boost::is_any_of(" "));
  }

  int c;
  while ((c = getopt_long(
              argc, argv, cliOpts.shortOpts(), cliOpts.longOpts(), nullptr)) !=
         -1) {
    switch (c) {
      case 'p':
        preserve = true;
        break;
      case 'P':
        preserve_on_failure = true;
        break;
      case 'v':
        verbose = true;
        break;
      case 'f':
        run_skipped_tests = true;
        break;
      case 'x':
        // Must convert to absolute path so it continues to work after working
        // directory is changed
        oidExe = fs::absolute(optarg);
        break;
      case '\0':
        global_oid_args.push_back("-f"s + optarg);
        break;
      case 'h':
      default:
        usage(argv[0]);
        return 0;
    }
  }

  return RUN_ALL_TESTS();
}

void IntegrationBase::SetUp() {
  // Move into a temporary directory to run the test in an isolated environment
  auto tmp_dir = TmpDirStr();
  char* res = mkdtemp(&tmp_dir[0]);
  if (!res)
    abort();
  workingDir = std::move(tmp_dir);
  fs::current_path(workingDir);
}

void IntegrationBase::TearDown() {
  const ::testing::TestInfo* const test_info =
      ::testing::UnitTest::GetInstance()->current_test_info();
  if (preserve || (preserve_on_failure && test_info->result()->Failed())) {
    std::cerr << "Working directory preserved at: " << workingDir << std::endl;
  } else {
    fs::remove_all(workingDir);
  }
}

int IntegrationBase::exit_code(Proc& proc) {
  proc.ctx.run();
  proc.proc.wait();

  // Read stdout and stderr into members, as required by certain tests (see
  // gen_tests.py)
  {
    std::ifstream stdout(workingDir / "stdout");
    stdout_.assign(std::istreambuf_iterator<char>(stdout),
                   std::istreambuf_iterator<char>());
  }
  {
    std::ifstream stderr(workingDir / "stderr");
    stderr_.assign(std::istreambuf_iterator<char>(stderr),
                   std::istreambuf_iterator<char>());
  }
  return proc.proc.exit_code();
}

std::optional<std::filesystem::path> IntegrationBase::writeCustomConfig(
    std::string_view filePrefix, std::string_view content) {
  if (content.empty())
    return std::nullopt;

  auto path = workingDir / (std::string{filePrefix} + ".config.toml");
  std::ofstream file{path, std::ios_base::app};
  file << content;
  return path;
}

std::string OidIntegration::TmpDirStr() {
  return std::string("/tmp/oid-integration-XXXXXX");
}

OidProc OidIntegration::runOidOnProcess(OidOpts opts,
                                        std::vector<std::string> extra_args,
                                        std::string configPrefix,
                                        std::string configSuffix) {
  // Binary paths are populated by CMake
  std::string targetExe =
      std::string(TARGET_EXE_PATH) + " " + opts.targetArgs + " 1000";

  /* Spawn the target process with all IOs redirected to /dev/null to not polute
   * the terminal */
  // clang-format off
  bp::child targetProcess(
      targetExe,
      bp::std_in  < bp::null,
      bp::std_out > bp::null,
      bp::std_err > bp::null,
      opts.ctx);
  // clang-format on

  {
    // Delete previous segconfig, in case oid re-used the PID
    fs::path segconfigPath =
        "/tmp/oid-segconfig-" + std::to_string(targetProcess.id());
    fs::remove(segconfigPath);
    // Create the segconfig, so we have the rights to delete it above
    std::ofstream touch(segconfigPath);
  }

  // Keep PID as the last argument to make it easier for users to directly copy
  // and modify the command from the verbose mode output.
  // clang-format off
  auto default_args = std::array{
      "--debug-level=3"s,
      "--timeout=20"s,
      "--dump-json"s,
      "--script-source"s, opts.scriptSource,
      "--mode=strict"s,
  };
  // clang-format on

  // The arguments are appended in ascending order of precedence (low -> high)
  std::vector<std::string> oid_args;
  oid_args.insert(
      oid_args.end(), global_oid_args.begin(), global_oid_args.end());
  oid_args.insert(oid_args.end(), extra_args.begin(), extra_args.end());
  oid_args.insert(oid_args.end(), default_args.begin(), default_args.end());

  if (auto prefix = writeCustomConfig("prefix", configPrefix)) {
    oid_args.emplace_back("--config-file");
    oid_args.emplace_back(*prefix);
  }

  oid_args.emplace_back("--config-file");
  oid_args.emplace_back(configFile);

  if (auto suffix = writeCustomConfig("suffix", configSuffix)) {
    oid_args.emplace_back("--config-file");
    oid_args.emplace_back(*suffix);
  }

  oid_args.emplace_back("--pid");
  oid_args.emplace_back(std::to_string(targetProcess.id()));

  if (verbose) {
    std::cerr << "Running: " << targetExe << "\n";
    std::cerr << "Running: " << oidExe << " ";
    for (const auto& arg : oid_args) {
      std::cerr << arg << " ";
    }
    std::cerr << std::endl;
  }

  // Use tee to write the output to files. If verbose is on, also redirect the
  // output to stderr.
  bp::async_pipe std_out_pipe(opts.ctx), std_err_pipe(opts.ctx);
  bp::child std_out, std_err;
  if (verbose) {
    // clang-format off
    std_out = bp::child(bp::search_path("tee"),
                        (workingDir / "stdout").string(),
                        bp::std_in < std_out_pipe,
                        bp::std_out > stderr,
                        opts.ctx);
    std_err = bp::child(bp::search_path("tee"),
                        (workingDir / "stderr").string(),
                        bp::std_in < std_err_pipe,
                        bp::std_out > stderr,
                        opts.ctx);
    // clang-format on
  } else {
    // clang-format off
    std_out = bp::child(bp::search_path("tee"),
                        (workingDir / "stdout").string(),
                        bp::std_in < std_out_pipe,
                        bp::std_out > bp::null,
                        opts.ctx);
    std_err = bp::child(bp::search_path("tee"),
                        (workingDir / "stderr").string(),
                        bp::std_in < std_err_pipe,
                        bp::std_out > bp::null,
                        opts.ctx);
    // clang-format on
  }

  /* Spawn `oid` with tracing on and IOs redirected */
  // clang-format off
  bp::child oidProcess(
      oidExe,
      bp::args(oid_args),
      bp::env["OID_METRICS_TRACE"] = "time",
      bp::std_in  < bp::null,
      bp::std_out > std_out_pipe,
      bp::std_err > std_err_pipe,
      opts.ctx);
  // clang-format on

  return OidProc{
      .target = Proc{opts.ctx, std::move(targetProcess), {}, {}},
      .oid = Proc{opts.ctx,
                  std::move(oidProcess),
                  std::move(std_out),
                  std::move(std_err)},
  };
}

void IntegrationBase::compare_json(const bpt::ptree& expected_json,
                                   const bpt::ptree& actual_json,
                                   const std::string& full_key,
                                   bool expect_eq) {
  if (expected_json.empty()) {
    if (expect_eq) {
      ASSERT_EQ(expected_json.data(), actual_json.data())
          << "Incorrect value for key: " << full_key;
    } else {
      ASSERT_NE(expected_json.data(), actual_json.data())
          << "Incorrect value for key: " << full_key;
    }
  }

  if (expected_json.begin()->first == "") {
    // Empty key - assume this is an array

    ASSERT_EQ(expected_json.size(), actual_json.size())
        << "Array size difference for key: " << full_key;

    int i = 0;
    for (auto e_it = expected_json.begin(), a_it = actual_json.begin();
         e_it != expected_json.end() && a_it != actual_json.end();
         e_it++, a_it++) {
      compare_json(e_it->second,
                   a_it->second,
                   full_key + "[" + std::to_string(i) + "]",
                   expect_eq);
      i++;
    }
    return;
  }

  // Compare as Key-Value pairs
  for (const auto& [key, val] : expected_json) {
    if (key == "NOT") {
      auto curr_key = full_key + ".NOT";
      ASSERT_EQ(true, expect_eq) << "Invalid expected data: " << curr_key
                                 << " - Can not use nested \"NOT\" expressions";
      if (val.empty()) {
        // Check that a given single key does not exist
        const auto& key_to_check = val.data();
        auto actual_it = actual_json.find(key_to_check);
        auto bad_key = full_key + "." + key_to_check;
        if (actual_it != actual_json.not_found()) {
          ADD_FAILURE() << "Unexpected key found in output: " << bad_key;
        }
      } else {
        // Check that a key's value is not equal to something
        compare_json(val, actual_json, curr_key, !expect_eq);
      }
      continue;
    }

    auto actual_it = actual_json.find(key);
    auto curr_key = full_key + "." + key;
    if (actual_it == actual_json.not_found()) {
      // TODO: Remove these with the switch to treebuilderv2. This is a hack to
      // make some old JSON output compatible with new JSON output.
      if (key == "typeName") {
        auto type_names = actual_json.find("typeNames");
        if (type_names != actual_json.not_found()) {
          if (auto name = type_names->second.rbegin();
              name != type_names->second.rend()) {
            compare_json(val, name->second, curr_key, expect_eq);
            continue;
          }
        }
      } else if (key == "dynamicSize" && val.get_value<size_t>() == 0) {
        continue;
      } else if (key == "size") {
        continue;
      }

      ADD_FAILURE() << "Expected key not found in output: " << curr_key;
      continue;
    }
    compare_json(val, actual_it->second, curr_key, expect_eq);
  }
}

std::string OilIntegration::TmpDirStr() {
  return std::string("/tmp/oil-integration-XXXXXX");
}

Proc OilIntegration::runOilTarget(OilOpts opts,
                                  std::string configPrefix,
                                  std::string configSuffix) {
  std::string targetExe =
      std::string(TARGET_EXE_PATH) + " " + opts.targetArgs + " ";
  if (auto prefix = writeCustomConfig("prefix", configPrefix)) {
    targetExe += *prefix;
    targetExe += " ";
  }
  targetExe += configFile;
  if (auto suffix = writeCustomConfig("suffix", configSuffix)) {
    targetExe += " ";
    targetExe += *suffix;
  }

  if (verbose) {
    std::cerr << "Running: " << targetExe << std::endl;
  }

  // Use tee to write the output to files. If verbose is on, also redirect the
  // output to stderr.
  bp::async_pipe std_out_pipe(opts.ctx), std_err_pipe(opts.ctx);
  bp::child std_out, std_err;
  if (verbose) {
    // clang-format off
    std_out = bp::child(bp::search_path("tee"),
                        (workingDir / "stdout").string(),
                        bp::std_in < std_out_pipe,
                        bp::std_out > stderr,
                        opts.ctx);
    std_err = bp::child(bp::search_path("tee"),
                        (workingDir / "stderr").string(),
                        bp::std_in < std_err_pipe,
                        bp::std_out > stderr,
                        opts.ctx);
    // clang-format on
  } else {
    // clang-format off
    std_out = bp::child(bp::search_path("tee"),
                        (workingDir / "stdout").string(),
                        bp::std_in < std_out_pipe,
                        bp::std_out > bp::null,
                        opts.ctx);
    std_err = bp::child(bp::search_path("tee"),
                        (workingDir / "stderr").string(),
                        bp::std_in < std_err_pipe,
                        bp::std_out > bp::null,
                        opts.ctx);
    // clang-format on
  }

  /* Spawn target with tracing on and IOs redirected in custom pipes to be read
   * later */
  // clang-format off
  bp::child targetProcess(
      targetExe,
      bp::std_in  < bp::null,
      bp::std_out > std_out_pipe,
      bp::std_err > std_err_pipe,
      opts.ctx);
  // clang-format on

  return Proc{opts.ctx,
              std::move(targetProcess),
              std::move(std_out),
              std::move(std_err)};
}