oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD

CMakeLists.txt

@@ -289,7 +289,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -365,6 +365,7 @@ add_executable(oilgen
 target_link_libraries(oilgen
   drgn_utils
   oicore
+  clangTooling
 )
 
 ### Object Introspection cache Printer (OIP)

oi/CodeGen.cpp

@@ -299,6 +299,10 @@ void genDefsThriftClass(const Class& c, std::string& code) {
 
 }  // namespace
 
+CodeGen::CodeGen(const OICodeGen::Config& config) : config_(config) {
+  DCHECK(!config.features[Feature::PolymorphicInheritance]) << "polymorphic inheritance requires symbol service!";
+}
+
 void CodeGen::genDefsThrift(const TypeGraph& typeGraph, std::string& code) {
   for (const Type& t : typeGraph.finalTypes) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
@@ -542,7 +546,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1115,15 +1119,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
                               std::string& code,
                               RootFunctionName name) {
-  try {
+  if (!registerContainers())
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
     return false;
-  }
 
   try {
     addDrgnRoot(drgnType, typeGraph_);
@@ -1148,6 +1145,19 @@ void CodeGen::exportDrgnTypes(TypeHierarchy& th,
   }
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1189,7 +1199,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(IdentifyContainers::createPass(containerInfos_));

oi/CodeGen.h

@@ -41,8 +41,9 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config);
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   struct ExactName {
@@ -65,6 +66,7 @@ class CodeGen {
                        std::list<drgn_type>& drgnTypes,
                        drgn_type** rootType) const;
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -77,7 +79,7 @@ class CodeGen {
  private:
   type_graph::TypeGraph typeGraph_;
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>

oi/Config.cpp

@@ -22,6 +22,8 @@
 #include <boost/property_tree/ini_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 #include <filesystem>
+#include <range/v3/action/sort.hpp>
+#include <range/v3/action/unique.hpp>
 
 #include "oi/support/Toml.h"
 
@@ -54,6 +56,9 @@ std::optional<FeatureSet> processConfigFiles(
     enables |= *fs;
   }
 
+  ranges::actions::sort(generatorConfig.containerConfigPaths);
+  ranges::actions::unique(generatorConfig.containerConfigPaths);
+
   // Override anything from the config files with command line options
   for (auto [k, v] : featureMap) {
     enables[k] = v;
@@ -108,8 +113,8 @@ std::optional<FeatureSet> processConfigFile(
           if the right path is absolute, else append the right path to the
           left path.
           */
-          generatorConfig.containerConfigPaths.emplace(configDirectory /
+          generatorConfig.containerConfigPaths.emplace_back(configDirectory /
-                                                       el.get());
+                                                            el.get());
         }
       });
     }

oi/ContainerInfo.cpp

@@ -202,7 +202,7 @@ std::regex getMatcher(const std::string& typeName) {
 ContainerInfo::ContainerInfo(const fs::path& path) {
   toml::table container;
   try {
-    container = toml::parse_file(std::string(path));
+    container = toml::parse_file(path.string());
   } catch (const toml::parse_error& err) {
     // Convert into a ContainerInfoError, just to avoid having to include
     // the huge TOML++ header in the caller's file. Use toml::parse_error's

oi/OICodeGen.h

@@ -66,7 +66,7 @@ class OICodeGen {
     };
 
     FeatureSet features;
-    std::set<std::filesystem::path> containerConfigPaths;
+    std::vector<std::filesystem::path> containerConfigPaths;
     std::set<std::string> defaultHeaders;
     std::set<std::string> defaultNamespaces;
     std::vector<std::pair<std::string, std::string>> membersToStub;

oi/OIGenerator.cpp

@@ -16,130 +16,156 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
 #include <glog/logging.h>
 
 #include <boost/core/demangle.hpp>
 #include <fstream>
 #include <iostream>
+#include <range/v3/action/transform.hpp>
+#include <range/v3/core.hpp>
+#include <range/v3/view/cache1.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/drop_while.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>  // TODO: remove
 #include <string_view>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
+class ConsumerContext;
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
 
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
+class CreateTypeGraphConsumer;
-      templateArgsToSymbolsMap;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
-
+ public:
-  auto symbols = prog.find_all_symbols();
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
+  void ExecuteAction() override;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
-    if (val.first.empty() || val.second.empty()) {
+      clang::CompilerInstance& CI, clang::StringRef file) override;
-      continue;
+
-    }
+ private:
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  return strongToWeakSymbols;
+  std::unique_ptr<clang::FrontendAction> create() override {
-}
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
 
-std::unordered_map<std::string, drgn_qualified_type>
+ private:
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
+  ConsumerContext& ctx;
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
+};
 
-  std::unordered_map<std::string, drgn_qualified_type> out;
+class ConsumerContext {
+ public:
+  ConsumerContext(const std::vector<std::unique_ptr<ContainerInfo>>& cis)
+      : containerInfos{cis} {
+  }
 
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
+  type_graph::TypeGraph typeGraph;
-    std::string strongLinkageName;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
-    {
+  std::optional<bool> pic;
-      const char* linkageNameCstr;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
-      if (auto err = drgnplusplus::error(
+
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
+ private:
-        // throw err;
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+}  // namespace
+
+int OIGenerator::generate(clang::tooling::CompilationDatabase& db,
+                          const std::vector<std::string>& sourcePaths) {
+  std::map<Feature, bool> featuresMap = {
+      {Feature::TypeGraph, true},
+      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},
+      {Feature::PackStructs, true},
+      {Feature::PruneTypeGraph, true},
+  };
+
+  OICodeGen::Config generatorConfig{};
+  OICompiler::Config compilerConfig{};
+
+  auto features = config::processConfigFiles(
+      configFilePaths, featuresMap, compilerConfig, generatorConfig);
+  if (!features) {
+    LOG(ERROR) << "failed to process config file";
+    return -1;
+  }
+  generatorConfig.features = *features;
+  compilerConfig.features = *features;
+
+  std::vector<std::unique_ptr<ContainerInfo>> containerInfos;
+  containerInfos.reserve(generatorConfig.containerConfigPaths.size());
+  try {
+    for (const auto& path : generatorConfig.containerConfigPaths) {
+      auto info = std::make_unique<ContainerInfo>(path);
+      if (info->requiredFeatures != (*features & info->requiredFeatures)) {
+        VLOG(1) << "Skipping container (feature conflict): " << info->typeName;
         continue;
       }
-      strongLinkageName = linkageNameCstr;
+      containerInfos.emplace_back(std::move(info));
     }
-
+  } catch (const ContainerInfoError& err) {
-    std::string weakLinkageName;
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
+    return -1;
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
   }
 
-  return out;
+  ConsumerContext ctx{containerInfos};
-}
+  CreateTypeGraphActionFactory factory{ctx};
 
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
+  clang::tooling::ClangTool tool{db, sourcePaths};
-                                      const OICompiler::Config& compilerConfig,
+  if (auto ret = tool.run(&factory); ret != 0) {
-                                      const drgn_qualified_type& type,
+    return ret;
-                                      const std::string& linkageName,
+  }
-                                      SymbolService& symbols) {
+
-  CodeGen codegen{generatorConfig, symbols};
+  if (ctx.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one type to generate for but we can't currently "
+        "handle this case");
+
+  if (ctx.nameToTypeMap.empty()) {
+    LOG(ERROR) << "Nothing to generate!";
+    return -1;
+  }
+  const auto& linkageName = ctx.nameToTypeMap.begin()->first;
+
+  compilerConfig.usePIC = ctx.pic.value();
+  CodeGen codegen{generatorConfig};
+  for (auto&& ptr : containerInfos)
+    codegen.registerContainer(std::move(ptr));
+  codegen.transform(ctx.typeGraph);
 
   std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
+  codegen.generate(ctx.typeGraph, code, CodeGen::ExactName{linkageName});
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
 
   std::string sourcePath = sourceFileDumpPath;
   if (sourceFileDumpPath.empty()) {
@@ -152,78 +178,110 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
   }
 
   OICompiler compiler{{}, compilerConfig};
-
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
-
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
 }
 
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
+namespace {
-  drgnplusplus::program prog;
 
-  {
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
+ private:
-    if (auto err = drgnplusplus::error(
+  ConsumerContext& ctx;
-            drgn_program_load_debug_info(prog.get(),
+
-                                         std::data(objectPaths),
+ public:
-                                         std::size(objectPaths),
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx(ctx_) {
-                                         false,
+  }
-                                         false))) {
+
-      LOG(ERROR) << "error loading debug info program: " << err;
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
-      throw err;
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
     }
-  }
 
-  auto oilTypes = findOilTypesAndNames(prog);
+    auto introspectImpl =
-
+        std::move(oi_namespaces) |
-  std::map<Feature, bool> featuresMap = {
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
-      {Feature::TypeGraph, true},
+        ranges::views::transform([](auto* p) {
-      {Feature::TreeBuilderV2, true},
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
-      {Feature::Library, true},
+        }) |
-      {Feature::PackStructs, true},
+        ranges::views::filter([](auto* td) {
-      {Feature::PruneTypeGraph, true},
+          return td != nullptr && td->getName() == "introspectImpl";
-  };
+        }) |
-
+        ranges::views::take(1) | ranges::to<std::vector>();
-  OICodeGen::Config generatorConfig{};
+    if (introspectImpl.empty()) {
-  OICompiler::Config compilerConfig{};
+      LOG(WARNING)
-  compilerConfig.usePIC = pic;
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
-
+             "you compile with `OIL_AOT_COMPILATION=1`?";
-  auto features = config::processConfigFiles(
+      return;
-      configFilePaths, featuresMap, compilerConfig, generatorConfig);
-  if (!features) {
-    LOG(ERROR) << "failed to process config file";
-    return -1;
-  }
-  generatorConfig.features = *features;
-  compilerConfig.features = *features;
-
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
     }
-  }
 
-  size_t successes = oilTypes.size() - failures;
+    auto nameToClangTypeMap =
-  LOG(INFO) << "object introspection generation complete. " << successes
+        ranges::views::single(introspectImpl[0]) |
-            << " successes and " << failures << " failures.";
+        ranges::views::for_each(
+            [](auto* td) { return td->specializations(); }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
 
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
+              assert(fd->getNumParams() == 1);
-    return -1;
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, opts};
+
+    auto& Sema = *ctx.sema;
+    ctx.nameToTypeMap =
+        nameToClangTypeMap |
+        ranges::views::transform(
+            [&parser, &Context, &Sema](
+                auto& p) -> std::pair<std::string, type_graph::Type*> {
+              return {p.first, &parser.parse(Context, Sema, *p.second)};
+            }) |
+        ranges::to<std::unordered_map>();
+
+    for (const auto& [name, type] : ctx.nameToTypeMap)
+      ctx.typeGraph.addRoot(*type);
   }
-  return 0;
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  // Compile the output as position independent if any input is position
+  // independent
+  bool pic = CI.getCodeGenOpts().RelocationModel == llvm::Reloc::PIC_;
+  ctx.pic = ctx.pic.value_or(false) || pic;
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
 }
 
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+}  // namespace
 }  // namespace oi::detail

oi/OIGenerator.h

@@ -23,11 +23,19 @@
 #include "oi/OICodeGen.h"
 #include "oi/OICompiler.h"
 
+namespace clang::tooling {
+class CompilationDatabase;
+}
+
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate(clang::tooling::CompilationDatabase&,
+               const std::vector<std::string>&);
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -41,8 +49,8 @@ class OIGenerator {
   void setFailIfNothingGenerated(bool fail) {
     failIfNothingGenerated = fail;
   }
-  void setUsePIC(bool pic_) {
+  void setClangArgs(std::vector<std::string> args_) {
-    pic = pic_;
+    clangArgs = std::move(args_);
   }
 
  private:
@@ -50,20 +58,7 @@ class OIGenerator {
   std::vector<std::filesystem::path> configFilePaths;
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
-  bool pic = false;
+  std::vector<std::string> clangArgs;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
 };
 
 }  // namespace oi::detail

oi/type_graph/CMakeLists.txt

@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnExporter.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
@@ -27,3 +28,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})

oi/type_graph/ClangTypeParser.cpp

@@ -0,0 +1,416 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Basic/DiagnosticSema.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <iostream>  // TODO: maybe remove
+#include <regex>
+#include <stdexcept>  // TODO: remove
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (VLOG_IS_ON(3)) {
+    std::string fqName = clang::TypeName::getFullyQualifiedName(
+        clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+    VLOG(3) << std::string(depth_ * 2, ' ') << fqName;
+  }
+
+  if (!requireCompleteType(*sema, ty))
+    return makeType<Incomplete>(ty, "incomplete (TODO naming)");
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(llvm::cast<const clang::RecordType>(ty));
+    case clang::Type::LValueReference:
+      return enumerateReference(
+          llvm::cast<const clang::LValueReferenceType>(ty));
+    case clang::Type::Pointer:
+      return enumeratePointer(llvm::cast<const clang::PointerType>(ty));
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          llvm::cast<const clang::SubstTemplateTypeParmType>(ty));
+    case clang::Type::Builtin:
+      return enumeratePrimitive(llvm::cast<const clang::BuiltinType>(ty));
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(
+          llvm::cast<const clang::ElaboratedType>(ty));
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          llvm::cast<const clang::TemplateSpecializationType>(ty));
+    case clang::Type::UnaryTransform:
+      return enumerateUnaryTransformType(
+          llvm::cast<const clang::UnaryTransformType>(ty));
+    case clang::Type::Decltype:
+      return enumerateDecltypeType(llvm::cast<const clang::DecltypeType>(ty));
+    case clang::Type::Typedef:
+      return enumerateTypedef(llvm::cast<const clang::TypedefType>(ty));
+    case clang::Type::Using:
+      return enumerateUsing(llvm::cast<const clang::UsingType>(ty));
+    case clang::Type::ConstantArray:
+      return enumerateArray(llvm::cast<const clang::ConstantArrayType>(ty));
+    case clang::Type::Enum:
+      return enumerateEnum(llvm::cast<const clang::EnumType>(ty));
+
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Type& ClangTypeParser::enumerateDecltypeType(const clang::DecltypeType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Type& ClangTypeParser::enumerateUnaryTransformType(
+    const clang::UnaryTransformType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Typedef& ClangTypeParser::enumerateUsing(const clang::UsingType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+  std::string name = ty.getFoundDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+
+  std::string name = ty.getDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string name = ty.getDecl()->getNameAsString();
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  std::map<int64_t, std::string> enumeratorMap;
+  if (options_.readEnumValues) {
+    for (const auto* enumerator : ty.getDecl()->enumerators()) {
+      enumeratorMap.emplace(enumerator->getInitVal().getExtValue(),
+                            enumerator->getNameAsString());
+    }
+  }
+
+  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+}
+
+Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
+  uint64_t len = ty.getSize().getLimitedValue();
+  auto& t = enumerateType(*ty.getElementType());
+  return makeType<Array>(ty, t, len);
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  if (auto* info = getContainerInfo(fqName)) {
+    auto& c = makeType<Container>(ty, *info, size, nullptr);
+    enumerateClassTemplateParams(ty, c.templateParams);
+    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)));
+    return c;
+  }
+
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+
+  params.reserve(list.size());
+  for (const auto& arg : list.asArray()) {
+    if (auto p = enumerateTemplateParam(arg))
+      params.emplace_back(std::move(p.value()));
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return TemplateParam{ttype, qualifiers};
+    }
+    case clang::TemplateArgument::Integral: {
+      auto& ty = enumerateType(*p.getIntegralType());
+      llvm::SmallString<32> val;
+      p.getAsIntegral().toString(val);
+      return TemplateParam{ty, std::string(val)};
+    }
+    case clang::TemplateArgument::Template: {
+      return enumerateTemplateTemplateParam(p.getAsTemplate());
+    }
+
+#define X(name)                       \
+  case clang::TemplateArgument::name: \
+    throw std::logic_error("unsupported template argument kind: " #name);
+
+      X(Null)
+      X(Declaration)
+      X(NullPtr)
+      X(TemplateExpansion)
+      X(Expression)
+      X(Pack)
+#undef X
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateTemplateParam(
+    const clang::TemplateName& tn) {
+  switch (tn.getKind()) {
+    case clang::TemplateName::Template:
+      return std::nullopt;
+
+#define X(name)                   \
+  case clang::TemplateName::name: \
+    throw std::logic_error("unsupported template name kind: " #name);
+
+      X(OverloadedTemplate)
+      X(AssumedTemplate)
+      X(QualifiedTemplate)
+      X(DependentTemplate)
+      X(SubstTemplateTemplateParm)
+      X(SubstTemplateTemplateParmPack)
+      X(UsingTemplate)
+#undef X
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function references
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Float:
+      return makeType<Primitive>(ty, Primitive::Kind::Float32);
+    case clang::BuiltinType::Double:
+    case clang::BuiltinType::LongDouble:
+      return makeType<Primitive>(ty, Primitive::Kind::Float64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+ContainerInfo* ClangTypeParser::getContainerInfo(
+    const std::string& fqName) const {
+  for (const auto& containerInfo : containers_) {
+    if (std::regex_search(fqName, containerInfo->matcher)) {
+      return containerInfo.get();
+    }
+  }
+  return nullptr;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
+    return true;  // treat as complete
+
+  // TODO: This is a terrible warning.
+  return !sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      clang::diag::warn_nsconsumed_attribute_mismatch);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph

oi/type_graph/ClangTypeParser.h

@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ASTContext;
+class BuiltinType;
+class ConstantArrayType;
+class DecltypeType;
+class ElaboratedType;
+class EnumType;
+class LValueReferenceType;
+class PointerType;
+class RecordType;
+class Sema;
+class SubstTemplateTypeParmType;
+class TemplateArgument;
+class TemplateName;
+class TemplateSpecializationType;
+class Type;
+class TypedefType;
+class UnaryTransformType;
+class UsingType;
+}  // namespace clang
+
+struct ContainerInfo;
+
+namespace oi::detail::type_graph {
+
+class Array;
+class Class;
+class Enum;
+class Member;
+class Primitive;
+class Reference;
+class Type;
+class TypeGraph;
+class Typedef;
+struct TemplateParam;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+  bool readEnumValues = false;
+};
+
+/*
+ * ClangTypeParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given drgn_type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph,
+                  const std::vector<std::unique_ptr<ContainerInfo>>& containers,
+                  ClangTypeParserOptions options)
+      : typeGraph_{typeGraph}, containers_{containers}, options_{options} {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containers_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Type& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+  Typedef& enumerateTypedef(const clang::TypedefType&);
+  Typedef& enumerateUsing(const clang::UsingType&);
+  Type& enumerateUnaryTransformType(const clang::UnaryTransformType&);
+  Type& enumerateDecltypeType(const clang::DecltypeType&);
+
+  Array& enumerateArray(const clang::ConstantArrayType&);
+  Enum& enumerateEnum(const clang::EnumType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  std::optional<TemplateParam> enumerateTemplateParam(
+      const clang::TemplateArgument&);
+  std::optional<TemplateParam> enumerateTemplateTemplateParam(
+      const clang::TemplateName&);
+
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  ContainerInfo* getContainerInfo(const std::string& fqName) const;
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph

oi/type_graph/DrgnExporter.cpp

@@ -200,6 +200,14 @@ drgn_type* DrgnExporter::visit(Pointer& p) {
   return drgnType;
 }
 
+drgn_type* DrgnExporter::visit(Reference& p) {
+  auto* drgnType =
+      makeDrgnType(DRGN_TYPE_POINTER, false, DRGN_NOT_PRIMITIVE_TYPE, p);
+  auto* pointeeType = accept(p.pointeeType());
+  th_.pointerToTypeMap[drgnType] = pointeeType;
+  return drgnType;
+}
+
 drgn_type* DrgnExporter::visit(Dummy& d) {
   return makeDrgnType(DRGN_TYPE_VOID, false, DRGN_C_TYPE_VOID, d);
 }

oi/type_graph/DrgnExporter.h

@@ -54,6 +54,7 @@ class DrgnExporter : public Visitor<drgn_type*> {
   drgn_type* visit(Array&) override;
   drgn_type* visit(Typedef&) override;
   drgn_type* visit(Pointer&) override;
+  drgn_type* visit(Reference&) override;
   drgn_type* visit(Dummy&) override;
   drgn_type* visit(DummyAllocator&) override;
   drgn_type* visit(CaptureKeys&) override;

oi/type_graph/NameGen.cpp

@@ -200,6 +200,14 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& p) {
+  RecursiveVisitor::visit(p);
+  p.regenerateName();
+  std::string inputName{p.pointeeType().inputName()};
+  inputName += '*';
+  p.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();

oi/type_graph/NameGen.h

@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 

oi/type_graph/Printer.cpp

@@ -150,6 +150,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& p) {
+  if (prefix(p))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = p.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(p.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;

oi/type_graph/Printer.h

@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& p) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;

oi/type_graph/RemoveTopLevelPointer.cpp

@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& p) {
+  topLevelType_ = &p.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph

oi/type_graph/RemoveTopLevelPointer.h

@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
 
  private:
   Type* topLevelType_ = nullptr;

oi/type_graph/Types.h

@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -734,6 +735,61 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    name_ = pointeeType_.get().name() + "&";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *

oi/type_graph/Visitor.h

@@ -108,6 +108,9 @@ class RecursiveVisitor : public Visitor<void> {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -175,6 +178,10 @@ class RecursiveMutator : public Visitor<Type&> {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }

test/integration/template_template_param.toml

@@ -0,0 +1,16 @@
+includes = ["vector", "utility", "string"]
+
+definitions = '''
+template <typename K, typename V, template <typename, typename> typename Pair>
+struct bad_map {
+  std::vector<Pair<K, K>> keys;
+  std::vector<Pair<V, V>> values;
+};
+'''
+
+[cases]
+  [cases.int_int_empty]
+    param_types = ["const bad_map<int, std::string, std::pair>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'

tools/OILGen.cpp

@@ -14,131 +14,81 @@
  * limitations under the License.
  */
 
+#include <clang/Tooling/CommonOptionsParser.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include <cstdlib>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/action/drop_while.hpp>
+#include <range/v3/algorithm/find.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/all.hpp>
+#include <range/v3/view/transform.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
 
-constexpr static OIOpts opts{
+static llvm::cl::OptionCategory OilgenCategory("oilgen options");
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
 
-void usage() {
+static llvm::cl::list<std::string> ConfigFiles("config-file",
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
+                                               llvm::cl::desc(R"(TODO HELP)"),
-  std::cerr << opts;
+                                               llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<int> DebugLevel("debug-level",
+                                     llvm::cl::desc(R"(TODO HELP)"),
+                                     llvm::cl::init(-1),
+                                     llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> DumpJit("dump-jit",
+                                          llvm::cl::desc(R"(TODO HELP)"),
+                                          llvm::cl::init("jit.cpp"),
+                                          llvm::cl::cat(OilgenCategory));
 
-  std::cerr << std::endl
+int main(int argc, const char* argv[]) {
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
-
-int main(int argc, char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
+  auto expectedParser =
-  std::vector<fs::path> configFilePaths;
+      clang::tooling::CommonOptionsParser::create(argc, argv, OilgenCategory);
-  fs::path sourceFileDumpPath = "";
+  if (!expectedParser) {
-  bool exitCode = false;
+    llvm::errs() << expectedParser.takeError();
-  bool pic = false;
+    return -1;
-
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
   }
+  clang::tooling::CommonOptionsParser& options = expectedParser.get();
+  auto& compilations = options.getCompilations();
 
-  if (optind >= argc) {
+  if (DebugLevel.getNumOccurrences()) {
-    usage();
+    google::LogToStderr();
-    return EXIT_FAILURE;
+    google::SetStderrLogging(google::INFO);
-  }
+    google::SetVLOGLevel("*", DebugLevel);
-  fs::path primaryObject = argv[optind];
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
+    gflags::SetCommandLineOption("minloglevel", "0");
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
+  oigen.setConfigFilePaths(ConfigFiles |
-  oigen.setConfigFilePaths(std::move(configFilePaths));
+                           ranges::views::transform([](const auto& p) {
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
+                             return std::filesystem::path(p);
-  oigen.setFailIfNothingGenerated(exitCode);
+                           }) |
-  oigen.setUsePIC(pic);
+                           ranges::to<std::vector>());
+  if (DumpJit.getNumOccurrences())
+    oigen.setSourceFileDumpPath(DumpJit.getValue());
 
-  SymbolService symbols(primaryObject);
+  oigen.setOutputPath("a.o");  // TODO: make this an opt
 
-  return oigen.generate(primaryObject, symbols);
+  // TODO: can we extract this from the original arguments?
+  // oigen.setUsePIC(args["pic"].as<bool>());
+
+  oigen.setFailIfNothingGenerated(true);
+  // compilations.getAllFiles();
+  // return oigen.generate(compilations,
+  // {"/data/users/jakehillion/fbsource/fbcode/object-introspection/oil/examples/compile-time/VectorOfStrings-Introspect.cpp"});
+  return oigen.generate(compilations, options.getSourcePathList());
 }

types/string_type.toml

@@ -35,3 +35,34 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
     );
 }
 """
+
+traversal_func = """
+    bool sso = ((uintptr_t)container.data() <
+                (uintptr_t)(&container + sizeof(std::__cxx11::basic_string<T0>))) &&
+               ((uintptr_t)container.data() >= (uintptr_t)&container);
+
+    return returnArg.write(container.capacity())
+        .write(sso)
+        .write(container.size());
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+uint64_t capacity = std::get<ParsedData::VarInt>(d.val).value;
+el.container_stats.emplace(result::Element::ContainerStats { .capacity = capacity });
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+bool sso = std::get<ParsedData::VarInt>(d.val).value;
+if (!sso)
+  el.exclusive_size += el.container_stats->capacity * sizeof(T0);
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats->length = std::get<ParsedData::VarInt>(d.val).value;
+"""