From 9a1d34a806458dfa633caa79c28612ff694da3ed Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 1 Nov 2023 13:40:16 -0700
Subject: [PATCH 001/188] incomplete: name type in compiler errors

---
 oi/CodeGen.cpp                      |  9 ++++++
 oi/OITraceCode.cpp                  | 15 ++++++++++
 oi/type_graph/NameGen.cpp           |  5 ++++
 oi/type_graph/NameGen.h             |  1 +
 oi/type_graph/Printer.cpp           |  4 ++-
 oi/type_graph/TopoSorter.cpp        |  4 +++
 oi/type_graph/TopoSorter.h          |  1 +
 oi/type_graph/Types.cpp             |  2 --
 oi/type_graph/Types.h               | 45 ++++++++++++++++++++++++-----
 test/TypeGraphParser.cpp            |  5 ++--
 test/test_drgn_parser.cpp           |  4 +--
 test/test_enforce_compatibility.cpp |  4 +--
 test/test_flattener.cpp             |  2 +-
 test/test_name_gen.cpp              | 13 +++++++++
 types/shrd_ptr_type.toml            |  6 ++--
 types/uniq_ptr_type.toml            |  6 ++--
 16 files changed, 103 insertions(+), 23 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index fef0a47..eec2d25 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -55,6 +55,7 @@ using type_graph::DrgnParserOptions;
 using type_graph::EnforceCompatibility;
 using type_graph::Enum;
 using type_graph::Flattener;
+using type_graph::Incomplete;
 using type_graph::KeyCapture;
 using type_graph::Member;
 using type_graph::NameGen;
@@ -178,12 +179,20 @@ void genDeclsEnum(const Enum& e, std::string& code) {
   code += " {};\n";
 }
 
+void genDeclsIncomplete(const Incomplete& i, std::string& code) {
+  code += "template<> struct ";
+  code += i.name();
+  code += ";\n";
+}
+
 void genDecls(const TypeGraph& typeGraph, std::string& code) {
   for (const Type& t : typeGraph.finalTypes) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
       genDeclsClass(*c, code);
     } else if (const auto* e = dynamic_cast<const Enum*>(&t)) {
       genDeclsEnum(*e, code);
+    } else if (const auto* i = dynamic_cast<const Incomplete*>(&t)) {
+      genDeclsIncomplete(*i, code);
     }
   }
 }
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index 47e98d7..3978c24 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -153,3 +153,18 @@ bool isStorageInline(const auto& c) {
   return (uintptr_t)std::data(c) < (uintptr_t)(&c + sizeof(c)) &&
          (uintptr_t)std::data(c) >= (uintptr_t)&c;
 }
+
+namespace OIInternal {
+namespace {
+
+template <typename T>
+struct Incomplete;
+
+template <typename T>
+constexpr bool oi_is_complete = true;
+
+template <typename T>
+constexpr bool oi_is_complete<Incomplete<T>> = false;
+
+}  // namespace
+}  // namespace OIInternal
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index c792766..115c06c 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -204,4 +204,9 @@ void NameGen::visit(CaptureKeys& c) {
   c.regenerateName();
 }
 
+void NameGen::visit(Incomplete& i) {
+  RecursiveVisitor::visit(i);
+  i.regenerateName();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..4857ded 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -48,6 +48,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
+  void visit(Incomplete& i) override;
 
   static const inline std::string AnonPrefix = "__oi_anon";
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index e7d70eb..fc97905 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -37,7 +37,9 @@ void Printer::print(const Type& type) {
 }
 
 void Printer::visit(const Incomplete& i) {
-  prefix();
+  if (prefix(i))
+    return;
+
   out_ << "Incomplete";
   if (auto underlyingType = i.underlyingType()) {
     out_ << std::endl;
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index 6ca10af..ff2715a 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -139,6 +139,10 @@ void TopoSorter::visit(CaptureKeys& c) {
   sortedTypes_.push_back(c);
 }
 
+void TopoSorter::visit(Incomplete& i) {
+  sortedTypes_.push_back(i);
+}
+
 /*
  * A type graph may contain cycles, so we need to slightly tweak the standard
  * topological sorting algorithm. Cycles can only be introduced by certain
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 374e943..44f2364 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -48,6 +48,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
+  void visit(Incomplete& i) override;
 
  private:
   std::unordered_set<Type*> visited_;
diff --git a/oi/type_graph/Types.cpp b/oi/type_graph/Types.cpp
index bed5f5d..6e690b5 100644
--- a/oi/type_graph/Types.cpp
+++ b/oi/type_graph/Types.cpp
@@ -29,8 +29,6 @@ namespace oi::detail::type_graph {
 OI_TYPE_LIST
 #undef X
 
-const std::string Incomplete::kName = "void";
-
 std::string Primitive::getName(Kind kind) {
   switch (kind) {
     case Kind::Int8:
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index 40f1628..cf36c5c 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -29,6 +29,7 @@
  * debugging.
  */
 
+#include <algorithm>
 #include <cstddef>
 #include <map>
 #include <optional>
@@ -193,19 +194,20 @@ struct TemplateParam {
  */
 class Incomplete : public Type {
  public:
-  Incomplete(Type& underlyingType) : underlyingType_(underlyingType) {
+  Incomplete(NodeId id, Type& underlyingType)
+      : id_(id), underlyingType_(underlyingType) {
   }
 
-  Incomplete(std::string underlyingTypeName)
-      : underlyingType_(std::move(underlyingTypeName)) {
+  Incomplete(NodeId id, std::string underlyingTypeName)
+      : id_(id), underlyingType_(std::move(underlyingTypeName)) {
   }
 
-  static inline constexpr bool has_node_id = false;
+  static inline constexpr bool has_node_id = true;
 
   DECLARE_ACCEPT
 
   const std::string& name() const override {
-    return kName;
+    return name_;
   }
 
   std::string_view inputName() const override {
@@ -227,7 +229,35 @@ class Incomplete : public Type {
   }
 
   NodeId id() const override {
-    return -1;
+    return id_;
+  }
+
+  void regenerateName() {
+    std::string_view subName{std::visit(
+        [](const auto& el) -> std::string_view {
+          using T = std::decay_t<decltype(el)>;
+          if constexpr (std::is_same_v<T, std::string>) {
+            return el;
+          } else {
+            return el.get().name();
+          }
+        },
+        underlyingType_)};
+
+    constexpr std::string_view kPrefix{"Incomplete<struct "};
+    name_.clear();
+    name_.reserve(kPrefix.size() + subName.size() + 2);
+
+    name_ = kPrefix;
+    for (const auto c : subName) {
+      if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') ||
+          (c >= 'A' && c <= 'Z')) {
+        name_ += c;
+      } else {
+        name_ += '_';
+      }
+    }
+    name_ += ">";
   }
 
   std::optional<std::reference_wrapper<Type>> underlyingType() const {
@@ -239,8 +269,9 @@ class Incomplete : public Type {
   }
 
  private:
+  NodeId id_ = -1;
   std::variant<std::string, std::reference_wrapper<Type>> underlyingType_;
-  static const std::string kName;
+  std::string name_;
 };
 
 /*
diff --git a/test/TypeGraphParser.cpp b/test/TypeGraphParser.cpp
index 1eca163..4bb2ba0 100644
--- a/test/TypeGraphParser.cpp
+++ b/test/TypeGraphParser.cpp
@@ -215,10 +215,11 @@ Type& TypeGraphParser::parseType(std::string_view& input, size_t rootIndent) {
       auto nameEndPos = line.find(']', nameStartPos);
       auto underlyingTypeName =
           line.substr(nameStartPos, nameEndPos - nameStartPos);
-      type = &typeGraph_.makeType<Incomplete>(std::string(underlyingTypeName));
+      type =
+          &typeGraph_.makeType<Incomplete>(id, std::string(underlyingTypeName));
     } else {
       auto& underlyingType = parseType(input, indent + 2);
-      type = &typeGraph_.makeType<Incomplete>(underlyingType);
+      type = &typeGraph_.makeType<Incomplete>(id, underlyingType);
     }
   } else if (nodeTypeName == "Class" || nodeTypeName == "Struct" ||
              nodeTypeName == "Union") {
diff --git a/test/test_drgn_parser.cpp b/test/test_drgn_parser.cpp
index 7777639..3a9e37d 100644
--- a/test/test_drgn_parser.cpp
+++ b/test/test_drgn_parser.cpp
@@ -366,8 +366,8 @@ TEST_F(DrgnParserTest, PointerNoFollow) {
 
 TEST_F(DrgnParserTest, PointerIncomplete) {
   test("oid_test_case_pointers_incomplete_raw", R"(
-[0] Pointer
-      Incomplete: [IncompleteType]
+[1] Pointer
+[0]   Incomplete: [IncompleteType]
 )");
 }
 
diff --git a/test/test_enforce_compatibility.cpp b/test/test_enforce_compatibility.cpp
index 89b92d7..71ebd64 100644
--- a/test/test_enforce_compatibility.cpp
+++ b/test/test_enforce_compatibility.cpp
@@ -35,8 +35,8 @@ TEST(EnforceCompatibilityTest, VoidPointer) {
   test(EnforceCompatibility::createPass(), R"(
 [0] Class: MyClass (size: 8)
       Member: p (offset: 0)
-[1]     Pointer
-          Incomplete
+[2]     Pointer
+[1]       Incomplete
             Primitive: void
 )",
        R"(
diff --git a/test/test_flattener.cpp b/test/test_flattener.cpp
index ae6900b..8cdc2c8 100644
--- a/test/test_flattener.cpp
+++ b/test/test_flattener.cpp
@@ -958,7 +958,7 @@ TEST(FlattenerTest, IncompleteParent) {
   test(Flattener::createPass(), R"(
 [0] Class: MyClass (size: 4)
       Parent (offset: 0)
-        Incomplete: [IncompleteParent]
+[1]     Incomplete: [IncompleteParent]
 )",
        R"(
 [0] Class: MyClass (size: 4)
diff --git a/test/test_name_gen.cpp b/test/test_name_gen.cpp
index 72177d7..d721f7d 100644
--- a/test/test_name_gen.cpp
+++ b/test/test_name_gen.cpp
@@ -475,3 +475,16 @@ TEST(NameGenTest, AnonymousTypes) {
   EXPECT_EQ(myenum.inputName(), "__oi_anon_1");
   EXPECT_EQ(mytypedef.inputName(), "");
 }
+
+TEST(NameGenTest, IncompleteTypes) {
+  auto myincompletevector = Incomplete{0, "std::vector<int>"};
+
+  auto myint = Primitive{Primitive::Kind::Int32};
+  auto myincompleteint = Incomplete{1, myint};
+
+  NameGen nameGen;
+  nameGen.generateNames({myincompletevector, myincompleteint});
+
+  EXPECT_EQ(myincompletevector.name(), "Incomplete<struct std__vector_int_>");
+  EXPECT_EQ(myincompleteint.name(), "Incomplete<struct int32_t>");
+}
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 3154fe4..3f66c54 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (!std::is_void<T>::value && oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
         if (s_ptr && pointers.add((uintptr_t)(s_ptr.get()))) {
@@ -50,7 +50,7 @@ struct TypeHandler<DB, %1%<T0>> {
     static types::st::Unit<DB> getSizeType(
         const %1%<T0>& container,
         typename TypeHandler<DB, %1%<T0>>::type returnArg) {
-      if constexpr (!std::is_void<T0>::value) {
+      if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
         auto r0 = returnArg.write((uintptr_t)(container.get()));
         if (container && pointers.add((uintptr_t)(container.get()))) {
           return r0.template delegate<1>([&container](auto ret) {
@@ -117,7 +117,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<DB, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<DB, T0>("ptr_val");
     stack_ins(element);
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index eb3be12..6d0d6d3 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T,Deleter>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (!std::is_void<T>::value && oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
         if (u_ptr && pointers.add((uintptr_t)(u_ptr.get()))) {
@@ -51,7 +51,7 @@ struct TypeHandler<DB, %1%<T0,T1>> {
     static types::st::Unit<DB> getSizeType(
         const %1%<T0,T1>& container,
         typename TypeHandler<DB, %1%<T0,T1>>::type returnArg) {
-      if constexpr (!std::is_void<T0>::value) {
+      if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
         auto r0 = returnArg.write((uintptr_t)(container.get()));
         if (container && pointers.add((uintptr_t)(container.get()))) {
           return r0.template delegate<1>([&container](auto ret) {
@@ -103,7 +103,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<DB, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<DB, T0>("ptr_val");
     stack_ins(element);

From d94f660e8f2fac5961235d8bbf15f7f0aa13705d Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 1 Nov 2023 13:56:07 -0700
Subject: [PATCH 002/188] incomplete: name type in compiler errors

Summary:

We have a good type representation in the Type Graph of an incomplete type and
the underlying type that represents. However, this incomplete type still ends
up in the generated code as `void` which loses information. For example, a
container that can't contain void may fail to compile because it was
initialised with `void` but really its because the type it was supposed to be
initialised with (say, `Foo`) had incomplete debug information.

This change identifies that a type is incomplete in the output by generating it
as an incomplete type `struct Incomplete<struct Foo>`. This allows us to name
the type correctly in the TreeBuilder output and filter for incomplete types,
as well as getting appropriate compiler errors if it mustn't be incomplete.

Test Plan:

TODO
---
 oi/CodeGen.cpp                      |  9 ++++++
 oi/OITraceCode.cpp                  | 15 ++++++++++
 oi/type_graph/NameGen.cpp           |  5 ++++
 oi/type_graph/NameGen.h             |  1 +
 oi/type_graph/Printer.cpp           |  4 ++-
 oi/type_graph/TopoSorter.cpp        |  4 +++
 oi/type_graph/TopoSorter.h          |  1 +
 oi/type_graph/Types.cpp             |  2 --
 oi/type_graph/Types.h               | 45 ++++++++++++++++++++++++-----
 test/TypeGraphParser.cpp            |  5 ++--
 test/test_drgn_parser.cpp           |  4 +--
 test/test_enforce_compatibility.cpp |  4 +--
 test/test_flattener.cpp             |  2 +-
 test/test_name_gen.cpp              | 13 +++++++++
 types/shrd_ptr_type.toml            |  6 ++--
 types/uniq_ptr_type.toml            |  6 ++--
 16 files changed, 103 insertions(+), 23 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index fef0a47..eec2d25 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -55,6 +55,7 @@ using type_graph::DrgnParserOptions;
 using type_graph::EnforceCompatibility;
 using type_graph::Enum;
 using type_graph::Flattener;
+using type_graph::Incomplete;
 using type_graph::KeyCapture;
 using type_graph::Member;
 using type_graph::NameGen;
@@ -178,12 +179,20 @@ void genDeclsEnum(const Enum& e, std::string& code) {
   code += " {};\n";
 }
 
+void genDeclsIncomplete(const Incomplete& i, std::string& code) {
+  code += "template<> struct ";
+  code += i.name();
+  code += ";\n";
+}
+
 void genDecls(const TypeGraph& typeGraph, std::string& code) {
   for (const Type& t : typeGraph.finalTypes) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
       genDeclsClass(*c, code);
     } else if (const auto* e = dynamic_cast<const Enum*>(&t)) {
       genDeclsEnum(*e, code);
+    } else if (const auto* i = dynamic_cast<const Incomplete*>(&t)) {
+      genDeclsIncomplete(*i, code);
     }
   }
 }
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index 47e98d7..3978c24 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -153,3 +153,18 @@ bool isStorageInline(const auto& c) {
   return (uintptr_t)std::data(c) < (uintptr_t)(&c + sizeof(c)) &&
          (uintptr_t)std::data(c) >= (uintptr_t)&c;
 }
+
+namespace OIInternal {
+namespace {
+
+template <typename T>
+struct Incomplete;
+
+template <typename T>
+constexpr bool oi_is_complete = true;
+
+template <typename T>
+constexpr bool oi_is_complete<Incomplete<T>> = false;
+
+}  // namespace
+}  // namespace OIInternal
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index c792766..115c06c 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -204,4 +204,9 @@ void NameGen::visit(CaptureKeys& c) {
   c.regenerateName();
 }
 
+void NameGen::visit(Incomplete& i) {
+  RecursiveVisitor::visit(i);
+  i.regenerateName();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..4857ded 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -48,6 +48,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
+  void visit(Incomplete& i) override;
 
   static const inline std::string AnonPrefix = "__oi_anon";
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index e7d70eb..fc97905 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -37,7 +37,9 @@ void Printer::print(const Type& type) {
 }
 
 void Printer::visit(const Incomplete& i) {
-  prefix();
+  if (prefix(i))
+    return;
+
   out_ << "Incomplete";
   if (auto underlyingType = i.underlyingType()) {
     out_ << std::endl;
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index 6ca10af..ff2715a 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -139,6 +139,10 @@ void TopoSorter::visit(CaptureKeys& c) {
   sortedTypes_.push_back(c);
 }
 
+void TopoSorter::visit(Incomplete& i) {
+  sortedTypes_.push_back(i);
+}
+
 /*
  * A type graph may contain cycles, so we need to slightly tweak the standard
  * topological sorting algorithm. Cycles can only be introduced by certain
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 374e943..44f2364 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -48,6 +48,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
+  void visit(Incomplete& i) override;
 
  private:
   std::unordered_set<Type*> visited_;
diff --git a/oi/type_graph/Types.cpp b/oi/type_graph/Types.cpp
index bed5f5d..6e690b5 100644
--- a/oi/type_graph/Types.cpp
+++ b/oi/type_graph/Types.cpp
@@ -29,8 +29,6 @@ namespace oi::detail::type_graph {
 OI_TYPE_LIST
 #undef X
 
-const std::string Incomplete::kName = "void";
-
 std::string Primitive::getName(Kind kind) {
   switch (kind) {
     case Kind::Int8:
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index 40f1628..cf36c5c 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -29,6 +29,7 @@
  * debugging.
  */
 
+#include <algorithm>
 #include <cstddef>
 #include <map>
 #include <optional>
@@ -193,19 +194,20 @@ struct TemplateParam {
  */
 class Incomplete : public Type {
  public:
-  Incomplete(Type& underlyingType) : underlyingType_(underlyingType) {
+  Incomplete(NodeId id, Type& underlyingType)
+      : id_(id), underlyingType_(underlyingType) {
   }
 
-  Incomplete(std::string underlyingTypeName)
-      : underlyingType_(std::move(underlyingTypeName)) {
+  Incomplete(NodeId id, std::string underlyingTypeName)
+      : id_(id), underlyingType_(std::move(underlyingTypeName)) {
   }
 
-  static inline constexpr bool has_node_id = false;
+  static inline constexpr bool has_node_id = true;
 
   DECLARE_ACCEPT
 
   const std::string& name() const override {
-    return kName;
+    return name_;
   }
 
   std::string_view inputName() const override {
@@ -227,7 +229,35 @@ class Incomplete : public Type {
   }
 
   NodeId id() const override {
-    return -1;
+    return id_;
+  }
+
+  void regenerateName() {
+    std::string_view subName{std::visit(
+        [](const auto& el) -> std::string_view {
+          using T = std::decay_t<decltype(el)>;
+          if constexpr (std::is_same_v<T, std::string>) {
+            return el;
+          } else {
+            return el.get().name();
+          }
+        },
+        underlyingType_)};
+
+    constexpr std::string_view kPrefix{"Incomplete<struct "};
+    name_.clear();
+    name_.reserve(kPrefix.size() + subName.size() + 2);
+
+    name_ = kPrefix;
+    for (const auto c : subName) {
+      if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') ||
+          (c >= 'A' && c <= 'Z')) {
+        name_ += c;
+      } else {
+        name_ += '_';
+      }
+    }
+    name_ += ">";
   }
 
   std::optional<std::reference_wrapper<Type>> underlyingType() const {
@@ -239,8 +269,9 @@ class Incomplete : public Type {
   }
 
  private:
+  NodeId id_ = -1;
   std::variant<std::string, std::reference_wrapper<Type>> underlyingType_;
-  static const std::string kName;
+  std::string name_;
 };
 
 /*
diff --git a/test/TypeGraphParser.cpp b/test/TypeGraphParser.cpp
index 1eca163..4bb2ba0 100644
--- a/test/TypeGraphParser.cpp
+++ b/test/TypeGraphParser.cpp
@@ -215,10 +215,11 @@ Type& TypeGraphParser::parseType(std::string_view& input, size_t rootIndent) {
       auto nameEndPos = line.find(']', nameStartPos);
       auto underlyingTypeName =
           line.substr(nameStartPos, nameEndPos - nameStartPos);
-      type = &typeGraph_.makeType<Incomplete>(std::string(underlyingTypeName));
+      type =
+          &typeGraph_.makeType<Incomplete>(id, std::string(underlyingTypeName));
     } else {
       auto& underlyingType = parseType(input, indent + 2);
-      type = &typeGraph_.makeType<Incomplete>(underlyingType);
+      type = &typeGraph_.makeType<Incomplete>(id, underlyingType);
     }
   } else if (nodeTypeName == "Class" || nodeTypeName == "Struct" ||
              nodeTypeName == "Union") {
diff --git a/test/test_drgn_parser.cpp b/test/test_drgn_parser.cpp
index 7777639..3a9e37d 100644
--- a/test/test_drgn_parser.cpp
+++ b/test/test_drgn_parser.cpp
@@ -366,8 +366,8 @@ TEST_F(DrgnParserTest, PointerNoFollow) {
 
 TEST_F(DrgnParserTest, PointerIncomplete) {
   test("oid_test_case_pointers_incomplete_raw", R"(
-[0] Pointer
-      Incomplete: [IncompleteType]
+[1] Pointer
+[0]   Incomplete: [IncompleteType]
 )");
 }
 
diff --git a/test/test_enforce_compatibility.cpp b/test/test_enforce_compatibility.cpp
index 89b92d7..71ebd64 100644
--- a/test/test_enforce_compatibility.cpp
+++ b/test/test_enforce_compatibility.cpp
@@ -35,8 +35,8 @@ TEST(EnforceCompatibilityTest, VoidPointer) {
   test(EnforceCompatibility::createPass(), R"(
 [0] Class: MyClass (size: 8)
       Member: p (offset: 0)
-[1]     Pointer
-          Incomplete
+[2]     Pointer
+[1]       Incomplete
             Primitive: void
 )",
        R"(
diff --git a/test/test_flattener.cpp b/test/test_flattener.cpp
index ae6900b..8cdc2c8 100644
--- a/test/test_flattener.cpp
+++ b/test/test_flattener.cpp
@@ -958,7 +958,7 @@ TEST(FlattenerTest, IncompleteParent) {
   test(Flattener::createPass(), R"(
 [0] Class: MyClass (size: 4)
       Parent (offset: 0)
-        Incomplete: [IncompleteParent]
+[1]     Incomplete: [IncompleteParent]
 )",
        R"(
 [0] Class: MyClass (size: 4)
diff --git a/test/test_name_gen.cpp b/test/test_name_gen.cpp
index 72177d7..d721f7d 100644
--- a/test/test_name_gen.cpp
+++ b/test/test_name_gen.cpp
@@ -475,3 +475,16 @@ TEST(NameGenTest, AnonymousTypes) {
   EXPECT_EQ(myenum.inputName(), "__oi_anon_1");
   EXPECT_EQ(mytypedef.inputName(), "");
 }
+
+TEST(NameGenTest, IncompleteTypes) {
+  auto myincompletevector = Incomplete{0, "std::vector<int>"};
+
+  auto myint = Primitive{Primitive::Kind::Int32};
+  auto myincompleteint = Incomplete{1, myint};
+
+  NameGen nameGen;
+  nameGen.generateNames({myincompletevector, myincompleteint});
+
+  EXPECT_EQ(myincompletevector.name(), "Incomplete<struct std__vector_int_>");
+  EXPECT_EQ(myincompleteint.name(), "Incomplete<struct int32_t>");
+}
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 3154fe4..3f66c54 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (!std::is_void<T>::value && oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
         if (s_ptr && pointers.add((uintptr_t)(s_ptr.get()))) {
@@ -50,7 +50,7 @@ struct TypeHandler<DB, %1%<T0>> {
     static types::st::Unit<DB> getSizeType(
         const %1%<T0>& container,
         typename TypeHandler<DB, %1%<T0>>::type returnArg) {
-      if constexpr (!std::is_void<T0>::value) {
+      if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
         auto r0 = returnArg.write((uintptr_t)(container.get()));
         if (container && pointers.add((uintptr_t)(container.get()))) {
           return r0.template delegate<1>([&container](auto ret) {
@@ -117,7 +117,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<DB, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<DB, T0>("ptr_val");
     stack_ins(element);
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index eb3be12..6d0d6d3 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T,Deleter>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (!std::is_void<T>::value && oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
         if (u_ptr && pointers.add((uintptr_t)(u_ptr.get()))) {
@@ -51,7 +51,7 @@ struct TypeHandler<DB, %1%<T0,T1>> {
     static types::st::Unit<DB> getSizeType(
         const %1%<T0,T1>& container,
         typename TypeHandler<DB, %1%<T0,T1>>::type returnArg) {
-      if constexpr (!std::is_void<T0>::value) {
+      if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
         auto r0 = returnArg.write((uintptr_t)(container.get()));
         if (container && pointers.add((uintptr_t)(container.get()))) {
           return r0.template delegate<1>([&container](auto ret) {
@@ -103,7 +103,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<DB, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<DB, T0>("ptr_val");
     stack_ins(element);

From 59cad54ea248b0376b69ea25677045af16216a43 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 3 Nov 2023 06:06:25 -0700
Subject: [PATCH 003/188] collapse TreeBuilderV2 features

Summary:

Currently there are two features between CodeGen v2 (TypeGraph) and TreeBuilder
v2. These are TypedDataSegment and TreeBuilderTypeChecking. Each of these
features currently has a full set of tests run in the CI and each have specific
exclusions.

Collapse these features into TreeBuilder v2. This allows for significantly
simplified testing as any OIL tests run under TreeBuilder v2 and any OID tests
run under TreeBuilder v1.

The reasoning behind this is I no longer intend to partially roll out this
feature. Full TreeBuilder v2 applies different conditions to containers than
the intermediate states, and writing these only to have them never deployed is
a waste of time.

Test Plan:
- it builds
- CI
---
 .circleci/config.yml | 14 -------
 oi/CodeGen.cpp       | 49 ++++++-----------------
 oi/ContainerInfo.cpp |  6 +--
 oi/ContainerInfo.h   |  1 -
 oi/Features.cpp      | 13 +-----
 oi/Features.h        |  2 -
 oi/FuncGen.cpp       | 94 --------------------------------------------
 oi/FuncGen.h         |  3 --
 oi/OICompiler.cpp    |  5 +--
 oi/OIGenerator.cpp   |  2 -
 oi/OILibraryImpl.cpp |  2 -
 types/README.md      | 10 ++---
 12 files changed, 20 insertions(+), 181 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 286f759..385d2e0 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -21,20 +21,6 @@ workflows:
           oid_test_args: "-ftype-graph"
           tests_regex: "OidIntegration\\..*"
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
-      - test:
-          name: test-typed-data-segment-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-ftyped-data-segment"
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
-      - test:
-          name: test-tree-builder-type-checking-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-ftree-builder-type-checking"
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
       - coverage:
           name: coverage
           requires:
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index fef0a47..1830624 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -117,17 +117,14 @@ void addIncludes(const TypeGraph& typeGraph,
                  FeatureSet features,
                  std::string& code) {
   std::set<std::string_view> includes{"cstddef"};
-  if (features[Feature::TypedDataSegment]) {
+  if (features[Feature::TreeBuilderV2]) {
+    code += "#define DEFINE_DESCRIBE 1\n";  // added before all includes
+
     includes.emplace("functional");
+    includes.emplace("oi/exporters/inst.h");
+    includes.emplace("oi/types/dy.h");
     includes.emplace("oi/types/st.h");
   }
-  if (features[Feature::TreeBuilderTypeChecking]) {
-    includes.emplace("oi/types/dy.h");
-
-    code += "#define DEFINE_DESCRIBE 1\n";  // added before all includes
-  }
-  if (features[Feature::TreeBuilderV2])
-    includes.emplace("oi/exporters/inst.h");
   if (features[Feature::Library]) {
     includes.emplace("vector");
     includes.emplace("oi/IntrospectionResult.h");
@@ -830,8 +827,7 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) {
   code += "  using type = ";
   genClassStaticType(c, code);
   code += ";\n";
-  if (config_.features[Feature::TreeBuilderV2])
-    genClassTreeBuilderInstructions(c, code);
+  genClassTreeBuilderInstructions(c, code);
   genClassTraversalFunction(c, code);
   code += "};\n";
 }
@@ -846,19 +842,6 @@ void genContainerTypeHandler(FeatureSet features,
   if (!used.insert(&c).second)
     return;
 
-  if (!features[Feature::TreeBuilderV2]) {
-    const auto& handler = c.codegen.handler;
-    if (handler.empty()) {
-      LOG(ERROR) << "`codegen.handler` must be specified for all containers "
-                    "under \"-ftyped-data-segment\", not specified for \"" +
-                        c.typeName + "\"";
-      throw std::runtime_error("missing `codegen.handler`");
-    }
-    auto fmt = boost::format(c.codegen.handler) % c.typeName;
-    code += fmt.str();
-    return;
-  }
-
   code += c.codegen.extra;
 
   // TODO: Move this check into the ContainerInfo parsing once always enabled.
@@ -1216,14 +1199,14 @@ void CodeGen::generate(
   if (!config_.features[Feature::Library]) {
     FuncGen::DeclareExterns(code);
   }
-  if (!config_.features[Feature::TypedDataSegment]) {
+  if (!config_.features[Feature::TreeBuilderV2]) {
     defineMacros(code);
   }
   addIncludes(typeGraph, config_.features, code);
   defineInternalTypes(code);
   FuncGen::DefineJitLog(code, config_.features);
 
-  if (config_.features[Feature::TypedDataSegment]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     if (config_.features[Feature::Library]) {
       FuncGen::DefineBackInserterDataBuffer(code);
     } else {
@@ -1231,10 +1214,8 @@ void CodeGen::generate(
     }
     code += "using namespace oi;\n";
     code += "using namespace oi::detail;\n";
-    if (config_.features[Feature::TreeBuilderV2]) {
-      code += "using oi::exporters::ParsedData;\n";
-      code += "using namespace oi::exporters;\n";
-    }
+    code += "using oi::exporters::ParsedData;\n";
+    code += "using namespace oi::exporters;\n";
     code += "namespace OIInternal {\nnamespace {\n";
     FuncGen::DefineBasicTypeHandlers(code, config_.features);
     code += "} // namespace\n} // namespace OIInternal\n";
@@ -1254,7 +1235,7 @@ void CodeGen::generate(
    * process faster.
    */
   code += "namespace OIInternal {\nnamespace {\n";
-  if (!config_.features[Feature::TypedDataSegment]) {
+  if (!config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineEncodeData(code);
     FuncGen::DefineEncodeDataSize(code);
     FuncGen::DefineStoreData(code);
@@ -1269,7 +1250,7 @@ void CodeGen::generate(
     genExclusiveSizes(typeGraph, code);
   }
 
-  if (config_.features[Feature::TypedDataSegment]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     addStandardTypeHandlers(typeGraph, config_.features, code);
     addTypeHandlers(typeGraph, code);
   } else {
@@ -1286,10 +1267,8 @@ void CodeGen::generate(
   code += "} // namespace\n} // namespace OIInternal\n";
 
   const auto typeName = SymbolService::getTypeName(drgnType);
-  if (config_.features[Feature::Library]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
-  } else if (config_.features[Feature::TypedDataSegment]) {
-    FuncGen::DefineTopLevelGetSizeRefTyped(code, typeName, config_.features);
   } else {
     FuncGen::DefineTopLevelGetSizeRef(code, typeName, config_.features);
   }
@@ -1298,8 +1277,6 @@ void CodeGen::generate(
     FuncGen::DefineTreeBuilderInstructions(code, typeName,
                                            calculateExclusiveSize(rootType),
                                            enumerateTypeNames(rootType));
-  } else if (config_.features[Feature::TreeBuilderTypeChecking]) {
-    FuncGen::DefineOutputType(code, typeName);
   }
 
   if (!linkageName_.empty())
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index a5703be..40ca6b4 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -274,10 +274,6 @@ ContainerInfo::ContainerInfo(const fs::path& path) {
   } else {
     throw ContainerInfoError(path, "`codegen.decl` is a required field");
   }
-  if (std::optional<std::string> str =
-          codegenToml["handler"].value<std::string>()) {
-    codegen.handler = std::move(*str);
-  }
   if (std::optional<std::string> str =
           codegenToml["traversal_func"].value<std::string>()) {
     codegen.traversalFunc = std::move(*str);
@@ -324,5 +320,5 @@ ContainerInfo::ContainerInfo(std::string typeName_,
       ctype(ctype_),
       header(std::move(header_)),
       codegen(Codegen{"// DummyDecl %1%\n", "// DummyFunc %1%\n",
-                      "// DummyHandler %1%\n", "// DummyFunc\n"}) {
+                      "// DummyFunc\n"}) {
 }
diff --git a/oi/ContainerInfo.h b/oi/ContainerInfo.h
index 0ad9da9..376274d 100644
--- a/oi/ContainerInfo.h
+++ b/oi/ContainerInfo.h
@@ -36,7 +36,6 @@ struct ContainerInfo {
   struct Codegen {
     std::string decl;
     std::string func;
-    std::string handler = "";
     std::string traversalFunc = "";
     std::string extra = "";
     std::vector<Processor> processors{};
diff --git a/oi/Features.cpp b/oi/Features.cpp
index baf9344..9d82b17 100644
--- a/oi/Features.cpp
+++ b/oi/Features.cpp
@@ -40,11 +40,6 @@ std::optional<std::string_view> featureHelp(Feature f) {
       return "Use Type Graph for code generation (CodeGen v2).";
     case Feature::PruneTypeGraph:
       return "Prune unreachable nodes from the type graph";
-    case Feature::TypedDataSegment:
-      return "Use Typed Data Segment in generated code.";
-    case Feature::TreeBuilderTypeChecking:
-      return "Use Typed Data Segment to perform runtime Type Checking in "
-             "TreeBuilder.";
     case Feature::Library:
       return std::nullopt;  // Hide in OID help
     case Feature::TreeBuilderV2:
@@ -65,14 +60,8 @@ std::optional<std::string_view> featureHelp(Feature f) {
 
 std::span<const Feature> requirements(Feature f) {
   switch (f) {
-    case Feature::TypedDataSegment:
-      static constexpr std::array tds = {Feature::TypeGraph};
-      return tds;
-    case Feature::TreeBuilderTypeChecking:
-      static constexpr std::array tc = {Feature::TypedDataSegment};
-      return tc;
     case Feature::TreeBuilderV2:
-      static constexpr std::array tb2 = {Feature::TreeBuilderTypeChecking};
+      static constexpr std::array tb2 = {Feature::TypeGraph};
       return tb2;
     case Feature::Library:
       static constexpr std::array lib = {Feature::TreeBuilderV2};
diff --git a/oi/Features.h b/oi/Features.h
index 2e92ba0..1d039b7 100644
--- a/oi/Features.h
+++ b/oi/Features.h
@@ -29,8 +29,6 @@
   X(CaptureThriftIsset, "capture-thrift-isset")            \
   X(TypeGraph, "type-graph")                               \
   X(PruneTypeGraph, "prune-type-graph")                    \
-  X(TypedDataSegment, "typed-data-segment")                \
-  X(TreeBuilderTypeChecking, "tree-builder-type-checking") \
   X(Library, "library")                                    \
   X(TreeBuilderV2, "tree-builder-v2")                      \
   X(GenJitDebug, "gen-jit-debug")                          \
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index dc494f9..d4f0e1e 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -352,83 +352,6 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
   testCode.append(fmt.str());
 }
 
-/*
- * DefineTopLevelGetSizeRefTyped
- *
- * Top level function to run OI on a type utilising static types and enabled
- * with feature '-ftyped-data-segment'.
- */
-void FuncGen::DefineTopLevelGetSizeRefTyped(std::string& testCode,
-                                            const std::string& rawType,
-                                            FeatureSet features) {
-  std::string func = R"(
-    #pragma GCC diagnostic push
-    #pragma GCC diagnostic ignored "-Wunknown-attributes"
-    /* RawType: %1% */
-    void __attribute__((used, retain)) getSize_%2$016x(const OIInternal::__ROOT_TYPE__& t)
-    #pragma GCC diagnostic pop
-    {
-    )";
-  if (features[Feature::JitTiming]) {
-    func += "      const auto startTime = std::chrono::steady_clock::now();\n";
-  }
-  func += R"(
-      pointers.initialize();
-      pointers.add((uintptr_t)&t);
-      auto data = reinterpret_cast<uintptr_t*>(dataBase);
-
-      // TODO: Replace these with types::st::Uint64 once the VarInt decoding
-      // logic is moved out of OIDebugger and into new TreeBuilder.
-      size_t dataSegOffset = 0;
-      data[dataSegOffset++] = oidMagicId;
-      data[dataSegOffset++] = cookieValue;
-      uintptr_t& writtenSize = data[dataSegOffset++];
-      writtenSize = 0;
-      uintptr_t& timeTakenNs = data[dataSegOffset++];
-
-      dataSegOffset *= sizeof(uintptr_t);
-      JLOG("%1% @");
-      JLOGPTR(&t);
-
-      using ContentType = OIInternal::TypeHandler<DataBuffer::DataSegment, OIInternal::__ROOT_TYPE__>::type;
-      using SuffixType = types::st::Pair<
-        DataBuffer::DataSegment,
-        types::st::VarInt<DataBuffer::DataSegment>,
-        types::st::VarInt<DataBuffer::DataSegment>
-      >;
-      using DataBufferType = types::st::Pair<
-        DataBuffer::DataSegment,
-        ContentType,
-        SuffixType
-      >;
-
-      DataBufferType db = DataBuffer::DataSegment(dataSegOffset);
-      SuffixType suffix = db.delegate([&t](auto ret) {
-        return OIInternal::getSizeType<DataBuffer::DataSegment>(t, ret);
-      });
-      types::st::Unit<DataBuffer::DataSegment> end = suffix
-        .write(123456789)
-        .write(123456789);
-
-      dataSegOffset = end.offset();
-      writtenSize = dataSegOffset;
-      dataBase += dataSegOffset;
-    )";
-  if (features[Feature::JitTiming]) {
-    func += R"(
-      timeTakenNs = std::chrono::duration_cast<std::chrono::nanoseconds>(
-        std::chrono::steady_clock::now() - startTime).count();
-      )";
-  }
-  func += R"(
-    }
-    )";
-
-  boost::format fmt =
-      boost::format(func) % rawType % std::hash<std::string>{}(rawType);
-  testCode.append(fmt.str());
-}
-
 /*
  * DefineOutputType
  *
@@ -803,23 +726,6 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
   ContainerInfo oiArray{"OIArray", UNKNOWN_TYPE,
                         "cstdint"};  // TODO: remove the need for a dummy header
 
-  oiArray.codegen.handler = R"(
-template<typename DB, typename T0, long unsigned int N>
-struct TypeHandler<DB, %1%<T0, N>> {
-  using type = types::st::List<DB, typename TypeHandler<DB, T0>::type>;
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, N> &container,
-      typename TypeHandler<DB, %1%<T0,N>>::type returnArg) {
-    auto tail = returnArg.write(N);
-    for (size_t i=0; i<N; i++) {
-      tail = tail.delegate([&container, i](auto ret) {
-          return TypeHandler<DB, T0>::getSizeType(container.vals[i], ret);
-      });
-    }
-    return tail.finish();
-  }
-};
-)";
   oiArray.codegen.traversalFunc = R"(
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
diff --git a/oi/FuncGen.h b/oi/FuncGen.h
index ff21841..042c0bc 100644
--- a/oi/FuncGen.h
+++ b/oi/FuncGen.h
@@ -61,9 +61,6 @@ class FuncGen {
   static void DefineTopLevelGetSizeRef(std::string& testCode,
                                        const std::string& rawType,
                                        FeatureSet features);
-  static void DefineTopLevelGetSizeRefTyped(std::string& testCode,
-                                            const std::string& rawType,
-                                            FeatureSet features);
   static void DefineOutputType(std::string& testCode,
                                const std::string& rawType);
   static void DefineTreeBuilderInstructions(
diff --git a/oi/OICompiler.cpp b/oi/OICompiler.cpp
index d036b33..fff8042 100644
--- a/oi/OICompiler.cpp
+++ b/oi/OICompiler.cpp
@@ -520,9 +520,8 @@ bool OICompiler::compile(const std::string& code,
 
   static const auto syntheticHeaders = std::array<
       std::pair<Feature, std::pair<std::string_view, std::string>>, 7>{{
-      {Feature::TypedDataSegment, {headers::oi_types_st_h, "oi/types/st.h"}},
-      {Feature::TreeBuilderTypeChecking,
-       {headers::oi_types_dy_h, "oi/types/dy.h"}},
+      {Feature::TreeBuilderV2, {headers::oi_types_st_h, "oi/types/st.h"}},
+      {Feature::TreeBuilderV2, {headers::oi_types_dy_h, "oi/types/dy.h"}},
       {Feature::TreeBuilderV2,
        {headers::oi_exporters_inst_h, "oi/exporters/inst.h"}},
       {Feature::TreeBuilderV2,
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index d98e7b8..d929de1 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -181,8 +181,6 @@ int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
 
   std::map<Feature, bool> featuresMap = {
       {Feature::TypeGraph, true},
-      {Feature::TypedDataSegment, true},
-      {Feature::TreeBuilderTypeChecking, true},
       {Feature::TreeBuilderV2, true},
       {Feature::Library, true},
       {Feature::PackStructs, true},
diff --git a/oi/OILibraryImpl.cpp b/oi/OILibraryImpl.cpp
index bade97c..6562cc8 100644
--- a/oi/OILibraryImpl.cpp
+++ b/oi/OILibraryImpl.cpp
@@ -174,8 +174,6 @@ namespace {
 std::map<Feature, bool> convertFeatures(std::unordered_set<oi::Feature> fs) {
   std::map<Feature, bool> out{
       {Feature::TypeGraph, true},
-      {Feature::TypedDataSegment, true},
-      {Feature::TreeBuilderTypeChecking, true},
       {Feature::TreeBuilderV2, true},
       {Feature::Library, true},
       {Feature::PackStructs, true},
diff --git a/types/README.md b/types/README.md
index ebde86c..e2b70fd 100644
--- a/types/README.md
+++ b/types/README.md
@@ -62,14 +62,10 @@ This document describes the format of the container definition files contained i
   will collide if they share the same name.
 
 
-## Changes introduced with Typed Data Segment
-- `decl` and `func` fields are ignored when using `-ftyped-data-segment` and the
-  `handler` field is used instead.
-
 ## Changes introduced with TreeBuilder V2
-- `decl`, `func`, and `handler` fields are ignored when using `-ftree-builder-v2`.
-   The `TypeHandler` is instead constructed from `traversal_func` and `processor`
-   entries.
+- `decl` and `func` fields are ignored when using `-ftree-builder-v2`. The
+  `TypeHandler` is constructed from `traversal_func` field and `processor`
+  entries.
 
 ## Changes introduced with TypeGraph
 - `typeName` and `matcher` fields have been merged into the single field `type_name`.

From 6c88c79083fe8167c4009ad05f57f5b45aa0075a Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 3 Nov 2023 06:06:25 -0700
Subject: [PATCH 004/188] collapse TreeBuilderV2 features

Summary:

Currently there are two features between CodeGen v2 (TypeGraph) and TreeBuilder
v2. These are TypedDataSegment and TreeBuilderTypeChecking. Each of these
features currently has a full set of tests run in the CI and each have specific
exclusions.

Collapse these features into TreeBuilder v2. This allows for significantly
simplified testing as any OIL tests run under TreeBuilder v2 and any OID tests
run under TreeBuilder v1.

The reasoning behind this is I no longer intend to partially roll out this
feature. Full TreeBuilder v2 applies different conditions to containers than
the intermediate states, and writing these only to have them never deployed is
a waste of time.

Test Plan:
- it builds
- CI
---
 .circleci/config.yml | 14 -------
 oi/CodeGen.cpp       | 49 ++++++-----------------
 oi/ContainerInfo.cpp |  6 +--
 oi/ContainerInfo.h   |  1 -
 oi/Features.cpp      | 13 +-----
 oi/Features.h        |  2 -
 oi/FuncGen.cpp       | 94 --------------------------------------------
 oi/FuncGen.h         |  3 --
 oi/OICompiler.cpp    |  5 +--
 oi/OIGenerator.cpp   |  2 -
 oi/OILibraryImpl.cpp |  2 -
 types/README.md      | 10 ++---
 12 files changed, 20 insertions(+), 181 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 286f759..385d2e0 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -21,20 +21,6 @@ workflows:
           oid_test_args: "-ftype-graph"
           tests_regex: "OidIntegration\\..*"
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
-      - test:
-          name: test-typed-data-segment-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-ftyped-data-segment"
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
-      - test:
-          name: test-tree-builder-type-checking-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-ftree-builder-type-checking"
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
       - coverage:
           name: coverage
           requires:
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index fef0a47..1830624 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -117,17 +117,14 @@ void addIncludes(const TypeGraph& typeGraph,
                  FeatureSet features,
                  std::string& code) {
   std::set<std::string_view> includes{"cstddef"};
-  if (features[Feature::TypedDataSegment]) {
+  if (features[Feature::TreeBuilderV2]) {
+    code += "#define DEFINE_DESCRIBE 1\n";  // added before all includes
+
     includes.emplace("functional");
+    includes.emplace("oi/exporters/inst.h");
+    includes.emplace("oi/types/dy.h");
     includes.emplace("oi/types/st.h");
   }
-  if (features[Feature::TreeBuilderTypeChecking]) {
-    includes.emplace("oi/types/dy.h");
-
-    code += "#define DEFINE_DESCRIBE 1\n";  // added before all includes
-  }
-  if (features[Feature::TreeBuilderV2])
-    includes.emplace("oi/exporters/inst.h");
   if (features[Feature::Library]) {
     includes.emplace("vector");
     includes.emplace("oi/IntrospectionResult.h");
@@ -830,8 +827,7 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) {
   code += "  using type = ";
   genClassStaticType(c, code);
   code += ";\n";
-  if (config_.features[Feature::TreeBuilderV2])
-    genClassTreeBuilderInstructions(c, code);
+  genClassTreeBuilderInstructions(c, code);
   genClassTraversalFunction(c, code);
   code += "};\n";
 }
@@ -846,19 +842,6 @@ void genContainerTypeHandler(FeatureSet features,
   if (!used.insert(&c).second)
     return;
 
-  if (!features[Feature::TreeBuilderV2]) {
-    const auto& handler = c.codegen.handler;
-    if (handler.empty()) {
-      LOG(ERROR) << "`codegen.handler` must be specified for all containers "
-                    "under \"-ftyped-data-segment\", not specified for \"" +
-                        c.typeName + "\"";
-      throw std::runtime_error("missing `codegen.handler`");
-    }
-    auto fmt = boost::format(c.codegen.handler) % c.typeName;
-    code += fmt.str();
-    return;
-  }
-
   code += c.codegen.extra;
 
   // TODO: Move this check into the ContainerInfo parsing once always enabled.
@@ -1216,14 +1199,14 @@ void CodeGen::generate(
   if (!config_.features[Feature::Library]) {
     FuncGen::DeclareExterns(code);
   }
-  if (!config_.features[Feature::TypedDataSegment]) {
+  if (!config_.features[Feature::TreeBuilderV2]) {
     defineMacros(code);
   }
   addIncludes(typeGraph, config_.features, code);
   defineInternalTypes(code);
   FuncGen::DefineJitLog(code, config_.features);
 
-  if (config_.features[Feature::TypedDataSegment]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     if (config_.features[Feature::Library]) {
       FuncGen::DefineBackInserterDataBuffer(code);
     } else {
@@ -1231,10 +1214,8 @@ void CodeGen::generate(
     }
     code += "using namespace oi;\n";
     code += "using namespace oi::detail;\n";
-    if (config_.features[Feature::TreeBuilderV2]) {
-      code += "using oi::exporters::ParsedData;\n";
-      code += "using namespace oi::exporters;\n";
-    }
+    code += "using oi::exporters::ParsedData;\n";
+    code += "using namespace oi::exporters;\n";
     code += "namespace OIInternal {\nnamespace {\n";
     FuncGen::DefineBasicTypeHandlers(code, config_.features);
     code += "} // namespace\n} // namespace OIInternal\n";
@@ -1254,7 +1235,7 @@ void CodeGen::generate(
    * process faster.
    */
   code += "namespace OIInternal {\nnamespace {\n";
-  if (!config_.features[Feature::TypedDataSegment]) {
+  if (!config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineEncodeData(code);
     FuncGen::DefineEncodeDataSize(code);
     FuncGen::DefineStoreData(code);
@@ -1269,7 +1250,7 @@ void CodeGen::generate(
     genExclusiveSizes(typeGraph, code);
   }
 
-  if (config_.features[Feature::TypedDataSegment]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     addStandardTypeHandlers(typeGraph, config_.features, code);
     addTypeHandlers(typeGraph, code);
   } else {
@@ -1286,10 +1267,8 @@ void CodeGen::generate(
   code += "} // namespace\n} // namespace OIInternal\n";
 
   const auto typeName = SymbolService::getTypeName(drgnType);
-  if (config_.features[Feature::Library]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
-  } else if (config_.features[Feature::TypedDataSegment]) {
-    FuncGen::DefineTopLevelGetSizeRefTyped(code, typeName, config_.features);
   } else {
     FuncGen::DefineTopLevelGetSizeRef(code, typeName, config_.features);
   }
@@ -1298,8 +1277,6 @@ void CodeGen::generate(
     FuncGen::DefineTreeBuilderInstructions(code, typeName,
                                            calculateExclusiveSize(rootType),
                                            enumerateTypeNames(rootType));
-  } else if (config_.features[Feature::TreeBuilderTypeChecking]) {
-    FuncGen::DefineOutputType(code, typeName);
   }
 
   if (!linkageName_.empty())
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index a5703be..40ca6b4 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -274,10 +274,6 @@ ContainerInfo::ContainerInfo(const fs::path& path) {
   } else {
     throw ContainerInfoError(path, "`codegen.decl` is a required field");
   }
-  if (std::optional<std::string> str =
-          codegenToml["handler"].value<std::string>()) {
-    codegen.handler = std::move(*str);
-  }
   if (std::optional<std::string> str =
           codegenToml["traversal_func"].value<std::string>()) {
     codegen.traversalFunc = std::move(*str);
@@ -324,5 +320,5 @@ ContainerInfo::ContainerInfo(std::string typeName_,
       ctype(ctype_),
       header(std::move(header_)),
       codegen(Codegen{"// DummyDecl %1%\n", "// DummyFunc %1%\n",
-                      "// DummyHandler %1%\n", "// DummyFunc\n"}) {
+                      "// DummyFunc\n"}) {
 }
diff --git a/oi/ContainerInfo.h b/oi/ContainerInfo.h
index 0ad9da9..376274d 100644
--- a/oi/ContainerInfo.h
+++ b/oi/ContainerInfo.h
@@ -36,7 +36,6 @@ struct ContainerInfo {
   struct Codegen {
     std::string decl;
     std::string func;
-    std::string handler = "";
     std::string traversalFunc = "";
     std::string extra = "";
     std::vector<Processor> processors{};
diff --git a/oi/Features.cpp b/oi/Features.cpp
index baf9344..9d82b17 100644
--- a/oi/Features.cpp
+++ b/oi/Features.cpp
@@ -40,11 +40,6 @@ std::optional<std::string_view> featureHelp(Feature f) {
       return "Use Type Graph for code generation (CodeGen v2).";
     case Feature::PruneTypeGraph:
       return "Prune unreachable nodes from the type graph";
-    case Feature::TypedDataSegment:
-      return "Use Typed Data Segment in generated code.";
-    case Feature::TreeBuilderTypeChecking:
-      return "Use Typed Data Segment to perform runtime Type Checking in "
-             "TreeBuilder.";
     case Feature::Library:
       return std::nullopt;  // Hide in OID help
     case Feature::TreeBuilderV2:
@@ -65,14 +60,8 @@ std::optional<std::string_view> featureHelp(Feature f) {
 
 std::span<const Feature> requirements(Feature f) {
   switch (f) {
-    case Feature::TypedDataSegment:
-      static constexpr std::array tds = {Feature::TypeGraph};
-      return tds;
-    case Feature::TreeBuilderTypeChecking:
-      static constexpr std::array tc = {Feature::TypedDataSegment};
-      return tc;
     case Feature::TreeBuilderV2:
-      static constexpr std::array tb2 = {Feature::TreeBuilderTypeChecking};
+      static constexpr std::array tb2 = {Feature::TypeGraph};
       return tb2;
     case Feature::Library:
       static constexpr std::array lib = {Feature::TreeBuilderV2};
diff --git a/oi/Features.h b/oi/Features.h
index 2e92ba0..1d039b7 100644
--- a/oi/Features.h
+++ b/oi/Features.h
@@ -29,8 +29,6 @@
   X(CaptureThriftIsset, "capture-thrift-isset")            \
   X(TypeGraph, "type-graph")                               \
   X(PruneTypeGraph, "prune-type-graph")                    \
-  X(TypedDataSegment, "typed-data-segment")                \
-  X(TreeBuilderTypeChecking, "tree-builder-type-checking") \
   X(Library, "library")                                    \
   X(TreeBuilderV2, "tree-builder-v2")                      \
   X(GenJitDebug, "gen-jit-debug")                          \
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index dc494f9..d4f0e1e 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -352,83 +352,6 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
   testCode.append(fmt.str());
 }
 
-/*
- * DefineTopLevelGetSizeRefTyped
- *
- * Top level function to run OI on a type utilising static types and enabled
- * with feature '-ftyped-data-segment'.
- */
-void FuncGen::DefineTopLevelGetSizeRefTyped(std::string& testCode,
-                                            const std::string& rawType,
-                                            FeatureSet features) {
-  std::string func = R"(
-    #pragma GCC diagnostic push
-    #pragma GCC diagnostic ignored "-Wunknown-attributes"
-    /* RawType: %1% */
-    void __attribute__((used, retain)) getSize_%2$016x(const OIInternal::__ROOT_TYPE__& t)
-    #pragma GCC diagnostic pop
-    {
-    )";
-  if (features[Feature::JitTiming]) {
-    func += "      const auto startTime = std::chrono::steady_clock::now();\n";
-  }
-  func += R"(
-      pointers.initialize();
-      pointers.add((uintptr_t)&t);
-      auto data = reinterpret_cast<uintptr_t*>(dataBase);
-
-      // TODO: Replace these with types::st::Uint64 once the VarInt decoding
-      // logic is moved out of OIDebugger and into new TreeBuilder.
-      size_t dataSegOffset = 0;
-      data[dataSegOffset++] = oidMagicId;
-      data[dataSegOffset++] = cookieValue;
-      uintptr_t& writtenSize = data[dataSegOffset++];
-      writtenSize = 0;
-      uintptr_t& timeTakenNs = data[dataSegOffset++];
-
-      dataSegOffset *= sizeof(uintptr_t);
-      JLOG("%1% @");
-      JLOGPTR(&t);
-
-      using ContentType = OIInternal::TypeHandler<DataBuffer::DataSegment, OIInternal::__ROOT_TYPE__>::type;
-      using SuffixType = types::st::Pair<
-        DataBuffer::DataSegment,
-        types::st::VarInt<DataBuffer::DataSegment>,
-        types::st::VarInt<DataBuffer::DataSegment>
-      >;
-      using DataBufferType = types::st::Pair<
-        DataBuffer::DataSegment,
-        ContentType,
-        SuffixType
-      >;
-
-      DataBufferType db = DataBuffer::DataSegment(dataSegOffset);
-      SuffixType suffix = db.delegate([&t](auto ret) {
-        return OIInternal::getSizeType<DataBuffer::DataSegment>(t, ret);
-      });
-      types::st::Unit<DataBuffer::DataSegment> end = suffix
-        .write(123456789)
-        .write(123456789);
-
-      dataSegOffset = end.offset();
-      writtenSize = dataSegOffset;
-      dataBase += dataSegOffset;
-    )";
-  if (features[Feature::JitTiming]) {
-    func += R"(
-      timeTakenNs = std::chrono::duration_cast<std::chrono::nanoseconds>(
-        std::chrono::steady_clock::now() - startTime).count();
-      )";
-  }
-  func += R"(
-    }
-    )";
-
-  boost::format fmt =
-      boost::format(func) % rawType % std::hash<std::string>{}(rawType);
-  testCode.append(fmt.str());
-}
-
 /*
  * DefineOutputType
  *
@@ -803,23 +726,6 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
   ContainerInfo oiArray{"OIArray", UNKNOWN_TYPE,
                         "cstdint"};  // TODO: remove the need for a dummy header
 
-  oiArray.codegen.handler = R"(
-template<typename DB, typename T0, long unsigned int N>
-struct TypeHandler<DB, %1%<T0, N>> {
-  using type = types::st::List<DB, typename TypeHandler<DB, T0>::type>;
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, N> &container,
-      typename TypeHandler<DB, %1%<T0,N>>::type returnArg) {
-    auto tail = returnArg.write(N);
-    for (size_t i=0; i<N; i++) {
-      tail = tail.delegate([&container, i](auto ret) {
-          return TypeHandler<DB, T0>::getSizeType(container.vals[i], ret);
-      });
-    }
-    return tail.finish();
-  }
-};
-)";
   oiArray.codegen.traversalFunc = R"(
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
diff --git a/oi/FuncGen.h b/oi/FuncGen.h
index ff21841..042c0bc 100644
--- a/oi/FuncGen.h
+++ b/oi/FuncGen.h
@@ -61,9 +61,6 @@ class FuncGen {
   static void DefineTopLevelGetSizeRef(std::string& testCode,
                                        const std::string& rawType,
                                        FeatureSet features);
-  static void DefineTopLevelGetSizeRefTyped(std::string& testCode,
-                                            const std::string& rawType,
-                                            FeatureSet features);
   static void DefineOutputType(std::string& testCode,
                                const std::string& rawType);
   static void DefineTreeBuilderInstructions(
diff --git a/oi/OICompiler.cpp b/oi/OICompiler.cpp
index d036b33..fff8042 100644
--- a/oi/OICompiler.cpp
+++ b/oi/OICompiler.cpp
@@ -520,9 +520,8 @@ bool OICompiler::compile(const std::string& code,
 
   static const auto syntheticHeaders = std::array<
       std::pair<Feature, std::pair<std::string_view, std::string>>, 7>{{
-      {Feature::TypedDataSegment, {headers::oi_types_st_h, "oi/types/st.h"}},
-      {Feature::TreeBuilderTypeChecking,
-       {headers::oi_types_dy_h, "oi/types/dy.h"}},
+      {Feature::TreeBuilderV2, {headers::oi_types_st_h, "oi/types/st.h"}},
+      {Feature::TreeBuilderV2, {headers::oi_types_dy_h, "oi/types/dy.h"}},
       {Feature::TreeBuilderV2,
        {headers::oi_exporters_inst_h, "oi/exporters/inst.h"}},
       {Feature::TreeBuilderV2,
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index d98e7b8..d929de1 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -181,8 +181,6 @@ int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
 
   std::map<Feature, bool> featuresMap = {
       {Feature::TypeGraph, true},
-      {Feature::TypedDataSegment, true},
-      {Feature::TreeBuilderTypeChecking, true},
       {Feature::TreeBuilderV2, true},
       {Feature::Library, true},
       {Feature::PackStructs, true},
diff --git a/oi/OILibraryImpl.cpp b/oi/OILibraryImpl.cpp
index bade97c..6562cc8 100644
--- a/oi/OILibraryImpl.cpp
+++ b/oi/OILibraryImpl.cpp
@@ -174,8 +174,6 @@ namespace {
 std::map<Feature, bool> convertFeatures(std::unordered_set<oi::Feature> fs) {
   std::map<Feature, bool> out{
       {Feature::TypeGraph, true},
-      {Feature::TypedDataSegment, true},
-      {Feature::TreeBuilderTypeChecking, true},
       {Feature::TreeBuilderV2, true},
       {Feature::Library, true},
       {Feature::PackStructs, true},
diff --git a/types/README.md b/types/README.md
index ebde86c..e2b70fd 100644
--- a/types/README.md
+++ b/types/README.md
@@ -62,14 +62,10 @@ This document describes the format of the container definition files contained i
   will collide if they share the same name.
 
 
-## Changes introduced with Typed Data Segment
-- `decl` and `func` fields are ignored when using `-ftyped-data-segment` and the
-  `handler` field is used instead.
-
 ## Changes introduced with TreeBuilder V2
-- `decl`, `func`, and `handler` fields are ignored when using `-ftree-builder-v2`.
-   The `TypeHandler` is instead constructed from `traversal_func` and `processor`
-   entries.
+- `decl` and `func` fields are ignored when using `-ftree-builder-v2`. The
+  `TypeHandler` is constructed from `traversal_func` field and `processor`
+  entries.
 
 ## Changes introduced with TypeGraph
 - `typeName` and `matcher` fields have been merged into the single field `type_name`.

From 6edb951651874d12c7231503297e2c160ad7bd65 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 3 Nov 2023 06:06:25 -0700
Subject: [PATCH 005/188] collapse TreeBuilderV2 features

Summary:

Currently there are two features between CodeGen v2 (TypeGraph) and TreeBuilder
v2. These are TypedDataSegment and TreeBuilderTypeChecking. Each of these
features currently has a full set of tests run in the CI and each have specific
exclusions.

Collapse these features into TreeBuilder v2. This allows for significantly
simplified testing as any OIL tests run under TreeBuilder v2 and any OID tests
run under TreeBuilder v1.

The reasoning behind this is I no longer intend to partially roll out this
feature. Full TreeBuilder v2 applies different conditions to containers than
the intermediate states, and writing these only to have them never deployed is
a waste of time.

Test Plan:
- it builds
- CI
---
 .circleci/config.yml | 22 -----------
 oi/CodeGen.cpp       | 49 ++++++-----------------
 oi/ContainerInfo.cpp |  6 +--
 oi/ContainerInfo.h   |  1 -
 oi/Features.cpp      | 13 +-----
 oi/Features.h        |  2 -
 oi/FuncGen.cpp       | 94 --------------------------------------------
 oi/FuncGen.h         |  3 --
 oi/OICompiler.cpp    |  5 +--
 oi/OIGenerator.cpp   |  2 -
 oi/OILibraryImpl.cpp |  2 -
 types/README.md      | 10 ++---
 12 files changed, 20 insertions(+), 189 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 286f759..7e805ad 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -21,20 +21,6 @@ workflows:
           oid_test_args: "-ftype-graph"
           tests_regex: "OidIntegration\\..*"
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
-      - test:
-          name: test-typed-data-segment-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-ftyped-data-segment"
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
-      - test:
-          name: test-tree-builder-type-checking-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-ftree-builder-type-checking"
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
       - coverage:
           name: coverage
           requires:
@@ -43,14 +29,6 @@ workflows:
           name: coverage-type-graph
           requires:
             - test-type-graph-gcc
-      - coverage:
-          name: coverage-typed-data-segment
-          requires:
-            - test-typed-data-segment-gcc
-      - coverage:
-          name: coverage-tree-builder-type-checking
-          requires:
-            - test-tree-builder-type-checking-gcc
 
       - build:
           name: build-clang
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index fef0a47..1830624 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -117,17 +117,14 @@ void addIncludes(const TypeGraph& typeGraph,
                  FeatureSet features,
                  std::string& code) {
   std::set<std::string_view> includes{"cstddef"};
-  if (features[Feature::TypedDataSegment]) {
+  if (features[Feature::TreeBuilderV2]) {
+    code += "#define DEFINE_DESCRIBE 1\n";  // added before all includes
+
     includes.emplace("functional");
+    includes.emplace("oi/exporters/inst.h");
+    includes.emplace("oi/types/dy.h");
     includes.emplace("oi/types/st.h");
   }
-  if (features[Feature::TreeBuilderTypeChecking]) {
-    includes.emplace("oi/types/dy.h");
-
-    code += "#define DEFINE_DESCRIBE 1\n";  // added before all includes
-  }
-  if (features[Feature::TreeBuilderV2])
-    includes.emplace("oi/exporters/inst.h");
   if (features[Feature::Library]) {
     includes.emplace("vector");
     includes.emplace("oi/IntrospectionResult.h");
@@ -830,8 +827,7 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) {
   code += "  using type = ";
   genClassStaticType(c, code);
   code += ";\n";
-  if (config_.features[Feature::TreeBuilderV2])
-    genClassTreeBuilderInstructions(c, code);
+  genClassTreeBuilderInstructions(c, code);
   genClassTraversalFunction(c, code);
   code += "};\n";
 }
@@ -846,19 +842,6 @@ void genContainerTypeHandler(FeatureSet features,
   if (!used.insert(&c).second)
     return;
 
-  if (!features[Feature::TreeBuilderV2]) {
-    const auto& handler = c.codegen.handler;
-    if (handler.empty()) {
-      LOG(ERROR) << "`codegen.handler` must be specified for all containers "
-                    "under \"-ftyped-data-segment\", not specified for \"" +
-                        c.typeName + "\"";
-      throw std::runtime_error("missing `codegen.handler`");
-    }
-    auto fmt = boost::format(c.codegen.handler) % c.typeName;
-    code += fmt.str();
-    return;
-  }
-
   code += c.codegen.extra;
 
   // TODO: Move this check into the ContainerInfo parsing once always enabled.
@@ -1216,14 +1199,14 @@ void CodeGen::generate(
   if (!config_.features[Feature::Library]) {
     FuncGen::DeclareExterns(code);
   }
-  if (!config_.features[Feature::TypedDataSegment]) {
+  if (!config_.features[Feature::TreeBuilderV2]) {
     defineMacros(code);
   }
   addIncludes(typeGraph, config_.features, code);
   defineInternalTypes(code);
   FuncGen::DefineJitLog(code, config_.features);
 
-  if (config_.features[Feature::TypedDataSegment]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     if (config_.features[Feature::Library]) {
       FuncGen::DefineBackInserterDataBuffer(code);
     } else {
@@ -1231,10 +1214,8 @@ void CodeGen::generate(
     }
     code += "using namespace oi;\n";
     code += "using namespace oi::detail;\n";
-    if (config_.features[Feature::TreeBuilderV2]) {
-      code += "using oi::exporters::ParsedData;\n";
-      code += "using namespace oi::exporters;\n";
-    }
+    code += "using oi::exporters::ParsedData;\n";
+    code += "using namespace oi::exporters;\n";
     code += "namespace OIInternal {\nnamespace {\n";
     FuncGen::DefineBasicTypeHandlers(code, config_.features);
     code += "} // namespace\n} // namespace OIInternal\n";
@@ -1254,7 +1235,7 @@ void CodeGen::generate(
    * process faster.
    */
   code += "namespace OIInternal {\nnamespace {\n";
-  if (!config_.features[Feature::TypedDataSegment]) {
+  if (!config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineEncodeData(code);
     FuncGen::DefineEncodeDataSize(code);
     FuncGen::DefineStoreData(code);
@@ -1269,7 +1250,7 @@ void CodeGen::generate(
     genExclusiveSizes(typeGraph, code);
   }
 
-  if (config_.features[Feature::TypedDataSegment]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     addStandardTypeHandlers(typeGraph, config_.features, code);
     addTypeHandlers(typeGraph, code);
   } else {
@@ -1286,10 +1267,8 @@ void CodeGen::generate(
   code += "} // namespace\n} // namespace OIInternal\n";
 
   const auto typeName = SymbolService::getTypeName(drgnType);
-  if (config_.features[Feature::Library]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
-  } else if (config_.features[Feature::TypedDataSegment]) {
-    FuncGen::DefineTopLevelGetSizeRefTyped(code, typeName, config_.features);
   } else {
     FuncGen::DefineTopLevelGetSizeRef(code, typeName, config_.features);
   }
@@ -1298,8 +1277,6 @@ void CodeGen::generate(
     FuncGen::DefineTreeBuilderInstructions(code, typeName,
                                            calculateExclusiveSize(rootType),
                                            enumerateTypeNames(rootType));
-  } else if (config_.features[Feature::TreeBuilderTypeChecking]) {
-    FuncGen::DefineOutputType(code, typeName);
   }
 
   if (!linkageName_.empty())
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index a5703be..40ca6b4 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -274,10 +274,6 @@ ContainerInfo::ContainerInfo(const fs::path& path) {
   } else {
     throw ContainerInfoError(path, "`codegen.decl` is a required field");
   }
-  if (std::optional<std::string> str =
-          codegenToml["handler"].value<std::string>()) {
-    codegen.handler = std::move(*str);
-  }
   if (std::optional<std::string> str =
           codegenToml["traversal_func"].value<std::string>()) {
     codegen.traversalFunc = std::move(*str);
@@ -324,5 +320,5 @@ ContainerInfo::ContainerInfo(std::string typeName_,
       ctype(ctype_),
       header(std::move(header_)),
       codegen(Codegen{"// DummyDecl %1%\n", "// DummyFunc %1%\n",
-                      "// DummyHandler %1%\n", "// DummyFunc\n"}) {
+                      "// DummyFunc\n"}) {
 }
diff --git a/oi/ContainerInfo.h b/oi/ContainerInfo.h
index 0ad9da9..376274d 100644
--- a/oi/ContainerInfo.h
+++ b/oi/ContainerInfo.h
@@ -36,7 +36,6 @@ struct ContainerInfo {
   struct Codegen {
     std::string decl;
     std::string func;
-    std::string handler = "";
     std::string traversalFunc = "";
     std::string extra = "";
     std::vector<Processor> processors{};
diff --git a/oi/Features.cpp b/oi/Features.cpp
index baf9344..9d82b17 100644
--- a/oi/Features.cpp
+++ b/oi/Features.cpp
@@ -40,11 +40,6 @@ std::optional<std::string_view> featureHelp(Feature f) {
       return "Use Type Graph for code generation (CodeGen v2).";
     case Feature::PruneTypeGraph:
       return "Prune unreachable nodes from the type graph";
-    case Feature::TypedDataSegment:
-      return "Use Typed Data Segment in generated code.";
-    case Feature::TreeBuilderTypeChecking:
-      return "Use Typed Data Segment to perform runtime Type Checking in "
-             "TreeBuilder.";
     case Feature::Library:
       return std::nullopt;  // Hide in OID help
     case Feature::TreeBuilderV2:
@@ -65,14 +60,8 @@ std::optional<std::string_view> featureHelp(Feature f) {
 
 std::span<const Feature> requirements(Feature f) {
   switch (f) {
-    case Feature::TypedDataSegment:
-      static constexpr std::array tds = {Feature::TypeGraph};
-      return tds;
-    case Feature::TreeBuilderTypeChecking:
-      static constexpr std::array tc = {Feature::TypedDataSegment};
-      return tc;
     case Feature::TreeBuilderV2:
-      static constexpr std::array tb2 = {Feature::TreeBuilderTypeChecking};
+      static constexpr std::array tb2 = {Feature::TypeGraph};
       return tb2;
     case Feature::Library:
       static constexpr std::array lib = {Feature::TreeBuilderV2};
diff --git a/oi/Features.h b/oi/Features.h
index 2e92ba0..1d039b7 100644
--- a/oi/Features.h
+++ b/oi/Features.h
@@ -29,8 +29,6 @@
   X(CaptureThriftIsset, "capture-thrift-isset")            \
   X(TypeGraph, "type-graph")                               \
   X(PruneTypeGraph, "prune-type-graph")                    \
-  X(TypedDataSegment, "typed-data-segment")                \
-  X(TreeBuilderTypeChecking, "tree-builder-type-checking") \
   X(Library, "library")                                    \
   X(TreeBuilderV2, "tree-builder-v2")                      \
   X(GenJitDebug, "gen-jit-debug")                          \
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index dc494f9..d4f0e1e 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -352,83 +352,6 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
   testCode.append(fmt.str());
 }
 
-/*
- * DefineTopLevelGetSizeRefTyped
- *
- * Top level function to run OI on a type utilising static types and enabled
- * with feature '-ftyped-data-segment'.
- */
-void FuncGen::DefineTopLevelGetSizeRefTyped(std::string& testCode,
-                                            const std::string& rawType,
-                                            FeatureSet features) {
-  std::string func = R"(
-    #pragma GCC diagnostic push
-    #pragma GCC diagnostic ignored "-Wunknown-attributes"
-    /* RawType: %1% */
-    void __attribute__((used, retain)) getSize_%2$016x(const OIInternal::__ROOT_TYPE__& t)
-    #pragma GCC diagnostic pop
-    {
-    )";
-  if (features[Feature::JitTiming]) {
-    func += "      const auto startTime = std::chrono::steady_clock::now();\n";
-  }
-  func += R"(
-      pointers.initialize();
-      pointers.add((uintptr_t)&t);
-      auto data = reinterpret_cast<uintptr_t*>(dataBase);
-
-      // TODO: Replace these with types::st::Uint64 once the VarInt decoding
-      // logic is moved out of OIDebugger and into new TreeBuilder.
-      size_t dataSegOffset = 0;
-      data[dataSegOffset++] = oidMagicId;
-      data[dataSegOffset++] = cookieValue;
-      uintptr_t& writtenSize = data[dataSegOffset++];
-      writtenSize = 0;
-      uintptr_t& timeTakenNs = data[dataSegOffset++];
-
-      dataSegOffset *= sizeof(uintptr_t);
-      JLOG("%1% @");
-      JLOGPTR(&t);
-
-      using ContentType = OIInternal::TypeHandler<DataBuffer::DataSegment, OIInternal::__ROOT_TYPE__>::type;
-      using SuffixType = types::st::Pair<
-        DataBuffer::DataSegment,
-        types::st::VarInt<DataBuffer::DataSegment>,
-        types::st::VarInt<DataBuffer::DataSegment>
-      >;
-      using DataBufferType = types::st::Pair<
-        DataBuffer::DataSegment,
-        ContentType,
-        SuffixType
-      >;
-
-      DataBufferType db = DataBuffer::DataSegment(dataSegOffset);
-      SuffixType suffix = db.delegate([&t](auto ret) {
-        return OIInternal::getSizeType<DataBuffer::DataSegment>(t, ret);
-      });
-      types::st::Unit<DataBuffer::DataSegment> end = suffix
-        .write(123456789)
-        .write(123456789);
-
-      dataSegOffset = end.offset();
-      writtenSize = dataSegOffset;
-      dataBase += dataSegOffset;
-    )";
-  if (features[Feature::JitTiming]) {
-    func += R"(
-      timeTakenNs = std::chrono::duration_cast<std::chrono::nanoseconds>(
-        std::chrono::steady_clock::now() - startTime).count();
-      )";
-  }
-  func += R"(
-    }
-    )";
-
-  boost::format fmt =
-      boost::format(func) % rawType % std::hash<std::string>{}(rawType);
-  testCode.append(fmt.str());
-}
-
 /*
  * DefineOutputType
  *
@@ -803,23 +726,6 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
   ContainerInfo oiArray{"OIArray", UNKNOWN_TYPE,
                         "cstdint"};  // TODO: remove the need for a dummy header
 
-  oiArray.codegen.handler = R"(
-template<typename DB, typename T0, long unsigned int N>
-struct TypeHandler<DB, %1%<T0, N>> {
-  using type = types::st::List<DB, typename TypeHandler<DB, T0>::type>;
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, N> &container,
-      typename TypeHandler<DB, %1%<T0,N>>::type returnArg) {
-    auto tail = returnArg.write(N);
-    for (size_t i=0; i<N; i++) {
-      tail = tail.delegate([&container, i](auto ret) {
-          return TypeHandler<DB, T0>::getSizeType(container.vals[i], ret);
-      });
-    }
-    return tail.finish();
-  }
-};
-)";
   oiArray.codegen.traversalFunc = R"(
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
diff --git a/oi/FuncGen.h b/oi/FuncGen.h
index ff21841..042c0bc 100644
--- a/oi/FuncGen.h
+++ b/oi/FuncGen.h
@@ -61,9 +61,6 @@ class FuncGen {
   static void DefineTopLevelGetSizeRef(std::string& testCode,
                                        const std::string& rawType,
                                        FeatureSet features);
-  static void DefineTopLevelGetSizeRefTyped(std::string& testCode,
-                                            const std::string& rawType,
-                                            FeatureSet features);
   static void DefineOutputType(std::string& testCode,
                                const std::string& rawType);
   static void DefineTreeBuilderInstructions(
diff --git a/oi/OICompiler.cpp b/oi/OICompiler.cpp
index d036b33..fff8042 100644
--- a/oi/OICompiler.cpp
+++ b/oi/OICompiler.cpp
@@ -520,9 +520,8 @@ bool OICompiler::compile(const std::string& code,
 
   static const auto syntheticHeaders = std::array<
       std::pair<Feature, std::pair<std::string_view, std::string>>, 7>{{
-      {Feature::TypedDataSegment, {headers::oi_types_st_h, "oi/types/st.h"}},
-      {Feature::TreeBuilderTypeChecking,
-       {headers::oi_types_dy_h, "oi/types/dy.h"}},
+      {Feature::TreeBuilderV2, {headers::oi_types_st_h, "oi/types/st.h"}},
+      {Feature::TreeBuilderV2, {headers::oi_types_dy_h, "oi/types/dy.h"}},
       {Feature::TreeBuilderV2,
        {headers::oi_exporters_inst_h, "oi/exporters/inst.h"}},
       {Feature::TreeBuilderV2,
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index d98e7b8..d929de1 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -181,8 +181,6 @@ int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
 
   std::map<Feature, bool> featuresMap = {
       {Feature::TypeGraph, true},
-      {Feature::TypedDataSegment, true},
-      {Feature::TreeBuilderTypeChecking, true},
       {Feature::TreeBuilderV2, true},
       {Feature::Library, true},
       {Feature::PackStructs, true},
diff --git a/oi/OILibraryImpl.cpp b/oi/OILibraryImpl.cpp
index bade97c..6562cc8 100644
--- a/oi/OILibraryImpl.cpp
+++ b/oi/OILibraryImpl.cpp
@@ -174,8 +174,6 @@ namespace {
 std::map<Feature, bool> convertFeatures(std::unordered_set<oi::Feature> fs) {
   std::map<Feature, bool> out{
       {Feature::TypeGraph, true},
-      {Feature::TypedDataSegment, true},
-      {Feature::TreeBuilderTypeChecking, true},
       {Feature::TreeBuilderV2, true},
       {Feature::Library, true},
       {Feature::PackStructs, true},
diff --git a/types/README.md b/types/README.md
index ebde86c..e2b70fd 100644
--- a/types/README.md
+++ b/types/README.md
@@ -62,14 +62,10 @@ This document describes the format of the container definition files contained i
   will collide if they share the same name.
 
 
-## Changes introduced with Typed Data Segment
-- `decl` and `func` fields are ignored when using `-ftyped-data-segment` and the
-  `handler` field is used instead.
-
 ## Changes introduced with TreeBuilder V2
-- `decl`, `func`, and `handler` fields are ignored when using `-ftree-builder-v2`.
-   The `TypeHandler` is instead constructed from `traversal_func` and `processor`
-   entries.
+- `decl` and `func` fields are ignored when using `-ftree-builder-v2`. The
+  `TypeHandler` is constructed from `traversal_func` field and `processor`
+  entries.
 
 ## Changes introduced with TypeGraph
 - `typeName` and `matcher` fields have been merged into the single field `type_name`.

From fbbf6519e9363e3d22184df17409c90af013e97e Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 3 Nov 2023 06:06:25 -0700
Subject: [PATCH 006/188] collapse TreeBuilderV2 features

Summary:

Currently there are two features between CodeGen v2 (TypeGraph) and TreeBuilder
v2. These are TypedDataSegment and TreeBuilderTypeChecking. Each of these
features currently has a full set of tests run in the CI and each have specific
exclusions.

Collapse these features into TreeBuilder v2. This allows for significantly
simplified testing as any OIL tests run under TreeBuilder v2 and any OID tests
run under TreeBuilder v1.

The reasoning behind this is I no longer intend to partially roll out this
feature. Full TreeBuilder v2 applies different conditions to containers than
the intermediate states, and writing these only to have them never deployed is
a waste of time.

Test Plan:
- it builds
- CI
---
 .circleci/config.yml | 22 -----------
 oi/CodeGen.cpp       | 49 ++++++-----------------
 oi/ContainerInfo.cpp |  6 +--
 oi/ContainerInfo.h   |  1 -
 oi/Features.cpp      | 13 +-----
 oi/Features.h        | 26 ++++++------
 oi/FuncGen.cpp       | 94 --------------------------------------------
 oi/FuncGen.h         |  3 --
 oi/OICompiler.cpp    |  5 +--
 oi/OIGenerator.cpp   |  8 +---
 oi/OILibraryImpl.cpp |  8 +---
 types/README.md      | 10 ++---
 12 files changed, 36 insertions(+), 209 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 286f759..7e805ad 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -21,20 +21,6 @@ workflows:
           oid_test_args: "-ftype-graph"
           tests_regex: "OidIntegration\\..*"
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
-      - test:
-          name: test-typed-data-segment-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-ftyped-data-segment"
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
-      - test:
-          name: test-tree-builder-type-checking-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-ftree-builder-type-checking"
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
       - coverage:
           name: coverage
           requires:
@@ -43,14 +29,6 @@ workflows:
           name: coverage-type-graph
           requires:
             - test-type-graph-gcc
-      - coverage:
-          name: coverage-typed-data-segment
-          requires:
-            - test-typed-data-segment-gcc
-      - coverage:
-          name: coverage-tree-builder-type-checking
-          requires:
-            - test-tree-builder-type-checking-gcc
 
       - build:
           name: build-clang
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index fef0a47..1830624 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -117,17 +117,14 @@ void addIncludes(const TypeGraph& typeGraph,
                  FeatureSet features,
                  std::string& code) {
   std::set<std::string_view> includes{"cstddef"};
-  if (features[Feature::TypedDataSegment]) {
+  if (features[Feature::TreeBuilderV2]) {
+    code += "#define DEFINE_DESCRIBE 1\n";  // added before all includes
+
     includes.emplace("functional");
+    includes.emplace("oi/exporters/inst.h");
+    includes.emplace("oi/types/dy.h");
     includes.emplace("oi/types/st.h");
   }
-  if (features[Feature::TreeBuilderTypeChecking]) {
-    includes.emplace("oi/types/dy.h");
-
-    code += "#define DEFINE_DESCRIBE 1\n";  // added before all includes
-  }
-  if (features[Feature::TreeBuilderV2])
-    includes.emplace("oi/exporters/inst.h");
   if (features[Feature::Library]) {
     includes.emplace("vector");
     includes.emplace("oi/IntrospectionResult.h");
@@ -830,8 +827,7 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) {
   code += "  using type = ";
   genClassStaticType(c, code);
   code += ";\n";
-  if (config_.features[Feature::TreeBuilderV2])
-    genClassTreeBuilderInstructions(c, code);
+  genClassTreeBuilderInstructions(c, code);
   genClassTraversalFunction(c, code);
   code += "};\n";
 }
@@ -846,19 +842,6 @@ void genContainerTypeHandler(FeatureSet features,
   if (!used.insert(&c).second)
     return;
 
-  if (!features[Feature::TreeBuilderV2]) {
-    const auto& handler = c.codegen.handler;
-    if (handler.empty()) {
-      LOG(ERROR) << "`codegen.handler` must be specified for all containers "
-                    "under \"-ftyped-data-segment\", not specified for \"" +
-                        c.typeName + "\"";
-      throw std::runtime_error("missing `codegen.handler`");
-    }
-    auto fmt = boost::format(c.codegen.handler) % c.typeName;
-    code += fmt.str();
-    return;
-  }
-
   code += c.codegen.extra;
 
   // TODO: Move this check into the ContainerInfo parsing once always enabled.
@@ -1216,14 +1199,14 @@ void CodeGen::generate(
   if (!config_.features[Feature::Library]) {
     FuncGen::DeclareExterns(code);
   }
-  if (!config_.features[Feature::TypedDataSegment]) {
+  if (!config_.features[Feature::TreeBuilderV2]) {
     defineMacros(code);
   }
   addIncludes(typeGraph, config_.features, code);
   defineInternalTypes(code);
   FuncGen::DefineJitLog(code, config_.features);
 
-  if (config_.features[Feature::TypedDataSegment]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     if (config_.features[Feature::Library]) {
       FuncGen::DefineBackInserterDataBuffer(code);
     } else {
@@ -1231,10 +1214,8 @@ void CodeGen::generate(
     }
     code += "using namespace oi;\n";
     code += "using namespace oi::detail;\n";
-    if (config_.features[Feature::TreeBuilderV2]) {
-      code += "using oi::exporters::ParsedData;\n";
-      code += "using namespace oi::exporters;\n";
-    }
+    code += "using oi::exporters::ParsedData;\n";
+    code += "using namespace oi::exporters;\n";
     code += "namespace OIInternal {\nnamespace {\n";
     FuncGen::DefineBasicTypeHandlers(code, config_.features);
     code += "} // namespace\n} // namespace OIInternal\n";
@@ -1254,7 +1235,7 @@ void CodeGen::generate(
    * process faster.
    */
   code += "namespace OIInternal {\nnamespace {\n";
-  if (!config_.features[Feature::TypedDataSegment]) {
+  if (!config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineEncodeData(code);
     FuncGen::DefineEncodeDataSize(code);
     FuncGen::DefineStoreData(code);
@@ -1269,7 +1250,7 @@ void CodeGen::generate(
     genExclusiveSizes(typeGraph, code);
   }
 
-  if (config_.features[Feature::TypedDataSegment]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     addStandardTypeHandlers(typeGraph, config_.features, code);
     addTypeHandlers(typeGraph, code);
   } else {
@@ -1286,10 +1267,8 @@ void CodeGen::generate(
   code += "} // namespace\n} // namespace OIInternal\n";
 
   const auto typeName = SymbolService::getTypeName(drgnType);
-  if (config_.features[Feature::Library]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
-  } else if (config_.features[Feature::TypedDataSegment]) {
-    FuncGen::DefineTopLevelGetSizeRefTyped(code, typeName, config_.features);
   } else {
     FuncGen::DefineTopLevelGetSizeRef(code, typeName, config_.features);
   }
@@ -1298,8 +1277,6 @@ void CodeGen::generate(
     FuncGen::DefineTreeBuilderInstructions(code, typeName,
                                            calculateExclusiveSize(rootType),
                                            enumerateTypeNames(rootType));
-  } else if (config_.features[Feature::TreeBuilderTypeChecking]) {
-    FuncGen::DefineOutputType(code, typeName);
   }
 
   if (!linkageName_.empty())
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index a5703be..40ca6b4 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -274,10 +274,6 @@ ContainerInfo::ContainerInfo(const fs::path& path) {
   } else {
     throw ContainerInfoError(path, "`codegen.decl` is a required field");
   }
-  if (std::optional<std::string> str =
-          codegenToml["handler"].value<std::string>()) {
-    codegen.handler = std::move(*str);
-  }
   if (std::optional<std::string> str =
           codegenToml["traversal_func"].value<std::string>()) {
     codegen.traversalFunc = std::move(*str);
@@ -324,5 +320,5 @@ ContainerInfo::ContainerInfo(std::string typeName_,
       ctype(ctype_),
       header(std::move(header_)),
       codegen(Codegen{"// DummyDecl %1%\n", "// DummyFunc %1%\n",
-                      "// DummyHandler %1%\n", "// DummyFunc\n"}) {
+                      "// DummyFunc\n"}) {
 }
diff --git a/oi/ContainerInfo.h b/oi/ContainerInfo.h
index 0ad9da9..376274d 100644
--- a/oi/ContainerInfo.h
+++ b/oi/ContainerInfo.h
@@ -36,7 +36,6 @@ struct ContainerInfo {
   struct Codegen {
     std::string decl;
     std::string func;
-    std::string handler = "";
     std::string traversalFunc = "";
     std::string extra = "";
     std::vector<Processor> processors{};
diff --git a/oi/Features.cpp b/oi/Features.cpp
index baf9344..9d82b17 100644
--- a/oi/Features.cpp
+++ b/oi/Features.cpp
@@ -40,11 +40,6 @@ std::optional<std::string_view> featureHelp(Feature f) {
       return "Use Type Graph for code generation (CodeGen v2).";
     case Feature::PruneTypeGraph:
       return "Prune unreachable nodes from the type graph";
-    case Feature::TypedDataSegment:
-      return "Use Typed Data Segment in generated code.";
-    case Feature::TreeBuilderTypeChecking:
-      return "Use Typed Data Segment to perform runtime Type Checking in "
-             "TreeBuilder.";
     case Feature::Library:
       return std::nullopt;  // Hide in OID help
     case Feature::TreeBuilderV2:
@@ -65,14 +60,8 @@ std::optional<std::string_view> featureHelp(Feature f) {
 
 std::span<const Feature> requirements(Feature f) {
   switch (f) {
-    case Feature::TypedDataSegment:
-      static constexpr std::array tds = {Feature::TypeGraph};
-      return tds;
-    case Feature::TreeBuilderTypeChecking:
-      static constexpr std::array tc = {Feature::TypedDataSegment};
-      return tc;
     case Feature::TreeBuilderV2:
-      static constexpr std::array tb2 = {Feature::TreeBuilderTypeChecking};
+      static constexpr std::array tb2 = {Feature::TypeGraph};
       return tb2;
     case Feature::Library:
       static constexpr std::array lib = {Feature::TreeBuilderV2};
diff --git a/oi/Features.h b/oi/Features.h
index 2e92ba0..959a576 100644
--- a/oi/Features.h
+++ b/oi/Features.h
@@ -22,20 +22,18 @@
 
 #include "oi/EnumBitset.h"
 
-#define OI_FEATURE_LIST                                    \
-  X(ChaseRawPointers, "chase-raw-pointers")                \
-  X(PackStructs, "pack-structs")                           \
-  X(GenPaddingStats, "gen-padding-stats")                  \
-  X(CaptureThriftIsset, "capture-thrift-isset")            \
-  X(TypeGraph, "type-graph")                               \
-  X(PruneTypeGraph, "prune-type-graph")                    \
-  X(TypedDataSegment, "typed-data-segment")                \
-  X(TreeBuilderTypeChecking, "tree-builder-type-checking") \
-  X(Library, "library")                                    \
-  X(TreeBuilderV2, "tree-builder-v2")                      \
-  X(GenJitDebug, "gen-jit-debug")                          \
-  X(JitLogging, "jit-logging")                             \
-  X(JitTiming, "jit-timing")                               \
+#define OI_FEATURE_LIST                         \
+  X(ChaseRawPointers, "chase-raw-pointers")     \
+  X(PackStructs, "pack-structs")                \
+  X(GenPaddingStats, "gen-padding-stats")       \
+  X(CaptureThriftIsset, "capture-thrift-isset") \
+  X(TypeGraph, "type-graph")                    \
+  X(PruneTypeGraph, "prune-type-graph")         \
+  X(Library, "library")                         \
+  X(TreeBuilderV2, "tree-builder-v2")           \
+  X(GenJitDebug, "gen-jit-debug")               \
+  X(JitLogging, "jit-logging")                  \
+  X(JitTiming, "jit-timing")                    \
   X(PolymorphicInheritance, "polymorphic-inheritance")
 
 namespace oi::detail {
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index dc494f9..d4f0e1e 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -352,83 +352,6 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
   testCode.append(fmt.str());
 }
 
-/*
- * DefineTopLevelGetSizeRefTyped
- *
- * Top level function to run OI on a type utilising static types and enabled
- * with feature '-ftyped-data-segment'.
- */
-void FuncGen::DefineTopLevelGetSizeRefTyped(std::string& testCode,
-                                            const std::string& rawType,
-                                            FeatureSet features) {
-  std::string func = R"(
-    #pragma GCC diagnostic push
-    #pragma GCC diagnostic ignored "-Wunknown-attributes"
-    /* RawType: %1% */
-    void __attribute__((used, retain)) getSize_%2$016x(const OIInternal::__ROOT_TYPE__& t)
-    #pragma GCC diagnostic pop
-    {
-    )";
-  if (features[Feature::JitTiming]) {
-    func += "      const auto startTime = std::chrono::steady_clock::now();\n";
-  }
-  func += R"(
-      pointers.initialize();
-      pointers.add((uintptr_t)&t);
-      auto data = reinterpret_cast<uintptr_t*>(dataBase);
-
-      // TODO: Replace these with types::st::Uint64 once the VarInt decoding
-      // logic is moved out of OIDebugger and into new TreeBuilder.
-      size_t dataSegOffset = 0;
-      data[dataSegOffset++] = oidMagicId;
-      data[dataSegOffset++] = cookieValue;
-      uintptr_t& writtenSize = data[dataSegOffset++];
-      writtenSize = 0;
-      uintptr_t& timeTakenNs = data[dataSegOffset++];
-
-      dataSegOffset *= sizeof(uintptr_t);
-      JLOG("%1% @");
-      JLOGPTR(&t);
-
-      using ContentType = OIInternal::TypeHandler<DataBuffer::DataSegment, OIInternal::__ROOT_TYPE__>::type;
-      using SuffixType = types::st::Pair<
-        DataBuffer::DataSegment,
-        types::st::VarInt<DataBuffer::DataSegment>,
-        types::st::VarInt<DataBuffer::DataSegment>
-      >;
-      using DataBufferType = types::st::Pair<
-        DataBuffer::DataSegment,
-        ContentType,
-        SuffixType
-      >;
-
-      DataBufferType db = DataBuffer::DataSegment(dataSegOffset);
-      SuffixType suffix = db.delegate([&t](auto ret) {
-        return OIInternal::getSizeType<DataBuffer::DataSegment>(t, ret);
-      });
-      types::st::Unit<DataBuffer::DataSegment> end = suffix
-        .write(123456789)
-        .write(123456789);
-
-      dataSegOffset = end.offset();
-      writtenSize = dataSegOffset;
-      dataBase += dataSegOffset;
-    )";
-  if (features[Feature::JitTiming]) {
-    func += R"(
-      timeTakenNs = std::chrono::duration_cast<std::chrono::nanoseconds>(
-        std::chrono::steady_clock::now() - startTime).count();
-      )";
-  }
-  func += R"(
-    }
-    )";
-
-  boost::format fmt =
-      boost::format(func) % rawType % std::hash<std::string>{}(rawType);
-  testCode.append(fmt.str());
-}
-
 /*
  * DefineOutputType
  *
@@ -803,23 +726,6 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
   ContainerInfo oiArray{"OIArray", UNKNOWN_TYPE,
                         "cstdint"};  // TODO: remove the need for a dummy header
 
-  oiArray.codegen.handler = R"(
-template<typename DB, typename T0, long unsigned int N>
-struct TypeHandler<DB, %1%<T0, N>> {
-  using type = types::st::List<DB, typename TypeHandler<DB, T0>::type>;
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, N> &container,
-      typename TypeHandler<DB, %1%<T0,N>>::type returnArg) {
-    auto tail = returnArg.write(N);
-    for (size_t i=0; i<N; i++) {
-      tail = tail.delegate([&container, i](auto ret) {
-          return TypeHandler<DB, T0>::getSizeType(container.vals[i], ret);
-      });
-    }
-    return tail.finish();
-  }
-};
-)";
   oiArray.codegen.traversalFunc = R"(
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
diff --git a/oi/FuncGen.h b/oi/FuncGen.h
index ff21841..042c0bc 100644
--- a/oi/FuncGen.h
+++ b/oi/FuncGen.h
@@ -61,9 +61,6 @@ class FuncGen {
   static void DefineTopLevelGetSizeRef(std::string& testCode,
                                        const std::string& rawType,
                                        FeatureSet features);
-  static void DefineTopLevelGetSizeRefTyped(std::string& testCode,
-                                            const std::string& rawType,
-                                            FeatureSet features);
   static void DefineOutputType(std::string& testCode,
                                const std::string& rawType);
   static void DefineTreeBuilderInstructions(
diff --git a/oi/OICompiler.cpp b/oi/OICompiler.cpp
index d036b33..fff8042 100644
--- a/oi/OICompiler.cpp
+++ b/oi/OICompiler.cpp
@@ -520,9 +520,8 @@ bool OICompiler::compile(const std::string& code,
 
   static const auto syntheticHeaders = std::array<
       std::pair<Feature, std::pair<std::string_view, std::string>>, 7>{{
-      {Feature::TypedDataSegment, {headers::oi_types_st_h, "oi/types/st.h"}},
-      {Feature::TreeBuilderTypeChecking,
-       {headers::oi_types_dy_h, "oi/types/dy.h"}},
+      {Feature::TreeBuilderV2, {headers::oi_types_st_h, "oi/types/st.h"}},
+      {Feature::TreeBuilderV2, {headers::oi_types_dy_h, "oi/types/dy.h"}},
       {Feature::TreeBuilderV2,
        {headers::oi_exporters_inst_h, "oi/exporters/inst.h"}},
       {Feature::TreeBuilderV2,
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index d98e7b8..f91fe16 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -180,12 +180,8 @@ int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
   auto oilTypes = findOilTypesAndNames(prog);
 
   std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TypedDataSegment, true},
-      {Feature::TreeBuilderTypeChecking, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
+      {Feature::TypeGraph, true},      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},        {Feature::PackStructs, true},
       {Feature::PruneTypeGraph, true},
   };
 
diff --git a/oi/OILibraryImpl.cpp b/oi/OILibraryImpl.cpp
index bade97c..9254fe0 100644
--- a/oi/OILibraryImpl.cpp
+++ b/oi/OILibraryImpl.cpp
@@ -173,12 +173,8 @@ std::pair<void*, const exporters::inst::Inst&> OILibraryImpl::compileCode() {
 namespace {
 std::map<Feature, bool> convertFeatures(std::unordered_set<oi::Feature> fs) {
   std::map<Feature, bool> out{
-      {Feature::TypeGraph, true},
-      {Feature::TypedDataSegment, true},
-      {Feature::TreeBuilderTypeChecking, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
+      {Feature::TypeGraph, true},      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},        {Feature::PackStructs, true},
       {Feature::PruneTypeGraph, true},
   };
 
diff --git a/types/README.md b/types/README.md
index ebde86c..e2b70fd 100644
--- a/types/README.md
+++ b/types/README.md
@@ -62,14 +62,10 @@ This document describes the format of the container definition files contained i
   will collide if they share the same name.
 
 
-## Changes introduced with Typed Data Segment
-- `decl` and `func` fields are ignored when using `-ftyped-data-segment` and the
-  `handler` field is used instead.
-
 ## Changes introduced with TreeBuilder V2
-- `decl`, `func`, and `handler` fields are ignored when using `-ftree-builder-v2`.
-   The `TypeHandler` is instead constructed from `traversal_func` and `processor`
-   entries.
+- `decl` and `func` fields are ignored when using `-ftree-builder-v2`. The
+  `TypeHandler` is constructed from `traversal_func` field and `processor`
+  entries.
 
 ## Changes introduced with TypeGraph
 - `typeName` and `matcher` fields have been merged into the single field `type_name`.

From 42f87abca49072d67904855b5d559be2c192f6e9 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 3 Nov 2023 06:06:25 -0700
Subject: [PATCH 007/188] collapse TreeBuilderV2 features

Summary:

Currently there are two features between CodeGen v2 (TypeGraph) and TreeBuilder
v2. These are TypedDataSegment and TreeBuilderTypeChecking. Each of these
features currently has a full set of tests run in the CI and each have specific
exclusions.

Collapse these features into TreeBuilder v2. This allows for significantly
simplified testing as any OIL tests run under TreeBuilder v2 and any OID tests
run under TreeBuilder v1.

The reasoning behind this is I no longer intend to partially roll out this
feature. Full TreeBuilder v2 applies different conditions to containers than
the intermediate states, and writing these only to have them never deployed is
a waste of time.

Test Plan:
- it builds
- CI
---
 .circleci/config.yml | 22 -----------
 oi/CodeGen.cpp       | 56 ++++++++------------------
 oi/ContainerInfo.cpp |  6 +--
 oi/ContainerInfo.h   |  1 -
 oi/Features.cpp      | 13 +-----
 oi/Features.h        | 26 ++++++------
 oi/FuncGen.cpp       | 94 --------------------------------------------
 oi/FuncGen.h         |  3 --
 oi/OICompiler.cpp    |  5 +--
 oi/OIGenerator.cpp   |  8 +---
 oi/OILibraryImpl.cpp |  8 +---
 types/README.md      | 10 ++---
 12 files changed, 39 insertions(+), 213 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 286f759..7e805ad 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -21,20 +21,6 @@ workflows:
           oid_test_args: "-ftype-graph"
           tests_regex: "OidIntegration\\..*"
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
-      - test:
-          name: test-typed-data-segment-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-ftyped-data-segment"
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
-      - test:
-          name: test-tree-builder-type-checking-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-ftree-builder-type-checking"
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
       - coverage:
           name: coverage
           requires:
@@ -43,14 +29,6 @@ workflows:
           name: coverage-type-graph
           requires:
             - test-type-graph-gcc
-      - coverage:
-          name: coverage-typed-data-segment
-          requires:
-            - test-typed-data-segment-gcc
-      - coverage:
-          name: coverage-tree-builder-type-checking
-          requires:
-            - test-tree-builder-type-checking-gcc
 
       - build:
           name: build-clang
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index fef0a47..7ddb173 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -117,17 +117,14 @@ void addIncludes(const TypeGraph& typeGraph,
                  FeatureSet features,
                  std::string& code) {
   std::set<std::string_view> includes{"cstddef"};
-  if (features[Feature::TypedDataSegment]) {
+  if (features[Feature::TreeBuilderV2]) {
+    code += "#define DEFINE_DESCRIBE 1\n";  // added before all includes
+
     includes.emplace("functional");
+    includes.emplace("oi/exporters/inst.h");
+    includes.emplace("oi/types/dy.h");
     includes.emplace("oi/types/st.h");
   }
-  if (features[Feature::TreeBuilderTypeChecking]) {
-    includes.emplace("oi/types/dy.h");
-
-    code += "#define DEFINE_DESCRIBE 1\n";  // added before all includes
-  }
-  if (features[Feature::TreeBuilderV2])
-    includes.emplace("oi/exporters/inst.h");
   if (features[Feature::Library]) {
     includes.emplace("vector");
     includes.emplace("oi/IntrospectionResult.h");
@@ -830,35 +827,20 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) {
   code += "  using type = ";
   genClassStaticType(c, code);
   code += ";\n";
-  if (config_.features[Feature::TreeBuilderV2])
-    genClassTreeBuilderInstructions(c, code);
+  genClassTreeBuilderInstructions(c, code);
   genClassTraversalFunction(c, code);
   code += "};\n";
 }
 
 namespace {
 
-void genContainerTypeHandler(FeatureSet features,
-                             std::unordered_set<const ContainerInfo*>& used,
+void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
                              const ContainerInfo& c,
                              std::span<const TemplateParam> templateParams,
                              std::string& code) {
   if (!used.insert(&c).second)
     return;
 
-  if (!features[Feature::TreeBuilderV2]) {
-    const auto& handler = c.codegen.handler;
-    if (handler.empty()) {
-      LOG(ERROR) << "`codegen.handler` must be specified for all containers "
-                    "under \"-ftyped-data-segment\", not specified for \"" +
-                        c.typeName + "\"";
-      throw std::runtime_error("missing `codegen.handler`");
-    }
-    auto fmt = boost::format(c.codegen.handler) % c.typeName;
-    code += fmt.str();
-    return;
-  }
-
   code += c.codegen.extra;
 
   // TODO: Move this check into the ContainerInfo parsing once always enabled.
@@ -1085,8 +1067,8 @@ constexpr inst::Field make_field(std::string_view name) {
       TemplateParam{typeGraph.makeType<Primitive>(Primitive::Kind::UInt64),
                     "0"},
   };
-  genContainerTypeHandler(features, used, FuncGen::GetOiArrayContainerInfo(),
-                          arrayParams, code);
+  genContainerTypeHandler(used, FuncGen::GetOiArrayContainerInfo(), arrayParams,
+                          code);
 }
 
 }  // namespace
@@ -1216,14 +1198,14 @@ void CodeGen::generate(
   if (!config_.features[Feature::Library]) {
     FuncGen::DeclareExterns(code);
   }
-  if (!config_.features[Feature::TypedDataSegment]) {
+  if (!config_.features[Feature::TreeBuilderV2]) {
     defineMacros(code);
   }
   addIncludes(typeGraph, config_.features, code);
   defineInternalTypes(code);
   FuncGen::DefineJitLog(code, config_.features);
 
-  if (config_.features[Feature::TypedDataSegment]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     if (config_.features[Feature::Library]) {
       FuncGen::DefineBackInserterDataBuffer(code);
     } else {
@@ -1231,10 +1213,8 @@ void CodeGen::generate(
     }
     code += "using namespace oi;\n";
     code += "using namespace oi::detail;\n";
-    if (config_.features[Feature::TreeBuilderV2]) {
-      code += "using oi::exporters::ParsedData;\n";
-      code += "using namespace oi::exporters;\n";
-    }
+    code += "using oi::exporters::ParsedData;\n";
+    code += "using namespace oi::exporters;\n";
     code += "namespace OIInternal {\nnamespace {\n";
     FuncGen::DefineBasicTypeHandlers(code, config_.features);
     code += "} // namespace\n} // namespace OIInternal\n";
@@ -1254,7 +1234,7 @@ void CodeGen::generate(
    * process faster.
    */
   code += "namespace OIInternal {\nnamespace {\n";
-  if (!config_.features[Feature::TypedDataSegment]) {
+  if (!config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineEncodeData(code);
     FuncGen::DefineEncodeDataSize(code);
     FuncGen::DefineStoreData(code);
@@ -1269,7 +1249,7 @@ void CodeGen::generate(
     genExclusiveSizes(typeGraph, code);
   }
 
-  if (config_.features[Feature::TypedDataSegment]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     addStandardTypeHandlers(typeGraph, config_.features, code);
     addTypeHandlers(typeGraph, code);
   } else {
@@ -1286,10 +1266,8 @@ void CodeGen::generate(
   code += "} // namespace\n} // namespace OIInternal\n";
 
   const auto typeName = SymbolService::getTypeName(drgnType);
-  if (config_.features[Feature::Library]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
-  } else if (config_.features[Feature::TypedDataSegment]) {
-    FuncGen::DefineTopLevelGetSizeRefTyped(code, typeName, config_.features);
   } else {
     FuncGen::DefineTopLevelGetSizeRef(code, typeName, config_.features);
   }
@@ -1298,8 +1276,6 @@ void CodeGen::generate(
     FuncGen::DefineTreeBuilderInstructions(code, typeName,
                                            calculateExclusiveSize(rootType),
                                            enumerateTypeNames(rootType));
-  } else if (config_.features[Feature::TreeBuilderTypeChecking]) {
-    FuncGen::DefineOutputType(code, typeName);
   }
 
   if (!linkageName_.empty())
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index a5703be..40ca6b4 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -274,10 +274,6 @@ ContainerInfo::ContainerInfo(const fs::path& path) {
   } else {
     throw ContainerInfoError(path, "`codegen.decl` is a required field");
   }
-  if (std::optional<std::string> str =
-          codegenToml["handler"].value<std::string>()) {
-    codegen.handler = std::move(*str);
-  }
   if (std::optional<std::string> str =
           codegenToml["traversal_func"].value<std::string>()) {
     codegen.traversalFunc = std::move(*str);
@@ -324,5 +320,5 @@ ContainerInfo::ContainerInfo(std::string typeName_,
       ctype(ctype_),
       header(std::move(header_)),
       codegen(Codegen{"// DummyDecl %1%\n", "// DummyFunc %1%\n",
-                      "// DummyHandler %1%\n", "// DummyFunc\n"}) {
+                      "// DummyFunc\n"}) {
 }
diff --git a/oi/ContainerInfo.h b/oi/ContainerInfo.h
index 0ad9da9..376274d 100644
--- a/oi/ContainerInfo.h
+++ b/oi/ContainerInfo.h
@@ -36,7 +36,6 @@ struct ContainerInfo {
   struct Codegen {
     std::string decl;
     std::string func;
-    std::string handler = "";
     std::string traversalFunc = "";
     std::string extra = "";
     std::vector<Processor> processors{};
diff --git a/oi/Features.cpp b/oi/Features.cpp
index baf9344..9d82b17 100644
--- a/oi/Features.cpp
+++ b/oi/Features.cpp
@@ -40,11 +40,6 @@ std::optional<std::string_view> featureHelp(Feature f) {
       return "Use Type Graph for code generation (CodeGen v2).";
     case Feature::PruneTypeGraph:
       return "Prune unreachable nodes from the type graph";
-    case Feature::TypedDataSegment:
-      return "Use Typed Data Segment in generated code.";
-    case Feature::TreeBuilderTypeChecking:
-      return "Use Typed Data Segment to perform runtime Type Checking in "
-             "TreeBuilder.";
     case Feature::Library:
       return std::nullopt;  // Hide in OID help
     case Feature::TreeBuilderV2:
@@ -65,14 +60,8 @@ std::optional<std::string_view> featureHelp(Feature f) {
 
 std::span<const Feature> requirements(Feature f) {
   switch (f) {
-    case Feature::TypedDataSegment:
-      static constexpr std::array tds = {Feature::TypeGraph};
-      return tds;
-    case Feature::TreeBuilderTypeChecking:
-      static constexpr std::array tc = {Feature::TypedDataSegment};
-      return tc;
     case Feature::TreeBuilderV2:
-      static constexpr std::array tb2 = {Feature::TreeBuilderTypeChecking};
+      static constexpr std::array tb2 = {Feature::TypeGraph};
       return tb2;
     case Feature::Library:
       static constexpr std::array lib = {Feature::TreeBuilderV2};
diff --git a/oi/Features.h b/oi/Features.h
index 2e92ba0..959a576 100644
--- a/oi/Features.h
+++ b/oi/Features.h
@@ -22,20 +22,18 @@
 
 #include "oi/EnumBitset.h"
 
-#define OI_FEATURE_LIST                                    \
-  X(ChaseRawPointers, "chase-raw-pointers")                \
-  X(PackStructs, "pack-structs")                           \
-  X(GenPaddingStats, "gen-padding-stats")                  \
-  X(CaptureThriftIsset, "capture-thrift-isset")            \
-  X(TypeGraph, "type-graph")                               \
-  X(PruneTypeGraph, "prune-type-graph")                    \
-  X(TypedDataSegment, "typed-data-segment")                \
-  X(TreeBuilderTypeChecking, "tree-builder-type-checking") \
-  X(Library, "library")                                    \
-  X(TreeBuilderV2, "tree-builder-v2")                      \
-  X(GenJitDebug, "gen-jit-debug")                          \
-  X(JitLogging, "jit-logging")                             \
-  X(JitTiming, "jit-timing")                               \
+#define OI_FEATURE_LIST                         \
+  X(ChaseRawPointers, "chase-raw-pointers")     \
+  X(PackStructs, "pack-structs")                \
+  X(GenPaddingStats, "gen-padding-stats")       \
+  X(CaptureThriftIsset, "capture-thrift-isset") \
+  X(TypeGraph, "type-graph")                    \
+  X(PruneTypeGraph, "prune-type-graph")         \
+  X(Library, "library")                         \
+  X(TreeBuilderV2, "tree-builder-v2")           \
+  X(GenJitDebug, "gen-jit-debug")               \
+  X(JitLogging, "jit-logging")                  \
+  X(JitTiming, "jit-timing")                    \
   X(PolymorphicInheritance, "polymorphic-inheritance")
 
 namespace oi::detail {
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index dc494f9..d4f0e1e 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -352,83 +352,6 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
   testCode.append(fmt.str());
 }
 
-/*
- * DefineTopLevelGetSizeRefTyped
- *
- * Top level function to run OI on a type utilising static types and enabled
- * with feature '-ftyped-data-segment'.
- */
-void FuncGen::DefineTopLevelGetSizeRefTyped(std::string& testCode,
-                                            const std::string& rawType,
-                                            FeatureSet features) {
-  std::string func = R"(
-    #pragma GCC diagnostic push
-    #pragma GCC diagnostic ignored "-Wunknown-attributes"
-    /* RawType: %1% */
-    void __attribute__((used, retain)) getSize_%2$016x(const OIInternal::__ROOT_TYPE__& t)
-    #pragma GCC diagnostic pop
-    {
-    )";
-  if (features[Feature::JitTiming]) {
-    func += "      const auto startTime = std::chrono::steady_clock::now();\n";
-  }
-  func += R"(
-      pointers.initialize();
-      pointers.add((uintptr_t)&t);
-      auto data = reinterpret_cast<uintptr_t*>(dataBase);
-
-      // TODO: Replace these with types::st::Uint64 once the VarInt decoding
-      // logic is moved out of OIDebugger and into new TreeBuilder.
-      size_t dataSegOffset = 0;
-      data[dataSegOffset++] = oidMagicId;
-      data[dataSegOffset++] = cookieValue;
-      uintptr_t& writtenSize = data[dataSegOffset++];
-      writtenSize = 0;
-      uintptr_t& timeTakenNs = data[dataSegOffset++];
-
-      dataSegOffset *= sizeof(uintptr_t);
-      JLOG("%1% @");
-      JLOGPTR(&t);
-
-      using ContentType = OIInternal::TypeHandler<DataBuffer::DataSegment, OIInternal::__ROOT_TYPE__>::type;
-      using SuffixType = types::st::Pair<
-        DataBuffer::DataSegment,
-        types::st::VarInt<DataBuffer::DataSegment>,
-        types::st::VarInt<DataBuffer::DataSegment>
-      >;
-      using DataBufferType = types::st::Pair<
-        DataBuffer::DataSegment,
-        ContentType,
-        SuffixType
-      >;
-
-      DataBufferType db = DataBuffer::DataSegment(dataSegOffset);
-      SuffixType suffix = db.delegate([&t](auto ret) {
-        return OIInternal::getSizeType<DataBuffer::DataSegment>(t, ret);
-      });
-      types::st::Unit<DataBuffer::DataSegment> end = suffix
-        .write(123456789)
-        .write(123456789);
-
-      dataSegOffset = end.offset();
-      writtenSize = dataSegOffset;
-      dataBase += dataSegOffset;
-    )";
-  if (features[Feature::JitTiming]) {
-    func += R"(
-      timeTakenNs = std::chrono::duration_cast<std::chrono::nanoseconds>(
-        std::chrono::steady_clock::now() - startTime).count();
-      )";
-  }
-  func += R"(
-    }
-    )";
-
-  boost::format fmt =
-      boost::format(func) % rawType % std::hash<std::string>{}(rawType);
-  testCode.append(fmt.str());
-}
-
 /*
  * DefineOutputType
  *
@@ -803,23 +726,6 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
   ContainerInfo oiArray{"OIArray", UNKNOWN_TYPE,
                         "cstdint"};  // TODO: remove the need for a dummy header
 
-  oiArray.codegen.handler = R"(
-template<typename DB, typename T0, long unsigned int N>
-struct TypeHandler<DB, %1%<T0, N>> {
-  using type = types::st::List<DB, typename TypeHandler<DB, T0>::type>;
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, N> &container,
-      typename TypeHandler<DB, %1%<T0,N>>::type returnArg) {
-    auto tail = returnArg.write(N);
-    for (size_t i=0; i<N; i++) {
-      tail = tail.delegate([&container, i](auto ret) {
-          return TypeHandler<DB, T0>::getSizeType(container.vals[i], ret);
-      });
-    }
-    return tail.finish();
-  }
-};
-)";
   oiArray.codegen.traversalFunc = R"(
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
diff --git a/oi/FuncGen.h b/oi/FuncGen.h
index ff21841..042c0bc 100644
--- a/oi/FuncGen.h
+++ b/oi/FuncGen.h
@@ -61,9 +61,6 @@ class FuncGen {
   static void DefineTopLevelGetSizeRef(std::string& testCode,
                                        const std::string& rawType,
                                        FeatureSet features);
-  static void DefineTopLevelGetSizeRefTyped(std::string& testCode,
-                                            const std::string& rawType,
-                                            FeatureSet features);
   static void DefineOutputType(std::string& testCode,
                                const std::string& rawType);
   static void DefineTreeBuilderInstructions(
diff --git a/oi/OICompiler.cpp b/oi/OICompiler.cpp
index d036b33..fff8042 100644
--- a/oi/OICompiler.cpp
+++ b/oi/OICompiler.cpp
@@ -520,9 +520,8 @@ bool OICompiler::compile(const std::string& code,
 
   static const auto syntheticHeaders = std::array<
       std::pair<Feature, std::pair<std::string_view, std::string>>, 7>{{
-      {Feature::TypedDataSegment, {headers::oi_types_st_h, "oi/types/st.h"}},
-      {Feature::TreeBuilderTypeChecking,
-       {headers::oi_types_dy_h, "oi/types/dy.h"}},
+      {Feature::TreeBuilderV2, {headers::oi_types_st_h, "oi/types/st.h"}},
+      {Feature::TreeBuilderV2, {headers::oi_types_dy_h, "oi/types/dy.h"}},
       {Feature::TreeBuilderV2,
        {headers::oi_exporters_inst_h, "oi/exporters/inst.h"}},
       {Feature::TreeBuilderV2,
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index d98e7b8..f91fe16 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -180,12 +180,8 @@ int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
   auto oilTypes = findOilTypesAndNames(prog);
 
   std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TypedDataSegment, true},
-      {Feature::TreeBuilderTypeChecking, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
+      {Feature::TypeGraph, true},      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},        {Feature::PackStructs, true},
       {Feature::PruneTypeGraph, true},
   };
 
diff --git a/oi/OILibraryImpl.cpp b/oi/OILibraryImpl.cpp
index bade97c..9254fe0 100644
--- a/oi/OILibraryImpl.cpp
+++ b/oi/OILibraryImpl.cpp
@@ -173,12 +173,8 @@ std::pair<void*, const exporters::inst::Inst&> OILibraryImpl::compileCode() {
 namespace {
 std::map<Feature, bool> convertFeatures(std::unordered_set<oi::Feature> fs) {
   std::map<Feature, bool> out{
-      {Feature::TypeGraph, true},
-      {Feature::TypedDataSegment, true},
-      {Feature::TreeBuilderTypeChecking, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
+      {Feature::TypeGraph, true},      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},        {Feature::PackStructs, true},
       {Feature::PruneTypeGraph, true},
   };
 
diff --git a/types/README.md b/types/README.md
index ebde86c..e2b70fd 100644
--- a/types/README.md
+++ b/types/README.md
@@ -62,14 +62,10 @@ This document describes the format of the container definition files contained i
   will collide if they share the same name.
 
 
-## Changes introduced with Typed Data Segment
-- `decl` and `func` fields are ignored when using `-ftyped-data-segment` and the
-  `handler` field is used instead.
-
 ## Changes introduced with TreeBuilder V2
-- `decl`, `func`, and `handler` fields are ignored when using `-ftree-builder-v2`.
-   The `TypeHandler` is instead constructed from `traversal_func` and `processor`
-   entries.
+- `decl` and `func` fields are ignored when using `-ftree-builder-v2`. The
+  `TypeHandler` is constructed from `traversal_func` field and `processor`
+  entries.
 
 ## Changes introduced with TypeGraph
 - `typeName` and `matcher` fields have been merged into the single field `type_name`.

From d9af8baad1c086e60f53fc1df0d3d7b13508088b Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 3 Nov 2023 06:06:25 -0700
Subject: [PATCH 008/188] collapse TreeBuilderV2 features

Summary:

Currently there are two features between CodeGen v2 (TypeGraph) and TreeBuilder
v2. These are TypedDataSegment and TreeBuilderTypeChecking. Each of these
features currently has a full set of tests run in the CI and each have specific
exclusions.

Collapse these features into TreeBuilder v2. This allows for significantly
simplified testing as any OIL tests run under TreeBuilder v2 and any OID tests
run under TreeBuilder v1.

The reasoning behind this is I no longer intend to partially roll out this
feature. Full TreeBuilder v2 applies different conditions to containers than
the intermediate states, and writing these only to have them never deployed is
a waste of time.

Test Plan:
- it builds
- CI
---
 .circleci/config.yml | 22 -----------
 oi/CodeGen.cpp       | 63 +++++++++--------------------
 oi/ContainerInfo.cpp |  6 +--
 oi/ContainerInfo.h   |  1 -
 oi/Features.cpp      | 13 +-----
 oi/Features.h        | 26 ++++++------
 oi/FuncGen.cpp       | 94 --------------------------------------------
 oi/FuncGen.h         |  3 --
 oi/OICompiler.cpp    |  5 +--
 oi/OIGenerator.cpp   |  8 +---
 oi/OILibraryImpl.cpp |  8 +---
 types/README.md      | 10 ++---
 12 files changed, 42 insertions(+), 217 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 286f759..7e805ad 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -21,20 +21,6 @@ workflows:
           oid_test_args: "-ftype-graph"
           tests_regex: "OidIntegration\\..*"
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
-      - test:
-          name: test-typed-data-segment-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-ftyped-data-segment"
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
-      - test:
-          name: test-tree-builder-type-checking-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-ftree-builder-type-checking"
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
       - coverage:
           name: coverage
           requires:
@@ -43,14 +29,6 @@ workflows:
           name: coverage-type-graph
           requires:
             - test-type-graph-gcc
-      - coverage:
-          name: coverage-typed-data-segment
-          requires:
-            - test-typed-data-segment-gcc
-      - coverage:
-          name: coverage-tree-builder-type-checking
-          requires:
-            - test-tree-builder-type-checking-gcc
 
       - build:
           name: build-clang
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index fef0a47..0aeb255 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -117,17 +117,14 @@ void addIncludes(const TypeGraph& typeGraph,
                  FeatureSet features,
                  std::string& code) {
   std::set<std::string_view> includes{"cstddef"};
-  if (features[Feature::TypedDataSegment]) {
+  if (features[Feature::TreeBuilderV2]) {
+    code += "#define DEFINE_DESCRIBE 1\n";  // added before all includes
+
     includes.emplace("functional");
+    includes.emplace("oi/exporters/inst.h");
+    includes.emplace("oi/types/dy.h");
     includes.emplace("oi/types/st.h");
   }
-  if (features[Feature::TreeBuilderTypeChecking]) {
-    includes.emplace("oi/types/dy.h");
-
-    code += "#define DEFINE_DESCRIBE 1\n";  // added before all includes
-  }
-  if (features[Feature::TreeBuilderV2])
-    includes.emplace("oi/exporters/inst.h");
   if (features[Feature::Library]) {
     includes.emplace("vector");
     includes.emplace("oi/IntrospectionResult.h");
@@ -830,35 +827,20 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) {
   code += "  using type = ";
   genClassStaticType(c, code);
   code += ";\n";
-  if (config_.features[Feature::TreeBuilderV2])
-    genClassTreeBuilderInstructions(c, code);
+  genClassTreeBuilderInstructions(c, code);
   genClassTraversalFunction(c, code);
   code += "};\n";
 }
 
 namespace {
 
-void genContainerTypeHandler(FeatureSet features,
-                             std::unordered_set<const ContainerInfo*>& used,
+void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
                              const ContainerInfo& c,
                              std::span<const TemplateParam> templateParams,
                              std::string& code) {
   if (!used.insert(&c).second)
     return;
 
-  if (!features[Feature::TreeBuilderV2]) {
-    const auto& handler = c.codegen.handler;
-    if (handler.empty()) {
-      LOG(ERROR) << "`codegen.handler` must be specified for all containers "
-                    "under \"-ftyped-data-segment\", not specified for \"" +
-                        c.typeName + "\"";
-      throw std::runtime_error("missing `codegen.handler`");
-    }
-    auto fmt = boost::format(c.codegen.handler) % c.typeName;
-    code += fmt.str();
-    return;
-  }
-
   code += c.codegen.extra;
 
   // TODO: Move this check into the ContainerInfo parsing once always enabled.
@@ -1085,8 +1067,8 @@ constexpr inst::Field make_field(std::string_view name) {
       TemplateParam{typeGraph.makeType<Primitive>(Primitive::Kind::UInt64),
                     "0"},
   };
-  genContainerTypeHandler(features, used, FuncGen::GetOiArrayContainerInfo(),
-                          arrayParams, code);
+  genContainerTypeHandler(used, FuncGen::GetOiArrayContainerInfo(), arrayParams,
+                          code);
 }
 
 }  // namespace
@@ -1096,11 +1078,10 @@ void CodeGen::addTypeHandlers(const TypeGraph& typeGraph, std::string& code) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
       genClassTypeHandler(*c, code);
     } else if (const auto* con = dynamic_cast<const Container*>(&t)) {
-      genContainerTypeHandler(config_.features, definedContainers_,
-                              con->containerInfo_, con->templateParams, code);
+      genContainerTypeHandler(definedContainers_, con->containerInfo_,
+                              con->templateParams, code);
     } else if (const auto* cap = dynamic_cast<const CaptureKeys*>(&t)) {
-      genContainerTypeHandler(config_.features, definedContainers_,
-                              cap->containerInfo(),
+      genContainerTypeHandler(definedContainers_, cap->containerInfo(),
                               cap->container().templateParams, code);
     }
   }
@@ -1216,14 +1197,14 @@ void CodeGen::generate(
   if (!config_.features[Feature::Library]) {
     FuncGen::DeclareExterns(code);
   }
-  if (!config_.features[Feature::TypedDataSegment]) {
+  if (!config_.features[Feature::TreeBuilderV2]) {
     defineMacros(code);
   }
   addIncludes(typeGraph, config_.features, code);
   defineInternalTypes(code);
   FuncGen::DefineJitLog(code, config_.features);
 
-  if (config_.features[Feature::TypedDataSegment]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     if (config_.features[Feature::Library]) {
       FuncGen::DefineBackInserterDataBuffer(code);
     } else {
@@ -1231,10 +1212,8 @@ void CodeGen::generate(
     }
     code += "using namespace oi;\n";
     code += "using namespace oi::detail;\n";
-    if (config_.features[Feature::TreeBuilderV2]) {
-      code += "using oi::exporters::ParsedData;\n";
-      code += "using namespace oi::exporters;\n";
-    }
+    code += "using oi::exporters::ParsedData;\n";
+    code += "using namespace oi::exporters;\n";
     code += "namespace OIInternal {\nnamespace {\n";
     FuncGen::DefineBasicTypeHandlers(code, config_.features);
     code += "} // namespace\n} // namespace OIInternal\n";
@@ -1254,7 +1233,7 @@ void CodeGen::generate(
    * process faster.
    */
   code += "namespace OIInternal {\nnamespace {\n";
-  if (!config_.features[Feature::TypedDataSegment]) {
+  if (!config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineEncodeData(code);
     FuncGen::DefineEncodeDataSize(code);
     FuncGen::DefineStoreData(code);
@@ -1269,7 +1248,7 @@ void CodeGen::generate(
     genExclusiveSizes(typeGraph, code);
   }
 
-  if (config_.features[Feature::TypedDataSegment]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     addStandardTypeHandlers(typeGraph, config_.features, code);
     addTypeHandlers(typeGraph, code);
   } else {
@@ -1286,10 +1265,8 @@ void CodeGen::generate(
   code += "} // namespace\n} // namespace OIInternal\n";
 
   const auto typeName = SymbolService::getTypeName(drgnType);
-  if (config_.features[Feature::Library]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
-  } else if (config_.features[Feature::TypedDataSegment]) {
-    FuncGen::DefineTopLevelGetSizeRefTyped(code, typeName, config_.features);
   } else {
     FuncGen::DefineTopLevelGetSizeRef(code, typeName, config_.features);
   }
@@ -1298,8 +1275,6 @@ void CodeGen::generate(
     FuncGen::DefineTreeBuilderInstructions(code, typeName,
                                            calculateExclusiveSize(rootType),
                                            enumerateTypeNames(rootType));
-  } else if (config_.features[Feature::TreeBuilderTypeChecking]) {
-    FuncGen::DefineOutputType(code, typeName);
   }
 
   if (!linkageName_.empty())
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index a5703be..40ca6b4 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -274,10 +274,6 @@ ContainerInfo::ContainerInfo(const fs::path& path) {
   } else {
     throw ContainerInfoError(path, "`codegen.decl` is a required field");
   }
-  if (std::optional<std::string> str =
-          codegenToml["handler"].value<std::string>()) {
-    codegen.handler = std::move(*str);
-  }
   if (std::optional<std::string> str =
           codegenToml["traversal_func"].value<std::string>()) {
     codegen.traversalFunc = std::move(*str);
@@ -324,5 +320,5 @@ ContainerInfo::ContainerInfo(std::string typeName_,
       ctype(ctype_),
       header(std::move(header_)),
       codegen(Codegen{"// DummyDecl %1%\n", "// DummyFunc %1%\n",
-                      "// DummyHandler %1%\n", "// DummyFunc\n"}) {
+                      "// DummyFunc\n"}) {
 }
diff --git a/oi/ContainerInfo.h b/oi/ContainerInfo.h
index 0ad9da9..376274d 100644
--- a/oi/ContainerInfo.h
+++ b/oi/ContainerInfo.h
@@ -36,7 +36,6 @@ struct ContainerInfo {
   struct Codegen {
     std::string decl;
     std::string func;
-    std::string handler = "";
     std::string traversalFunc = "";
     std::string extra = "";
     std::vector<Processor> processors{};
diff --git a/oi/Features.cpp b/oi/Features.cpp
index baf9344..9d82b17 100644
--- a/oi/Features.cpp
+++ b/oi/Features.cpp
@@ -40,11 +40,6 @@ std::optional<std::string_view> featureHelp(Feature f) {
       return "Use Type Graph for code generation (CodeGen v2).";
     case Feature::PruneTypeGraph:
       return "Prune unreachable nodes from the type graph";
-    case Feature::TypedDataSegment:
-      return "Use Typed Data Segment in generated code.";
-    case Feature::TreeBuilderTypeChecking:
-      return "Use Typed Data Segment to perform runtime Type Checking in "
-             "TreeBuilder.";
     case Feature::Library:
       return std::nullopt;  // Hide in OID help
     case Feature::TreeBuilderV2:
@@ -65,14 +60,8 @@ std::optional<std::string_view> featureHelp(Feature f) {
 
 std::span<const Feature> requirements(Feature f) {
   switch (f) {
-    case Feature::TypedDataSegment:
-      static constexpr std::array tds = {Feature::TypeGraph};
-      return tds;
-    case Feature::TreeBuilderTypeChecking:
-      static constexpr std::array tc = {Feature::TypedDataSegment};
-      return tc;
     case Feature::TreeBuilderV2:
-      static constexpr std::array tb2 = {Feature::TreeBuilderTypeChecking};
+      static constexpr std::array tb2 = {Feature::TypeGraph};
       return tb2;
     case Feature::Library:
       static constexpr std::array lib = {Feature::TreeBuilderV2};
diff --git a/oi/Features.h b/oi/Features.h
index 2e92ba0..959a576 100644
--- a/oi/Features.h
+++ b/oi/Features.h
@@ -22,20 +22,18 @@
 
 #include "oi/EnumBitset.h"
 
-#define OI_FEATURE_LIST                                    \
-  X(ChaseRawPointers, "chase-raw-pointers")                \
-  X(PackStructs, "pack-structs")                           \
-  X(GenPaddingStats, "gen-padding-stats")                  \
-  X(CaptureThriftIsset, "capture-thrift-isset")            \
-  X(TypeGraph, "type-graph")                               \
-  X(PruneTypeGraph, "prune-type-graph")                    \
-  X(TypedDataSegment, "typed-data-segment")                \
-  X(TreeBuilderTypeChecking, "tree-builder-type-checking") \
-  X(Library, "library")                                    \
-  X(TreeBuilderV2, "tree-builder-v2")                      \
-  X(GenJitDebug, "gen-jit-debug")                          \
-  X(JitLogging, "jit-logging")                             \
-  X(JitTiming, "jit-timing")                               \
+#define OI_FEATURE_LIST                         \
+  X(ChaseRawPointers, "chase-raw-pointers")     \
+  X(PackStructs, "pack-structs")                \
+  X(GenPaddingStats, "gen-padding-stats")       \
+  X(CaptureThriftIsset, "capture-thrift-isset") \
+  X(TypeGraph, "type-graph")                    \
+  X(PruneTypeGraph, "prune-type-graph")         \
+  X(Library, "library")                         \
+  X(TreeBuilderV2, "tree-builder-v2")           \
+  X(GenJitDebug, "gen-jit-debug")               \
+  X(JitLogging, "jit-logging")                  \
+  X(JitTiming, "jit-timing")                    \
   X(PolymorphicInheritance, "polymorphic-inheritance")
 
 namespace oi::detail {
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index dc494f9..d4f0e1e 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -352,83 +352,6 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
   testCode.append(fmt.str());
 }
 
-/*
- * DefineTopLevelGetSizeRefTyped
- *
- * Top level function to run OI on a type utilising static types and enabled
- * with feature '-ftyped-data-segment'.
- */
-void FuncGen::DefineTopLevelGetSizeRefTyped(std::string& testCode,
-                                            const std::string& rawType,
-                                            FeatureSet features) {
-  std::string func = R"(
-    #pragma GCC diagnostic push
-    #pragma GCC diagnostic ignored "-Wunknown-attributes"
-    /* RawType: %1% */
-    void __attribute__((used, retain)) getSize_%2$016x(const OIInternal::__ROOT_TYPE__& t)
-    #pragma GCC diagnostic pop
-    {
-    )";
-  if (features[Feature::JitTiming]) {
-    func += "      const auto startTime = std::chrono::steady_clock::now();\n";
-  }
-  func += R"(
-      pointers.initialize();
-      pointers.add((uintptr_t)&t);
-      auto data = reinterpret_cast<uintptr_t*>(dataBase);
-
-      // TODO: Replace these with types::st::Uint64 once the VarInt decoding
-      // logic is moved out of OIDebugger and into new TreeBuilder.
-      size_t dataSegOffset = 0;
-      data[dataSegOffset++] = oidMagicId;
-      data[dataSegOffset++] = cookieValue;
-      uintptr_t& writtenSize = data[dataSegOffset++];
-      writtenSize = 0;
-      uintptr_t& timeTakenNs = data[dataSegOffset++];
-
-      dataSegOffset *= sizeof(uintptr_t);
-      JLOG("%1% @");
-      JLOGPTR(&t);
-
-      using ContentType = OIInternal::TypeHandler<DataBuffer::DataSegment, OIInternal::__ROOT_TYPE__>::type;
-      using SuffixType = types::st::Pair<
-        DataBuffer::DataSegment,
-        types::st::VarInt<DataBuffer::DataSegment>,
-        types::st::VarInt<DataBuffer::DataSegment>
-      >;
-      using DataBufferType = types::st::Pair<
-        DataBuffer::DataSegment,
-        ContentType,
-        SuffixType
-      >;
-
-      DataBufferType db = DataBuffer::DataSegment(dataSegOffset);
-      SuffixType suffix = db.delegate([&t](auto ret) {
-        return OIInternal::getSizeType<DataBuffer::DataSegment>(t, ret);
-      });
-      types::st::Unit<DataBuffer::DataSegment> end = suffix
-        .write(123456789)
-        .write(123456789);
-
-      dataSegOffset = end.offset();
-      writtenSize = dataSegOffset;
-      dataBase += dataSegOffset;
-    )";
-  if (features[Feature::JitTiming]) {
-    func += R"(
-      timeTakenNs = std::chrono::duration_cast<std::chrono::nanoseconds>(
-        std::chrono::steady_clock::now() - startTime).count();
-      )";
-  }
-  func += R"(
-    }
-    )";
-
-  boost::format fmt =
-      boost::format(func) % rawType % std::hash<std::string>{}(rawType);
-  testCode.append(fmt.str());
-}
-
 /*
  * DefineOutputType
  *
@@ -803,23 +726,6 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
   ContainerInfo oiArray{"OIArray", UNKNOWN_TYPE,
                         "cstdint"};  // TODO: remove the need for a dummy header
 
-  oiArray.codegen.handler = R"(
-template<typename DB, typename T0, long unsigned int N>
-struct TypeHandler<DB, %1%<T0, N>> {
-  using type = types::st::List<DB, typename TypeHandler<DB, T0>::type>;
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, N> &container,
-      typename TypeHandler<DB, %1%<T0,N>>::type returnArg) {
-    auto tail = returnArg.write(N);
-    for (size_t i=0; i<N; i++) {
-      tail = tail.delegate([&container, i](auto ret) {
-          return TypeHandler<DB, T0>::getSizeType(container.vals[i], ret);
-      });
-    }
-    return tail.finish();
-  }
-};
-)";
   oiArray.codegen.traversalFunc = R"(
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
diff --git a/oi/FuncGen.h b/oi/FuncGen.h
index ff21841..042c0bc 100644
--- a/oi/FuncGen.h
+++ b/oi/FuncGen.h
@@ -61,9 +61,6 @@ class FuncGen {
   static void DefineTopLevelGetSizeRef(std::string& testCode,
                                        const std::string& rawType,
                                        FeatureSet features);
-  static void DefineTopLevelGetSizeRefTyped(std::string& testCode,
-                                            const std::string& rawType,
-                                            FeatureSet features);
   static void DefineOutputType(std::string& testCode,
                                const std::string& rawType);
   static void DefineTreeBuilderInstructions(
diff --git a/oi/OICompiler.cpp b/oi/OICompiler.cpp
index d036b33..fff8042 100644
--- a/oi/OICompiler.cpp
+++ b/oi/OICompiler.cpp
@@ -520,9 +520,8 @@ bool OICompiler::compile(const std::string& code,
 
   static const auto syntheticHeaders = std::array<
       std::pair<Feature, std::pair<std::string_view, std::string>>, 7>{{
-      {Feature::TypedDataSegment, {headers::oi_types_st_h, "oi/types/st.h"}},
-      {Feature::TreeBuilderTypeChecking,
-       {headers::oi_types_dy_h, "oi/types/dy.h"}},
+      {Feature::TreeBuilderV2, {headers::oi_types_st_h, "oi/types/st.h"}},
+      {Feature::TreeBuilderV2, {headers::oi_types_dy_h, "oi/types/dy.h"}},
       {Feature::TreeBuilderV2,
        {headers::oi_exporters_inst_h, "oi/exporters/inst.h"}},
       {Feature::TreeBuilderV2,
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index d98e7b8..f91fe16 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -180,12 +180,8 @@ int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
   auto oilTypes = findOilTypesAndNames(prog);
 
   std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TypedDataSegment, true},
-      {Feature::TreeBuilderTypeChecking, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
+      {Feature::TypeGraph, true},      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},        {Feature::PackStructs, true},
       {Feature::PruneTypeGraph, true},
   };
 
diff --git a/oi/OILibraryImpl.cpp b/oi/OILibraryImpl.cpp
index bade97c..9254fe0 100644
--- a/oi/OILibraryImpl.cpp
+++ b/oi/OILibraryImpl.cpp
@@ -173,12 +173,8 @@ std::pair<void*, const exporters::inst::Inst&> OILibraryImpl::compileCode() {
 namespace {
 std::map<Feature, bool> convertFeatures(std::unordered_set<oi::Feature> fs) {
   std::map<Feature, bool> out{
-      {Feature::TypeGraph, true},
-      {Feature::TypedDataSegment, true},
-      {Feature::TreeBuilderTypeChecking, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
+      {Feature::TypeGraph, true},      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},        {Feature::PackStructs, true},
       {Feature::PruneTypeGraph, true},
   };
 
diff --git a/types/README.md b/types/README.md
index ebde86c..e2b70fd 100644
--- a/types/README.md
+++ b/types/README.md
@@ -62,14 +62,10 @@ This document describes the format of the container definition files contained i
   will collide if they share the same name.
 
 
-## Changes introduced with Typed Data Segment
-- `decl` and `func` fields are ignored when using `-ftyped-data-segment` and the
-  `handler` field is used instead.
-
 ## Changes introduced with TreeBuilder V2
-- `decl`, `func`, and `handler` fields are ignored when using `-ftree-builder-v2`.
-   The `TypeHandler` is instead constructed from `traversal_func` and `processor`
-   entries.
+- `decl` and `func` fields are ignored when using `-ftree-builder-v2`. The
+  `TypeHandler` is constructed from `traversal_func` field and `processor`
+  entries.
 
 ## Changes introduced with TypeGraph
 - `typeName` and `matcher` fields have been merged into the single field `type_name`.

From 44dda5248d2e9333b4dba417b39600ea462d4fe6 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 3 Nov 2023 09:22:59 -0700
Subject: [PATCH 009/188] collapse TreeBuilderV2 features

Summary:

Currently there are two features between CodeGen v2 (TypeGraph) and TreeBuilder
v2. These are TypedDataSegment and TreeBuilderTypeChecking. Each of these
features currently has a full set of tests run in the CI and each have specific
exclusions.

Collapse these features into TreeBuilder v2. This allows for significantly
simplified testing as any OIL tests run under TreeBuilder v2 and any OID tests
run under TreeBuilder v1.

The reasoning behind this is I no longer intend to partially roll out this
feature. Full TreeBuilder v2 applies different conditions to containers than
the intermediate states, and writing these only to have them never deployed is
a waste of time.

Test Plan:
- it builds
- CI
---
 .circleci/config.yml         | 22 ---------
 oi/CodeGen.cpp               | 63 ++++++++----------------
 oi/ContainerInfo.cpp         |  6 +--
 oi/ContainerInfo.h           |  1 -
 oi/Features.cpp              | 13 +----
 oi/Features.h                | 26 +++++-----
 oi/FuncGen.cpp               | 94 ------------------------------------
 oi/FuncGen.h                 |  3 --
 oi/OICompiler.cpp            |  5 +-
 oi/OIGenerator.cpp           |  8 +--
 oi/OILibraryImpl.cpp         |  8 +--
 test/integration/arrays.toml |  2 +-
 types/README.md              | 10 ++--
 13 files changed, 43 insertions(+), 218 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 286f759..7e805ad 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -21,20 +21,6 @@ workflows:
           oid_test_args: "-ftype-graph"
           tests_regex: "OidIntegration\\..*"
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
-      - test:
-          name: test-typed-data-segment-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-ftyped-data-segment"
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
-      - test:
-          name: test-tree-builder-type-checking-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-ftree-builder-type-checking"
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
       - coverage:
           name: coverage
           requires:
@@ -43,14 +29,6 @@ workflows:
           name: coverage-type-graph
           requires:
             - test-type-graph-gcc
-      - coverage:
-          name: coverage-typed-data-segment
-          requires:
-            - test-typed-data-segment-gcc
-      - coverage:
-          name: coverage-tree-builder-type-checking
-          requires:
-            - test-tree-builder-type-checking-gcc
 
       - build:
           name: build-clang
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index fef0a47..0aeb255 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -117,17 +117,14 @@ void addIncludes(const TypeGraph& typeGraph,
                  FeatureSet features,
                  std::string& code) {
   std::set<std::string_view> includes{"cstddef"};
-  if (features[Feature::TypedDataSegment]) {
+  if (features[Feature::TreeBuilderV2]) {
+    code += "#define DEFINE_DESCRIBE 1\n";  // added before all includes
+
     includes.emplace("functional");
+    includes.emplace("oi/exporters/inst.h");
+    includes.emplace("oi/types/dy.h");
     includes.emplace("oi/types/st.h");
   }
-  if (features[Feature::TreeBuilderTypeChecking]) {
-    includes.emplace("oi/types/dy.h");
-
-    code += "#define DEFINE_DESCRIBE 1\n";  // added before all includes
-  }
-  if (features[Feature::TreeBuilderV2])
-    includes.emplace("oi/exporters/inst.h");
   if (features[Feature::Library]) {
     includes.emplace("vector");
     includes.emplace("oi/IntrospectionResult.h");
@@ -830,35 +827,20 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) {
   code += "  using type = ";
   genClassStaticType(c, code);
   code += ";\n";
-  if (config_.features[Feature::TreeBuilderV2])
-    genClassTreeBuilderInstructions(c, code);
+  genClassTreeBuilderInstructions(c, code);
   genClassTraversalFunction(c, code);
   code += "};\n";
 }
 
 namespace {
 
-void genContainerTypeHandler(FeatureSet features,
-                             std::unordered_set<const ContainerInfo*>& used,
+void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
                              const ContainerInfo& c,
                              std::span<const TemplateParam> templateParams,
                              std::string& code) {
   if (!used.insert(&c).second)
     return;
 
-  if (!features[Feature::TreeBuilderV2]) {
-    const auto& handler = c.codegen.handler;
-    if (handler.empty()) {
-      LOG(ERROR) << "`codegen.handler` must be specified for all containers "
-                    "under \"-ftyped-data-segment\", not specified for \"" +
-                        c.typeName + "\"";
-      throw std::runtime_error("missing `codegen.handler`");
-    }
-    auto fmt = boost::format(c.codegen.handler) % c.typeName;
-    code += fmt.str();
-    return;
-  }
-
   code += c.codegen.extra;
 
   // TODO: Move this check into the ContainerInfo parsing once always enabled.
@@ -1085,8 +1067,8 @@ constexpr inst::Field make_field(std::string_view name) {
       TemplateParam{typeGraph.makeType<Primitive>(Primitive::Kind::UInt64),
                     "0"},
   };
-  genContainerTypeHandler(features, used, FuncGen::GetOiArrayContainerInfo(),
-                          arrayParams, code);
+  genContainerTypeHandler(used, FuncGen::GetOiArrayContainerInfo(), arrayParams,
+                          code);
 }
 
 }  // namespace
@@ -1096,11 +1078,10 @@ void CodeGen::addTypeHandlers(const TypeGraph& typeGraph, std::string& code) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
       genClassTypeHandler(*c, code);
     } else if (const auto* con = dynamic_cast<const Container*>(&t)) {
-      genContainerTypeHandler(config_.features, definedContainers_,
-                              con->containerInfo_, con->templateParams, code);
+      genContainerTypeHandler(definedContainers_, con->containerInfo_,
+                              con->templateParams, code);
     } else if (const auto* cap = dynamic_cast<const CaptureKeys*>(&t)) {
-      genContainerTypeHandler(config_.features, definedContainers_,
-                              cap->containerInfo(),
+      genContainerTypeHandler(definedContainers_, cap->containerInfo(),
                               cap->container().templateParams, code);
     }
   }
@@ -1216,14 +1197,14 @@ void CodeGen::generate(
   if (!config_.features[Feature::Library]) {
     FuncGen::DeclareExterns(code);
   }
-  if (!config_.features[Feature::TypedDataSegment]) {
+  if (!config_.features[Feature::TreeBuilderV2]) {
     defineMacros(code);
   }
   addIncludes(typeGraph, config_.features, code);
   defineInternalTypes(code);
   FuncGen::DefineJitLog(code, config_.features);
 
-  if (config_.features[Feature::TypedDataSegment]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     if (config_.features[Feature::Library]) {
       FuncGen::DefineBackInserterDataBuffer(code);
     } else {
@@ -1231,10 +1212,8 @@ void CodeGen::generate(
     }
     code += "using namespace oi;\n";
     code += "using namespace oi::detail;\n";
-    if (config_.features[Feature::TreeBuilderV2]) {
-      code += "using oi::exporters::ParsedData;\n";
-      code += "using namespace oi::exporters;\n";
-    }
+    code += "using oi::exporters::ParsedData;\n";
+    code += "using namespace oi::exporters;\n";
     code += "namespace OIInternal {\nnamespace {\n";
     FuncGen::DefineBasicTypeHandlers(code, config_.features);
     code += "} // namespace\n} // namespace OIInternal\n";
@@ -1254,7 +1233,7 @@ void CodeGen::generate(
    * process faster.
    */
   code += "namespace OIInternal {\nnamespace {\n";
-  if (!config_.features[Feature::TypedDataSegment]) {
+  if (!config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineEncodeData(code);
     FuncGen::DefineEncodeDataSize(code);
     FuncGen::DefineStoreData(code);
@@ -1269,7 +1248,7 @@ void CodeGen::generate(
     genExclusiveSizes(typeGraph, code);
   }
 
-  if (config_.features[Feature::TypedDataSegment]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     addStandardTypeHandlers(typeGraph, config_.features, code);
     addTypeHandlers(typeGraph, code);
   } else {
@@ -1286,10 +1265,8 @@ void CodeGen::generate(
   code += "} // namespace\n} // namespace OIInternal\n";
 
   const auto typeName = SymbolService::getTypeName(drgnType);
-  if (config_.features[Feature::Library]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
-  } else if (config_.features[Feature::TypedDataSegment]) {
-    FuncGen::DefineTopLevelGetSizeRefTyped(code, typeName, config_.features);
   } else {
     FuncGen::DefineTopLevelGetSizeRef(code, typeName, config_.features);
   }
@@ -1298,8 +1275,6 @@ void CodeGen::generate(
     FuncGen::DefineTreeBuilderInstructions(code, typeName,
                                            calculateExclusiveSize(rootType),
                                            enumerateTypeNames(rootType));
-  } else if (config_.features[Feature::TreeBuilderTypeChecking]) {
-    FuncGen::DefineOutputType(code, typeName);
   }
 
   if (!linkageName_.empty())
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index a5703be..40ca6b4 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -274,10 +274,6 @@ ContainerInfo::ContainerInfo(const fs::path& path) {
   } else {
     throw ContainerInfoError(path, "`codegen.decl` is a required field");
   }
-  if (std::optional<std::string> str =
-          codegenToml["handler"].value<std::string>()) {
-    codegen.handler = std::move(*str);
-  }
   if (std::optional<std::string> str =
           codegenToml["traversal_func"].value<std::string>()) {
     codegen.traversalFunc = std::move(*str);
@@ -324,5 +320,5 @@ ContainerInfo::ContainerInfo(std::string typeName_,
       ctype(ctype_),
       header(std::move(header_)),
       codegen(Codegen{"// DummyDecl %1%\n", "// DummyFunc %1%\n",
-                      "// DummyHandler %1%\n", "// DummyFunc\n"}) {
+                      "// DummyFunc\n"}) {
 }
diff --git a/oi/ContainerInfo.h b/oi/ContainerInfo.h
index 0ad9da9..376274d 100644
--- a/oi/ContainerInfo.h
+++ b/oi/ContainerInfo.h
@@ -36,7 +36,6 @@ struct ContainerInfo {
   struct Codegen {
     std::string decl;
     std::string func;
-    std::string handler = "";
     std::string traversalFunc = "";
     std::string extra = "";
     std::vector<Processor> processors{};
diff --git a/oi/Features.cpp b/oi/Features.cpp
index baf9344..9d82b17 100644
--- a/oi/Features.cpp
+++ b/oi/Features.cpp
@@ -40,11 +40,6 @@ std::optional<std::string_view> featureHelp(Feature f) {
       return "Use Type Graph for code generation (CodeGen v2).";
     case Feature::PruneTypeGraph:
       return "Prune unreachable nodes from the type graph";
-    case Feature::TypedDataSegment:
-      return "Use Typed Data Segment in generated code.";
-    case Feature::TreeBuilderTypeChecking:
-      return "Use Typed Data Segment to perform runtime Type Checking in "
-             "TreeBuilder.";
     case Feature::Library:
       return std::nullopt;  // Hide in OID help
     case Feature::TreeBuilderV2:
@@ -65,14 +60,8 @@ std::optional<std::string_view> featureHelp(Feature f) {
 
 std::span<const Feature> requirements(Feature f) {
   switch (f) {
-    case Feature::TypedDataSegment:
-      static constexpr std::array tds = {Feature::TypeGraph};
-      return tds;
-    case Feature::TreeBuilderTypeChecking:
-      static constexpr std::array tc = {Feature::TypedDataSegment};
-      return tc;
     case Feature::TreeBuilderV2:
-      static constexpr std::array tb2 = {Feature::TreeBuilderTypeChecking};
+      static constexpr std::array tb2 = {Feature::TypeGraph};
       return tb2;
     case Feature::Library:
       static constexpr std::array lib = {Feature::TreeBuilderV2};
diff --git a/oi/Features.h b/oi/Features.h
index 2e92ba0..959a576 100644
--- a/oi/Features.h
+++ b/oi/Features.h
@@ -22,20 +22,18 @@
 
 #include "oi/EnumBitset.h"
 
-#define OI_FEATURE_LIST                                    \
-  X(ChaseRawPointers, "chase-raw-pointers")                \
-  X(PackStructs, "pack-structs")                           \
-  X(GenPaddingStats, "gen-padding-stats")                  \
-  X(CaptureThriftIsset, "capture-thrift-isset")            \
-  X(TypeGraph, "type-graph")                               \
-  X(PruneTypeGraph, "prune-type-graph")                    \
-  X(TypedDataSegment, "typed-data-segment")                \
-  X(TreeBuilderTypeChecking, "tree-builder-type-checking") \
-  X(Library, "library")                                    \
-  X(TreeBuilderV2, "tree-builder-v2")                      \
-  X(GenJitDebug, "gen-jit-debug")                          \
-  X(JitLogging, "jit-logging")                             \
-  X(JitTiming, "jit-timing")                               \
+#define OI_FEATURE_LIST                         \
+  X(ChaseRawPointers, "chase-raw-pointers")     \
+  X(PackStructs, "pack-structs")                \
+  X(GenPaddingStats, "gen-padding-stats")       \
+  X(CaptureThriftIsset, "capture-thrift-isset") \
+  X(TypeGraph, "type-graph")                    \
+  X(PruneTypeGraph, "prune-type-graph")         \
+  X(Library, "library")                         \
+  X(TreeBuilderV2, "tree-builder-v2")           \
+  X(GenJitDebug, "gen-jit-debug")               \
+  X(JitLogging, "jit-logging")                  \
+  X(JitTiming, "jit-timing")                    \
   X(PolymorphicInheritance, "polymorphic-inheritance")
 
 namespace oi::detail {
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index dc494f9..d4f0e1e 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -352,83 +352,6 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
   testCode.append(fmt.str());
 }
 
-/*
- * DefineTopLevelGetSizeRefTyped
- *
- * Top level function to run OI on a type utilising static types and enabled
- * with feature '-ftyped-data-segment'.
- */
-void FuncGen::DefineTopLevelGetSizeRefTyped(std::string& testCode,
-                                            const std::string& rawType,
-                                            FeatureSet features) {
-  std::string func = R"(
-    #pragma GCC diagnostic push
-    #pragma GCC diagnostic ignored "-Wunknown-attributes"
-    /* RawType: %1% */
-    void __attribute__((used, retain)) getSize_%2$016x(const OIInternal::__ROOT_TYPE__& t)
-    #pragma GCC diagnostic pop
-    {
-    )";
-  if (features[Feature::JitTiming]) {
-    func += "      const auto startTime = std::chrono::steady_clock::now();\n";
-  }
-  func += R"(
-      pointers.initialize();
-      pointers.add((uintptr_t)&t);
-      auto data = reinterpret_cast<uintptr_t*>(dataBase);
-
-      // TODO: Replace these with types::st::Uint64 once the VarInt decoding
-      // logic is moved out of OIDebugger and into new TreeBuilder.
-      size_t dataSegOffset = 0;
-      data[dataSegOffset++] = oidMagicId;
-      data[dataSegOffset++] = cookieValue;
-      uintptr_t& writtenSize = data[dataSegOffset++];
-      writtenSize = 0;
-      uintptr_t& timeTakenNs = data[dataSegOffset++];
-
-      dataSegOffset *= sizeof(uintptr_t);
-      JLOG("%1% @");
-      JLOGPTR(&t);
-
-      using ContentType = OIInternal::TypeHandler<DataBuffer::DataSegment, OIInternal::__ROOT_TYPE__>::type;
-      using SuffixType = types::st::Pair<
-        DataBuffer::DataSegment,
-        types::st::VarInt<DataBuffer::DataSegment>,
-        types::st::VarInt<DataBuffer::DataSegment>
-      >;
-      using DataBufferType = types::st::Pair<
-        DataBuffer::DataSegment,
-        ContentType,
-        SuffixType
-      >;
-
-      DataBufferType db = DataBuffer::DataSegment(dataSegOffset);
-      SuffixType suffix = db.delegate([&t](auto ret) {
-        return OIInternal::getSizeType<DataBuffer::DataSegment>(t, ret);
-      });
-      types::st::Unit<DataBuffer::DataSegment> end = suffix
-        .write(123456789)
-        .write(123456789);
-
-      dataSegOffset = end.offset();
-      writtenSize = dataSegOffset;
-      dataBase += dataSegOffset;
-    )";
-  if (features[Feature::JitTiming]) {
-    func += R"(
-      timeTakenNs = std::chrono::duration_cast<std::chrono::nanoseconds>(
-        std::chrono::steady_clock::now() - startTime).count();
-      )";
-  }
-  func += R"(
-    }
-    )";
-
-  boost::format fmt =
-      boost::format(func) % rawType % std::hash<std::string>{}(rawType);
-  testCode.append(fmt.str());
-}
-
 /*
  * DefineOutputType
  *
@@ -803,23 +726,6 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
   ContainerInfo oiArray{"OIArray", UNKNOWN_TYPE,
                         "cstdint"};  // TODO: remove the need for a dummy header
 
-  oiArray.codegen.handler = R"(
-template<typename DB, typename T0, long unsigned int N>
-struct TypeHandler<DB, %1%<T0, N>> {
-  using type = types::st::List<DB, typename TypeHandler<DB, T0>::type>;
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, N> &container,
-      typename TypeHandler<DB, %1%<T0,N>>::type returnArg) {
-    auto tail = returnArg.write(N);
-    for (size_t i=0; i<N; i++) {
-      tail = tail.delegate([&container, i](auto ret) {
-          return TypeHandler<DB, T0>::getSizeType(container.vals[i], ret);
-      });
-    }
-    return tail.finish();
-  }
-};
-)";
   oiArray.codegen.traversalFunc = R"(
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
diff --git a/oi/FuncGen.h b/oi/FuncGen.h
index ff21841..042c0bc 100644
--- a/oi/FuncGen.h
+++ b/oi/FuncGen.h
@@ -61,9 +61,6 @@ class FuncGen {
   static void DefineTopLevelGetSizeRef(std::string& testCode,
                                        const std::string& rawType,
                                        FeatureSet features);
-  static void DefineTopLevelGetSizeRefTyped(std::string& testCode,
-                                            const std::string& rawType,
-                                            FeatureSet features);
   static void DefineOutputType(std::string& testCode,
                                const std::string& rawType);
   static void DefineTreeBuilderInstructions(
diff --git a/oi/OICompiler.cpp b/oi/OICompiler.cpp
index d036b33..fff8042 100644
--- a/oi/OICompiler.cpp
+++ b/oi/OICompiler.cpp
@@ -520,9 +520,8 @@ bool OICompiler::compile(const std::string& code,
 
   static const auto syntheticHeaders = std::array<
       std::pair<Feature, std::pair<std::string_view, std::string>>, 7>{{
-      {Feature::TypedDataSegment, {headers::oi_types_st_h, "oi/types/st.h"}},
-      {Feature::TreeBuilderTypeChecking,
-       {headers::oi_types_dy_h, "oi/types/dy.h"}},
+      {Feature::TreeBuilderV2, {headers::oi_types_st_h, "oi/types/st.h"}},
+      {Feature::TreeBuilderV2, {headers::oi_types_dy_h, "oi/types/dy.h"}},
       {Feature::TreeBuilderV2,
        {headers::oi_exporters_inst_h, "oi/exporters/inst.h"}},
       {Feature::TreeBuilderV2,
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index d98e7b8..f91fe16 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -180,12 +180,8 @@ int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
   auto oilTypes = findOilTypesAndNames(prog);
 
   std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TypedDataSegment, true},
-      {Feature::TreeBuilderTypeChecking, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
+      {Feature::TypeGraph, true},      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},        {Feature::PackStructs, true},
       {Feature::PruneTypeGraph, true},
   };
 
diff --git a/oi/OILibraryImpl.cpp b/oi/OILibraryImpl.cpp
index bade97c..9254fe0 100644
--- a/oi/OILibraryImpl.cpp
+++ b/oi/OILibraryImpl.cpp
@@ -173,12 +173,8 @@ std::pair<void*, const exporters::inst::Inst&> OILibraryImpl::compileCode() {
 namespace {
 std::map<Feature, bool> convertFeatures(std::unordered_set<oi::Feature> fs) {
   std::map<Feature, bool> out{
-      {Feature::TypeGraph, true},
-      {Feature::TypedDataSegment, true},
-      {Feature::TreeBuilderTypeChecking, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
+      {Feature::TypeGraph, true},      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},        {Feature::PackStructs, true},
       {Feature::PruneTypeGraph, true},
   };
 
diff --git a/test/integration/arrays.toml b/test/integration/arrays.toml
index 3efb2d2..4a06b13 100644
--- a/test/integration/arrays.toml
+++ b/test/integration/arrays.toml
@@ -62,7 +62,7 @@ definitions = '''
       }]}]'''
   [cases.multidim_legacy] # Test for legacy behaviour. Remove with OICodeGen
     oil_disable = 'oil only runs on codegen v2'
-    cli_options = ["-Ftype-graph", "-Ftyped-data-segment", "-Ftree-builder-type-checking", "-Ftree-builder-v2"]
+    cli_options = ["-Ftype-graph", "-Ftree-builder-v2"]
     param_types = ["const MultiDim&"]
     setup = "return {};"
     expect_json = '''[{
diff --git a/types/README.md b/types/README.md
index ebde86c..e2b70fd 100644
--- a/types/README.md
+++ b/types/README.md
@@ -62,14 +62,10 @@ This document describes the format of the container definition files contained i
   will collide if they share the same name.
 
 
-## Changes introduced with Typed Data Segment
-- `decl` and `func` fields are ignored when using `-ftyped-data-segment` and the
-  `handler` field is used instead.
-
 ## Changes introduced with TreeBuilder V2
-- `decl`, `func`, and `handler` fields are ignored when using `-ftree-builder-v2`.
-   The `TypeHandler` is instead constructed from `traversal_func` and `processor`
-   entries.
+- `decl` and `func` fields are ignored when using `-ftree-builder-v2`. The
+  `TypeHandler` is constructed from `traversal_func` field and `processor`
+  entries.
 
 ## Changes introduced with TypeGraph
 - `typeName` and `matcher` fields have been merged into the single field `type_name`.

From f39cfa9537e4d9cd64941a7a833aaa4686edb8d3 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 3 Nov 2023 04:17:51 -0700
Subject: [PATCH 010/188] add range-v3 library

---
 CMakeLists.txt               | 10 ++++++++++
 oi/OICompiler.h              | 18 +++++-------------
 oi/type_graph/KeyCapture.cpp |  3 +--
 3 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 28ccdf4..d7f9ee7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -95,6 +95,15 @@ FetchContent_Declare(
 )
 FetchContent_Populate(folly)
 
+### range-v3
+FetchContent_Declare(
+  range-v3
+  GIT_REPOSITORY https://github.com/ericniebler/range-v3.git
+  GIT_TAG        a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
+  GIT_PROGRESS   TRUE
+)
+FetchContent_MakeAvailable(range-v3)
+
 set_project_warnings()
 
 if (ASAN)
@@ -291,6 +300,7 @@ target_link_libraries(oicore
   ${Boost_LIBRARIES}
   Boost::headers
   glog::glog
+  range-v3
   resources
 )
 if (FORCE_LLVM_STATIC)
diff --git a/oi/OICompiler.h b/oi/OICompiler.h
index b08528c..ca060ab 100644
--- a/oi/OICompiler.h
+++ b/oi/OICompiler.h
@@ -20,6 +20,7 @@
 #include <filesystem>
 #include <memory>
 #include <optional>
+#include <range/v3/algorithm/starts_with.hpp>
 #include <ranges>
 #include <set>
 #include <span>
@@ -204,19 +205,10 @@ std::optional<std::vector<uintptr_t>> OICompiler::locateOpcodes(
 
   std::vector<uintptr_t> locs;
   while (auto inst = DG()) {
-    auto it = std::find_if(
-        std::begin(needles), std::end(needles), [&](const auto& needle) {
-          // std::ranges::starts_with(inst->opcodes, needle)
-          if (std::ranges::size(needle) > std::ranges::size(inst->opcodes))
-            return false;
-          auto it1 = std::ranges::begin(inst->opcodes);
-          auto it2 = std::ranges::begin(needle);
-          for (; it2 != std::ranges::end(needle); ++it1, ++it2) {
-            if (*it1 != *it2)
-              return false;
-          }
-          return true;
-        });
+    auto it = std::find_if(std::begin(needles), std::end(needles),
+                           [&](const auto& needle) {
+                             return ranges::starts_with(inst->opcodes, needle);
+                           });
 
     if (it != std::end(needles)) {
       locs.push_back(inst->offset);
diff --git a/oi/type_graph/KeyCapture.cpp b/oi/type_graph/KeyCapture.cpp
index 5e09020..e3f7d11 100644
--- a/oi/type_graph/KeyCapture.cpp
+++ b/oi/type_graph/KeyCapture.cpp
@@ -66,8 +66,7 @@ void KeyCapture::visit(Class& c) {
       continue;
     if (!keyToCapture.member.has_value())
       continue;
-    for (size_t i = 0; i < c.members.size(); i++) {
-      auto& member = c.members[i];
+    for (auto& member : c.members) {
       if (member.name != *keyToCapture.member)
         continue;
 

From 72986046a01c51d658a7cff01b242726bde01bd0 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Mon, 13 Nov 2023 05:14:16 -0800
Subject: [PATCH 011/188] collapse TreeBuilderV2 features

Summary:

Currently there are two features between CodeGen v2 (TypeGraph) and TreeBuilder
v2. These are TypedDataSegment and TreeBuilderTypeChecking. Each of these
features currently has a full set of tests run in the CI and each have specific
exclusions.

Collapse these features into TreeBuilder v2. This allows for significantly
simplified testing as any OIL tests run under TreeBuilder v2 and any OID tests
run under TreeBuilder v1.

The reasoning behind this is I no longer intend to partially roll out this
feature. Full TreeBuilder v2 applies different conditions to containers than
the intermediate states, and writing these only to have them never deployed is
a waste of time.

Test Plan:
- it builds
- CI
---
 .circleci/config.yml         |  22 -------
 oi/CodeGen.cpp               |  63 ++++++-------------
 oi/ContainerInfo.cpp         |   6 +-
 oi/ContainerInfo.h           |   1 -
 oi/Features.cpp              |  13 +---
 oi/Features.h                |  26 ++++----
 oi/FuncGen.cpp               | 114 -----------------------------------
 oi/FuncGen.h                 |   5 --
 oi/OICompiler.cpp            |   5 +-
 oi/OIGenerator.cpp           |   8 +--
 oi/OILibraryImpl.cpp         |   8 +--
 test/integration/arrays.toml |   2 +-
 types/README.md              |  10 +--
 13 files changed, 43 insertions(+), 240 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 286f759..7e805ad 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -21,20 +21,6 @@ workflows:
           oid_test_args: "-ftype-graph"
           tests_regex: "OidIntegration\\..*"
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
-      - test:
-          name: test-typed-data-segment-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-ftyped-data-segment"
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
-      - test:
-          name: test-tree-builder-type-checking-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-ftree-builder-type-checking"
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
       - coverage:
           name: coverage
           requires:
@@ -43,14 +29,6 @@ workflows:
           name: coverage-type-graph
           requires:
             - test-type-graph-gcc
-      - coverage:
-          name: coverage-typed-data-segment
-          requires:
-            - test-typed-data-segment-gcc
-      - coverage:
-          name: coverage-tree-builder-type-checking
-          requires:
-            - test-tree-builder-type-checking-gcc
 
       - build:
           name: build-clang
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 3fa0374..f2c77eb 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -119,17 +119,14 @@ void addIncludes(const TypeGraph& typeGraph,
                  FeatureSet features,
                  std::string& code) {
   std::set<std::string_view> includes{"cstddef"};
-  if (features[Feature::TypedDataSegment]) {
+  if (features[Feature::TreeBuilderV2]) {
+    code += "#define DEFINE_DESCRIBE 1\n";  // added before all includes
+
     includes.emplace("functional");
+    includes.emplace("oi/exporters/inst.h");
+    includes.emplace("oi/types/dy.h");
     includes.emplace("oi/types/st.h");
   }
-  if (features[Feature::TreeBuilderTypeChecking]) {
-    includes.emplace("oi/types/dy.h");
-
-    code += "#define DEFINE_DESCRIBE 1\n";  // added before all includes
-  }
-  if (features[Feature::TreeBuilderV2])
-    includes.emplace("oi/exporters/inst.h");
   if (features[Feature::Library]) {
     includes.emplace("vector");
     includes.emplace("oi/IntrospectionResult.h");
@@ -832,35 +829,20 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) {
   code += "  using type = ";
   genClassStaticType(c, code);
   code += ";\n";
-  if (config_.features[Feature::TreeBuilderV2])
-    genClassTreeBuilderInstructions(c, code);
+  genClassTreeBuilderInstructions(c, code);
   genClassTraversalFunction(c, code);
   code += "};\n";
 }
 
 namespace {
 
-void genContainerTypeHandler(FeatureSet features,
-                             std::unordered_set<const ContainerInfo*>& used,
+void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
                              const ContainerInfo& c,
                              std::span<const TemplateParam> templateParams,
                              std::string& code) {
   if (!used.insert(&c).second)
     return;
 
-  if (!features[Feature::TreeBuilderV2]) {
-    const auto& handler = c.codegen.handler;
-    if (handler.empty()) {
-      LOG(ERROR) << "`codegen.handler` must be specified for all containers "
-                    "under \"-ftyped-data-segment\", not specified for \"" +
-                        c.typeName + "\"";
-      throw std::runtime_error("missing `codegen.handler`");
-    }
-    auto fmt = boost::format(c.codegen.handler) % c.typeName;
-    code += fmt.str();
-    return;
-  }
-
   code += c.codegen.extra;
 
   // TODO: Move this check into the ContainerInfo parsing once always enabled.
@@ -1087,8 +1069,8 @@ constexpr inst::Field make_field(std::string_view name) {
       TemplateParam{typeGraph.makeType<Primitive>(Primitive::Kind::UInt64),
                     "0"},
   };
-  genContainerTypeHandler(features, used, FuncGen::GetOiArrayContainerInfo(),
-                          arrayParams, code);
+  genContainerTypeHandler(used, FuncGen::GetOiArrayContainerInfo(), arrayParams,
+                          code);
 }
 
 }  // namespace
@@ -1098,11 +1080,10 @@ void CodeGen::addTypeHandlers(const TypeGraph& typeGraph, std::string& code) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
       genClassTypeHandler(*c, code);
     } else if (const auto* con = dynamic_cast<const Container*>(&t)) {
-      genContainerTypeHandler(config_.features, definedContainers_,
-                              con->containerInfo_, con->templateParams, code);
+      genContainerTypeHandler(definedContainers_, con->containerInfo_,
+                              con->templateParams, code);
     } else if (const auto* cap = dynamic_cast<const CaptureKeys*>(&t)) {
-      genContainerTypeHandler(config_.features, definedContainers_,
-                              cap->containerInfo(),
+      genContainerTypeHandler(definedContainers_, cap->containerInfo(),
                               cap->container().templateParams, code);
     }
   }
@@ -1222,14 +1203,14 @@ void CodeGen::generate(
   if (!config_.features[Feature::Library]) {
     FuncGen::DeclareExterns(code);
   }
-  if (!config_.features[Feature::TypedDataSegment]) {
+  if (!config_.features[Feature::TreeBuilderV2]) {
     defineMacros(code);
   }
   addIncludes(typeGraph, config_.features, code);
   defineInternalTypes(code);
   FuncGen::DefineJitLog(code, config_.features);
 
-  if (config_.features[Feature::TypedDataSegment]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     if (config_.features[Feature::Library]) {
       FuncGen::DefineBackInserterDataBuffer(code);
     } else {
@@ -1237,10 +1218,8 @@ void CodeGen::generate(
     }
     code += "using namespace oi;\n";
     code += "using namespace oi::detail;\n";
-    if (config_.features[Feature::TreeBuilderV2]) {
-      code += "using oi::exporters::ParsedData;\n";
-      code += "using namespace oi::exporters;\n";
-    }
+    code += "using oi::exporters::ParsedData;\n";
+    code += "using namespace oi::exporters;\n";
     code += "namespace OIInternal {\nnamespace {\n";
     FuncGen::DefineBasicTypeHandlers(code, config_.features);
     code += "} // namespace\n} // namespace OIInternal\n";
@@ -1260,7 +1239,7 @@ void CodeGen::generate(
    * process faster.
    */
   code += "namespace OIInternal {\nnamespace {\n";
-  if (!config_.features[Feature::TypedDataSegment]) {
+  if (!config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineEncodeData(code);
     FuncGen::DefineEncodeDataSize(code);
     FuncGen::DefineStoreData(code);
@@ -1275,7 +1254,7 @@ void CodeGen::generate(
     genExclusiveSizes(typeGraph, code);
   }
 
-  if (config_.features[Feature::TypedDataSegment]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     addStandardTypeHandlers(typeGraph, config_.features, code);
     addTypeHandlers(typeGraph, code);
   } else {
@@ -1292,10 +1271,8 @@ void CodeGen::generate(
   code += "} // namespace\n} // namespace OIInternal\n";
 
   const auto typeName = SymbolService::getTypeName(drgnType);
-  if (config_.features[Feature::Library]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
-  } else if (config_.features[Feature::TypedDataSegment]) {
-    FuncGen::DefineTopLevelGetSizeRefTyped(code, typeName, config_.features);
   } else {
     FuncGen::DefineTopLevelGetSizeRef(code, typeName, config_.features);
   }
@@ -1304,8 +1281,6 @@ void CodeGen::generate(
     FuncGen::DefineTreeBuilderInstructions(code, typeName,
                                            calculateExclusiveSize(rootType),
                                            enumerateTypeNames(rootType));
-  } else if (config_.features[Feature::TreeBuilderTypeChecking]) {
-    FuncGen::DefineOutputType(code, typeName);
   }
 
   if (!linkageName_.empty())
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index a5703be..40ca6b4 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -274,10 +274,6 @@ ContainerInfo::ContainerInfo(const fs::path& path) {
   } else {
     throw ContainerInfoError(path, "`codegen.decl` is a required field");
   }
-  if (std::optional<std::string> str =
-          codegenToml["handler"].value<std::string>()) {
-    codegen.handler = std::move(*str);
-  }
   if (std::optional<std::string> str =
           codegenToml["traversal_func"].value<std::string>()) {
     codegen.traversalFunc = std::move(*str);
@@ -324,5 +320,5 @@ ContainerInfo::ContainerInfo(std::string typeName_,
       ctype(ctype_),
       header(std::move(header_)),
       codegen(Codegen{"// DummyDecl %1%\n", "// DummyFunc %1%\n",
-                      "// DummyHandler %1%\n", "// DummyFunc\n"}) {
+                      "// DummyFunc\n"}) {
 }
diff --git a/oi/ContainerInfo.h b/oi/ContainerInfo.h
index 0ad9da9..376274d 100644
--- a/oi/ContainerInfo.h
+++ b/oi/ContainerInfo.h
@@ -36,7 +36,6 @@ struct ContainerInfo {
   struct Codegen {
     std::string decl;
     std::string func;
-    std::string handler = "";
     std::string traversalFunc = "";
     std::string extra = "";
     std::vector<Processor> processors{};
diff --git a/oi/Features.cpp b/oi/Features.cpp
index baf9344..9d82b17 100644
--- a/oi/Features.cpp
+++ b/oi/Features.cpp
@@ -40,11 +40,6 @@ std::optional<std::string_view> featureHelp(Feature f) {
       return "Use Type Graph for code generation (CodeGen v2).";
     case Feature::PruneTypeGraph:
       return "Prune unreachable nodes from the type graph";
-    case Feature::TypedDataSegment:
-      return "Use Typed Data Segment in generated code.";
-    case Feature::TreeBuilderTypeChecking:
-      return "Use Typed Data Segment to perform runtime Type Checking in "
-             "TreeBuilder.";
     case Feature::Library:
       return std::nullopt;  // Hide in OID help
     case Feature::TreeBuilderV2:
@@ -65,14 +60,8 @@ std::optional<std::string_view> featureHelp(Feature f) {
 
 std::span<const Feature> requirements(Feature f) {
   switch (f) {
-    case Feature::TypedDataSegment:
-      static constexpr std::array tds = {Feature::TypeGraph};
-      return tds;
-    case Feature::TreeBuilderTypeChecking:
-      static constexpr std::array tc = {Feature::TypedDataSegment};
-      return tc;
     case Feature::TreeBuilderV2:
-      static constexpr std::array tb2 = {Feature::TreeBuilderTypeChecking};
+      static constexpr std::array tb2 = {Feature::TypeGraph};
       return tb2;
     case Feature::Library:
       static constexpr std::array lib = {Feature::TreeBuilderV2};
diff --git a/oi/Features.h b/oi/Features.h
index 2e92ba0..959a576 100644
--- a/oi/Features.h
+++ b/oi/Features.h
@@ -22,20 +22,18 @@
 
 #include "oi/EnumBitset.h"
 
-#define OI_FEATURE_LIST                                    \
-  X(ChaseRawPointers, "chase-raw-pointers")                \
-  X(PackStructs, "pack-structs")                           \
-  X(GenPaddingStats, "gen-padding-stats")                  \
-  X(CaptureThriftIsset, "capture-thrift-isset")            \
-  X(TypeGraph, "type-graph")                               \
-  X(PruneTypeGraph, "prune-type-graph")                    \
-  X(TypedDataSegment, "typed-data-segment")                \
-  X(TreeBuilderTypeChecking, "tree-builder-type-checking") \
-  X(Library, "library")                                    \
-  X(TreeBuilderV2, "tree-builder-v2")                      \
-  X(GenJitDebug, "gen-jit-debug")                          \
-  X(JitLogging, "jit-logging")                             \
-  X(JitTiming, "jit-timing")                               \
+#define OI_FEATURE_LIST                         \
+  X(ChaseRawPointers, "chase-raw-pointers")     \
+  X(PackStructs, "pack-structs")                \
+  X(GenPaddingStats, "gen-padding-stats")       \
+  X(CaptureThriftIsset, "capture-thrift-isset") \
+  X(TypeGraph, "type-graph")                    \
+  X(PruneTypeGraph, "prune-type-graph")         \
+  X(Library, "library")                         \
+  X(TreeBuilderV2, "tree-builder-v2")           \
+  X(GenJitDebug, "gen-jit-debug")               \
+  X(JitLogging, "jit-logging")                  \
+  X(JitTiming, "jit-timing")                    \
   X(PolymorphicInheritance, "polymorphic-inheritance")
 
 namespace oi::detail {
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index dc494f9..c11c1e5 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -352,103 +352,6 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
   testCode.append(fmt.str());
 }
 
-/*
- * DefineTopLevelGetSizeRefTyped
- *
- * Top level function to run OI on a type utilising static types and enabled
- * with feature '-ftyped-data-segment'.
- */
-void FuncGen::DefineTopLevelGetSizeRefTyped(std::string& testCode,
-                                            const std::string& rawType,
-                                            FeatureSet features) {
-  std::string func = R"(
-    #pragma GCC diagnostic push
-    #pragma GCC diagnostic ignored "-Wunknown-attributes"
-    /* RawType: %1% */
-    void __attribute__((used, retain)) getSize_%2$016x(const OIInternal::__ROOT_TYPE__& t)
-    #pragma GCC diagnostic pop
-    {
-    )";
-  if (features[Feature::JitTiming]) {
-    func += "      const auto startTime = std::chrono::steady_clock::now();\n";
-  }
-  func += R"(
-      pointers.initialize();
-      pointers.add((uintptr_t)&t);
-      auto data = reinterpret_cast<uintptr_t*>(dataBase);
-
-      // TODO: Replace these with types::st::Uint64 once the VarInt decoding
-      // logic is moved out of OIDebugger and into new TreeBuilder.
-      size_t dataSegOffset = 0;
-      data[dataSegOffset++] = oidMagicId;
-      data[dataSegOffset++] = cookieValue;
-      uintptr_t& writtenSize = data[dataSegOffset++];
-      writtenSize = 0;
-      uintptr_t& timeTakenNs = data[dataSegOffset++];
-
-      dataSegOffset *= sizeof(uintptr_t);
-      JLOG("%1% @");
-      JLOGPTR(&t);
-
-      using ContentType = OIInternal::TypeHandler<DataBuffer::DataSegment, OIInternal::__ROOT_TYPE__>::type;
-      using SuffixType = types::st::Pair<
-        DataBuffer::DataSegment,
-        types::st::VarInt<DataBuffer::DataSegment>,
-        types::st::VarInt<DataBuffer::DataSegment>
-      >;
-      using DataBufferType = types::st::Pair<
-        DataBuffer::DataSegment,
-        ContentType,
-        SuffixType
-      >;
-
-      DataBufferType db = DataBuffer::DataSegment(dataSegOffset);
-      SuffixType suffix = db.delegate([&t](auto ret) {
-        return OIInternal::getSizeType<DataBuffer::DataSegment>(t, ret);
-      });
-      types::st::Unit<DataBuffer::DataSegment> end = suffix
-        .write(123456789)
-        .write(123456789);
-
-      dataSegOffset = end.offset();
-      writtenSize = dataSegOffset;
-      dataBase += dataSegOffset;
-    )";
-  if (features[Feature::JitTiming]) {
-    func += R"(
-      timeTakenNs = std::chrono::duration_cast<std::chrono::nanoseconds>(
-        std::chrono::steady_clock::now() - startTime).count();
-      )";
-  }
-  func += R"(
-    }
-    )";
-
-  boost::format fmt =
-      boost::format(func) % rawType % std::hash<std::string>{}(rawType);
-  testCode.append(fmt.str());
-}
-
-/*
- * DefineOutputType
- *
- * Present the dynamic type of an object for OID/OIL/OITB to link against.
- */
-void FuncGen::DefineOutputType(std::string& code, const std::string& rawType) {
-  std::string func = R"(
-    #pragma GCC diagnostic push
-    #pragma GCC diagnostic ignored "-Wunknown-attributes"
-    /* RawType: %1% */
-    extern const types::dy::Dynamic __attribute__((used, retain)) outputType%2$016x =
-          OIInternal::TypeHandler<DataBuffer::DataSegment, OIInternal::__ROOT_TYPE__>::type::describe;
-    #pragma GCC diagnostic pop
-)";
-
-  boost::format fmt =
-      boost::format(func) % rawType % std::hash<std::string>{}(rawType);
-  code.append(fmt.str());
-}
-
 void FuncGen::DefineTreeBuilderInstructions(
     std::string& code,
     const std::string& rawType,
@@ -803,23 +706,6 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
   ContainerInfo oiArray{"OIArray", UNKNOWN_TYPE,
                         "cstdint"};  // TODO: remove the need for a dummy header
 
-  oiArray.codegen.handler = R"(
-template<typename DB, typename T0, long unsigned int N>
-struct TypeHandler<DB, %1%<T0, N>> {
-  using type = types::st::List<DB, typename TypeHandler<DB, T0>::type>;
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, N> &container,
-      typename TypeHandler<DB, %1%<T0,N>>::type returnArg) {
-    auto tail = returnArg.write(N);
-    for (size_t i=0; i<N; i++) {
-      tail = tail.delegate([&container, i](auto ret) {
-          return TypeHandler<DB, T0>::getSizeType(container.vals[i], ret);
-      });
-    }
-    return tail.finish();
-  }
-};
-)";
   oiArray.codegen.traversalFunc = R"(
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
diff --git a/oi/FuncGen.h b/oi/FuncGen.h
index ff21841..33aaf9c 100644
--- a/oi/FuncGen.h
+++ b/oi/FuncGen.h
@@ -61,11 +61,6 @@ class FuncGen {
   static void DefineTopLevelGetSizeRef(std::string& testCode,
                                        const std::string& rawType,
                                        FeatureSet features);
-  static void DefineTopLevelGetSizeRefTyped(std::string& testCode,
-                                            const std::string& rawType,
-                                            FeatureSet features);
-  static void DefineOutputType(std::string& testCode,
-                               const std::string& rawType);
   static void DefineTreeBuilderInstructions(
       std::string& testCode,
       const std::string& rawType,
diff --git a/oi/OICompiler.cpp b/oi/OICompiler.cpp
index d036b33..fff8042 100644
--- a/oi/OICompiler.cpp
+++ b/oi/OICompiler.cpp
@@ -520,9 +520,8 @@ bool OICompiler::compile(const std::string& code,
 
   static const auto syntheticHeaders = std::array<
       std::pair<Feature, std::pair<std::string_view, std::string>>, 7>{{
-      {Feature::TypedDataSegment, {headers::oi_types_st_h, "oi/types/st.h"}},
-      {Feature::TreeBuilderTypeChecking,
-       {headers::oi_types_dy_h, "oi/types/dy.h"}},
+      {Feature::TreeBuilderV2, {headers::oi_types_st_h, "oi/types/st.h"}},
+      {Feature::TreeBuilderV2, {headers::oi_types_dy_h, "oi/types/dy.h"}},
       {Feature::TreeBuilderV2,
        {headers::oi_exporters_inst_h, "oi/exporters/inst.h"}},
       {Feature::TreeBuilderV2,
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index d98e7b8..f91fe16 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -180,12 +180,8 @@ int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
   auto oilTypes = findOilTypesAndNames(prog);
 
   std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TypedDataSegment, true},
-      {Feature::TreeBuilderTypeChecking, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
+      {Feature::TypeGraph, true},      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},        {Feature::PackStructs, true},
       {Feature::PruneTypeGraph, true},
   };
 
diff --git a/oi/OILibraryImpl.cpp b/oi/OILibraryImpl.cpp
index bade97c..9254fe0 100644
--- a/oi/OILibraryImpl.cpp
+++ b/oi/OILibraryImpl.cpp
@@ -173,12 +173,8 @@ std::pair<void*, const exporters::inst::Inst&> OILibraryImpl::compileCode() {
 namespace {
 std::map<Feature, bool> convertFeatures(std::unordered_set<oi::Feature> fs) {
   std::map<Feature, bool> out{
-      {Feature::TypeGraph, true},
-      {Feature::TypedDataSegment, true},
-      {Feature::TreeBuilderTypeChecking, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
+      {Feature::TypeGraph, true},      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},        {Feature::PackStructs, true},
       {Feature::PruneTypeGraph, true},
   };
 
diff --git a/test/integration/arrays.toml b/test/integration/arrays.toml
index 3efb2d2..4a06b13 100644
--- a/test/integration/arrays.toml
+++ b/test/integration/arrays.toml
@@ -62,7 +62,7 @@ definitions = '''
       }]}]'''
   [cases.multidim_legacy] # Test for legacy behaviour. Remove with OICodeGen
     oil_disable = 'oil only runs on codegen v2'
-    cli_options = ["-Ftype-graph", "-Ftyped-data-segment", "-Ftree-builder-type-checking", "-Ftree-builder-v2"]
+    cli_options = ["-Ftype-graph", "-Ftree-builder-v2"]
     param_types = ["const MultiDim&"]
     setup = "return {};"
     expect_json = '''[{
diff --git a/types/README.md b/types/README.md
index ebde86c..e2b70fd 100644
--- a/types/README.md
+++ b/types/README.md
@@ -62,14 +62,10 @@ This document describes the format of the container definition files contained i
   will collide if they share the same name.
 
 
-## Changes introduced with Typed Data Segment
-- `decl` and `func` fields are ignored when using `-ftyped-data-segment` and the
-  `handler` field is used instead.
-
 ## Changes introduced with TreeBuilder V2
-- `decl`, `func`, and `handler` fields are ignored when using `-ftree-builder-v2`.
-   The `TypeHandler` is instead constructed from `traversal_func` and `processor`
-   entries.
+- `decl` and `func` fields are ignored when using `-ftree-builder-v2`. The
+  `TypeHandler` is constructed from `traversal_func` field and `processor`
+  entries.
 
 ## Changes introduced with TypeGraph
 - `typeName` and `matcher` fields have been merged into the single field `type_name`.

From 79029948f2039d7c76ae912eb21919827924ed3e Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Mon, 13 Nov 2023 08:50:09 -0800
Subject: [PATCH 012/188] clang-format: disable bin packing

Bin packing often makes code hard to read. Disable it entirely.

Test plan:
- CI
---
 .clang-format                                |   1 +
 include/oi/exporters/CSV.h                   |  15 +-
 oi/CodeGen.cpp                               |  32 +++--
 oi/Config.cpp                                |   8 +-
 oi/ContainerInfo.cpp                         |   6 +-
 oi/FuncGen.cpp                               |   3 +-
 oi/Metrics.cpp                               |   7 +-
 oi/OICodeGen.cpp                             |  93 ++++++++-----
 oi/OICompiler.cpp                            |  69 ++++++----
 oi/OID.cpp                                   | 101 +++++++++-----
 oi/OIDebugger.cpp                            | 138 ++++++++++++-------
 oi/OIDebugger.h                              |   6 +-
 oi/OIGenerator.cpp                           |  17 ++-
 oi/OILibraryImpl.cpp                         |  18 ++-
 oi/OIParser.h                                |   3 +-
 oi/PaddingHunter.cpp                         |   3 +-
 oi/SymbolService.cpp                         |  28 ++--
 oi/TreeBuilder.cpp                           |  52 +++----
 oi/exporters/Json.cpp                        |   4 +-
 oi/exporters/test/TypeCheckingWalkerTest.cpp |   4 +-
 oi/type_graph/AddPadding.cpp                 |   4 +-
 oi/type_graph/DrgnParser.cpp                 |   7 +-
 oi/type_graph/Flattener.cpp                  |  11 +-
 oi/type_graph/TypeIdentifier.cpp             |   4 +-
 test/TypeGraphParser.cpp                     |   4 +-
 test/integration/runner_common.cpp           |  53 +++++--
 test/test_add_children.cpp                   |   3 +-
 test/test_add_padding.cpp                    |  24 ++--
 test/test_alignment_calc.cpp                 |  36 +++--
 test/test_codegen.cpp                        |   3 +-
 test/test_compiler.cpp                       |  30 ++--
 test/test_drgn_parser.cpp                    |  12 +-
 test/test_enforce_compatibility.cpp          |   9 +-
 test/test_flattener.cpp                      |  84 +++++++----
 test/test_identify_containers.cpp            |   3 +-
 test/test_key_capture.cpp                    |   9 +-
 test/test_parser.cpp                         |   3 +-
 test/test_prune.cpp                          |   9 +-
 test/test_remove_members.cpp                 |  18 ++-
 test/test_remove_top_level_pointer.cpp       |   3 +-
 test/test_type_identifier.cpp                |  18 ++-
 tools/OILGen.cpp                             |  34 +++--
 tools/OIP.cpp                                |   8 +-
 tools/OIRP.cpp                               |   6 +-
 tools/OITB.cpp                               |  23 +++-
 45 files changed, 668 insertions(+), 358 deletions(-)

diff --git a/.clang-format b/.clang-format
index 0341afb..795cdc3 100644
--- a/.clang-format
+++ b/.clang-format
@@ -9,3 +9,4 @@ AllowShortLoopsOnASingleLine: false
 DerivePointerAlignment: false
 PointerAlignment: Left
 BinPackParameters: false
+BinPackArguments: false
diff --git a/include/oi/exporters/CSV.h b/include/oi/exporters/CSV.h
index 0afe61b..b71e045 100644
--- a/include/oi/exporters/CSV.h
+++ b/include/oi/exporters/CSV.h
@@ -39,10 +39,17 @@ class CSV {
   static constexpr std::string_view kEscapedQuote = "\\\"";
   static constexpr std::string_view kListDelimiter = ";";
 
-  static constexpr std::string_view kColumns[] = {
-      "id",         "name",          "typePath", "typeNames",
-      "staticSize", "exclusiveSize", "pointer",  "length",
-      "capacity",   "is_set",        "parent_id"};
+  static constexpr std::string_view kColumns[] = {"id",
+                                                  "name",
+                                                  "typePath",
+                                                  "typeNames",
+                                                  "staticSize",
+                                                  "exclusiveSize",
+                                                  "pointer",
+                                                  "length",
+                                                  "capacity",
+                                                  "is_set",
+                                                  "parent_id"};
 
   size_t id_ = 0;
   std::vector<size_t> parentIdStack_ = {0};
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 3fa0374..2fe0566 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -194,12 +194,12 @@ namespace {
 
 size_t calculateExclusiveSize(const Type& t) {
   if (const auto* c = dynamic_cast<const Class*>(&t)) {
-    return std::accumulate(c->members.cbegin(), c->members.cend(), 0,
-                           [](size_t a, const auto& m) {
-                             if (m.name.starts_with(AddPadding::MemberPrefix))
-                               return a + m.type().size();
-                             return a;
-                           });
+    return std::accumulate(
+        c->members.cbegin(), c->members.cend(), 0, [](size_t a, const auto& m) {
+          if (m.name.starts_with(AddPadding::MemberPrefix))
+            return a + m.type().size();
+          return a;
+        });
   }
   return t.size();
 }
@@ -1087,8 +1087,8 @@ constexpr inst::Field make_field(std::string_view name) {
       TemplateParam{typeGraph.makeType<Primitive>(Primitive::Kind::UInt64),
                     "0"},
   };
-  genContainerTypeHandler(features, used, FuncGen::GetOiArrayContainerInfo(),
-                          arrayParams, code);
+  genContainerTypeHandler(
+      features, used, FuncGen::GetOiArrayContainerInfo(), arrayParams, code);
 }
 
 }  // namespace
@@ -1098,12 +1098,17 @@ void CodeGen::addTypeHandlers(const TypeGraph& typeGraph, std::string& code) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
       genClassTypeHandler(*c, code);
     } else if (const auto* con = dynamic_cast<const Container*>(&t)) {
-      genContainerTypeHandler(config_.features, definedContainers_,
-                              con->containerInfo_, con->templateParams, code);
+      genContainerTypeHandler(config_.features,
+                              definedContainers_,
+                              con->containerInfo_,
+                              con->templateParams,
+                              code);
     } else if (const auto* cap = dynamic_cast<const CaptureKeys*>(&t)) {
-      genContainerTypeHandler(config_.features, definedContainers_,
+      genContainerTypeHandler(config_.features,
+                              definedContainers_,
                               cap->containerInfo(),
-                              cap->container().templateParams, code);
+                              cap->container().templateParams,
+                              code);
     }
   }
 }
@@ -1301,7 +1306,8 @@ void CodeGen::generate(
   }
 
   if (config_.features[Feature::TreeBuilderV2]) {
-    FuncGen::DefineTreeBuilderInstructions(code, typeName,
+    FuncGen::DefineTreeBuilderInstructions(code,
+                                           typeName,
                                            calculateExclusiveSize(rootType),
                                            enumerateTypeNames(rootType));
   } else if (config_.features[Feature::TreeBuilderTypeChecking]) {
diff --git a/oi/Config.cpp b/oi/Config.cpp
index a752a7e..157b279 100644
--- a/oi/Config.cpp
+++ b/oi/Config.cpp
@@ -215,8 +215,8 @@ std::optional<FeatureSet> processConfigFile(
             auto* members = (*captureKeys)["members"].as_array();
             if (!members) {
               generatorConfig.keysToCapture.push_back(
-                  OICodeGen::Config::KeyToCapture{type->value_or(""), "*",
-                                                  false});
+                  OICodeGen::Config::KeyToCapture{
+                      type->value_or(""), "*", false});
             } else {
               for (auto&& member : *members) {
                 generatorConfig.keysToCapture.push_back(
@@ -226,8 +226,8 @@ std::optional<FeatureSet> processConfigFile(
             }
           } else if (topLevel) {
             generatorConfig.keysToCapture.push_back(
-                OICodeGen::Config::KeyToCapture{std::nullopt, std::nullopt,
-                                                true});
+                OICodeGen::Config::KeyToCapture{
+                    std::nullopt, std::nullopt, true});
           }
         }
       }
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index a5703be..068da95 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -323,6 +323,8 @@ ContainerInfo::ContainerInfo(std::string typeName_,
       matcher(getMatcher(typeName)),
       ctype(ctype_),
       header(std::move(header_)),
-      codegen(Codegen{"// DummyDecl %1%\n", "// DummyFunc %1%\n",
-                      "// DummyHandler %1%\n", "// DummyFunc\n"}) {
+      codegen(Codegen{"// DummyDecl %1%\n",
+                      "// DummyFunc %1%\n",
+                      "// DummyHandler %1%\n",
+                      "// DummyFunc\n"}) {
 }
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index dc494f9..9d6805b 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -800,7 +800,8 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
 }
 
 ContainerInfo FuncGen::GetOiArrayContainerInfo() {
-  ContainerInfo oiArray{"OIArray", UNKNOWN_TYPE,
+  ContainerInfo oiArray{"OIArray",
+                        UNKNOWN_TYPE,
                         "cstdint"};  // TODO: remove the need for a dummy header
 
   oiArray.codegen.handler = R"(
diff --git a/oi/Metrics.cpp b/oi/Metrics.cpp
index 975affa..e3fa6a4 100644
--- a/oi/Metrics.cpp
+++ b/oi/Metrics.cpp
@@ -130,8 +130,11 @@ void Tracing::stop() {
 
   std::lock_guard<std::mutex> guard{static_.mutex};
   // Can't use emplace_back() because of old clang++ on CI
-  static_.traces.push_back({getNextIndex(), std::move(traceName),
-                            duration.count(), rssBeforeBytes, rssAfterBytes});
+  static_.traces.push_back({getNextIndex(),
+                            std::move(traceName),
+                            duration.count(),
+                            rssBeforeBytes,
+                            rssAfterBytes});
 }
 
 void Tracing::saveTraces(const std::filesystem::path& output) {
diff --git a/oi/OICodeGen.cpp b/oi/OICodeGen.cpp
index 0cbbfc9..ee18429 100644
--- a/oi/OICodeGen.cpp
+++ b/oi/OICodeGen.cpp
@@ -204,8 +204,8 @@ std::string OICodeGen::preProcessUniquePtr(drgn_type* type, std::string name) {
     } else if (typeSize == cFunctionDeleterSize) {
       name.replace(begin, end - begin + 1, "void(*)(" + typeName + "*)");
     } else if (typeSize == stdFunctionDeleterSize) {
-      name.replace(begin, end - begin + 1,
-                   "std::function<void (" + typeName + "*)>");
+      name.replace(
+          begin, end - begin + 1, "std::function<void (" + typeName + "*)>");
     } else {
       LOG(ERROR) << "Unhandled case, unique_ptr size: " << typeSize;
     }
@@ -540,8 +540,8 @@ bool OICodeGen::buildNameInt(drgn_type* type,
     templateParamsStrings.push_back(templateParamName);
   }
 
-  replaceTemplateParameters(type, templateParams, templateParamsStrings,
-                            nameWithoutTemplate);
+  replaceTemplateParameters(
+      type, templateParams, templateParamsStrings, nameWithoutTemplate);
 
   outName = nameWithoutTemplate;
   for (size_t i = 0; i < templateParamsStrings.size(); ++i) {
@@ -765,8 +765,9 @@ bool OICodeGen::enumerateTemplateParamIdxs(drgn_type* type,
 
   auto& templateTypes =
       containerTypeMapDrgn
-          .emplace(type, std::pair(std::ref(containerInfo),
-                                   std::vector<drgn_qualified_type>()))
+          .emplace(type,
+                   std::pair(std::ref(containerInfo),
+                             std::vector<drgn_qualified_type>()))
           .first->second.second;
 
   for (auto i : paramIdxs) {
@@ -1721,8 +1722,8 @@ void OICodeGen::enumerateDescendants(drgn_type* type, drgn_type* baseType) {
 
   // TODO this list may end up containing duplicates
   const auto& children = it->second;
-  descendantClasses[baseType].insert(descendantClasses[baseType].end(),
-                                     children.begin(), children.end());
+  descendantClasses[baseType].insert(
+      descendantClasses[baseType].end(), children.begin(), children.end());
 
   for (const auto& child : children) {
     enumerateDescendants(child, baseType);
@@ -1878,7 +1879,8 @@ void OICodeGen::memberTransformName(
     sortedTypes.push_back(e.first);
   }
 
-  std::sort(sortedTypes.begin(), sortedTypes.end(),
+  std::sort(sortedTypes.begin(),
+            sortedTypes.end(),
             [](const std::string& first, const std::string& second) {
               return first.size() > second.size();
             });
@@ -2212,8 +2214,13 @@ bool OICodeGen::generateStructDef(drgn_type* e, std::string& code) {
 
   uint64_t offsetBits = 0;
   std::unordered_map<std::string, int> memberNames;
-  if (!generateStructMembers(e, memberNames, generatedMembers, offsetBits,
-                             paddingInfo, violatesAlignmentRequirement, 0)) {
+  if (!generateStructMembers(e,
+                             memberNames,
+                             generatedMembers,
+                             offsetBits,
+                             paddingInfo,
+                             violatesAlignmentRequirement,
+                             0)) {
     return false;
   }
 
@@ -2471,8 +2478,8 @@ std::optional<uint64_t> OICodeGen::generateMember(
       currOffsetBits = 0;
       VLOG(1) << "Member size: " << memberSize;
     } else {
-      addSizeComment(feature(Feature::GenPaddingStats), code, currOffsetBits,
-                     memberSize);
+      addSizeComment(
+          feature(Feature::GenPaddingStats), code, currOffsetBits, memberSize);
       currOffsetBits = currOffsetBits + memberSize;
     }
 
@@ -2499,8 +2506,12 @@ bool OICodeGen::generateParent(
   auto* underlyingType = drgn_utils::underlyingType(p);
   uint64_t offsetBits = 0;
 
-  if (!generateStructMembers(underlyingType, memberNames, code, offsetBits,
-                             paddingInfo, violatesAlignmentRequirement,
+  if (!generateStructMembers(underlyingType,
+                             memberNames,
+                             code,
+                             offsetBits,
+                             paddingInfo,
+                             violatesAlignmentRequirement,
                              offsetToNextMember)) {
     return false;
   }
@@ -2657,8 +2668,11 @@ bool OICodeGen::generateStructMembers(
       size_t prevOffsetBits = currOffsetBits;
 
       auto newCurrOffsetBits =
-          generateMember(members[memberIndex], memberNames, currOffsetBits,
-                         code, drgn_type_kind(e) == DRGN_TYPE_UNION);
+          generateMember(members[memberIndex],
+                         memberNames,
+                         currOffsetBits,
+                         code,
+                         drgn_type_kind(e) == DRGN_TYPE_UNION);
 
       if (!newCurrOffsetBits.has_value()) {
         return false;
@@ -2770,8 +2784,11 @@ bool OICodeGen::generateStructMembers(
         }
       }
 
-      if (!generateParent(parentClasses[e][parentIndex].type, memberNames,
-                          currOffsetBits, code, offsetToNextMember)) {
+      if (!generateParent(parentClasses[e][parentIndex].type,
+                          memberNames,
+                          currOffsetBits,
+                          code,
+                          offsetToNextMember)) {
         return false;
       }
 
@@ -2869,8 +2886,8 @@ bool OICodeGen::generateStructDefs(std::string& code) {
       if (parentClassesCopy.find(e) != parentClassesCopy.end()) {
         auto& parents = parentClassesCopy[e];
         for (auto& p : parents) {
-          auto it2 = std::find(structDefTypeCopy.begin(),
-                               structDefTypeCopy.end(), p.type);
+          auto it2 = std::find(
+              structDefTypeCopy.begin(), structDefTypeCopy.end(), p.type);
           if (it2 != structDefTypeCopy.cend()) {
             skip = true;
             break;
@@ -2886,7 +2903,8 @@ bool OICodeGen::generateStructDefs(std::string& code) {
 
           if (underlyingType != e) {
             auto it2 = std::find(structDefTypeCopy.begin(),
-                                 structDefTypeCopy.end(), underlyingType);
+                                 structDefTypeCopy.end(),
+                                 underlyingType);
             if (it2 != structDefTypeCopy.cend()) {
               skip = true;
               break;
@@ -3204,8 +3222,8 @@ bool OICodeGen::generateJitCode(std::string& code) {
   std::string functionsCode;
   functionsCode.append("namespace OIInternal {\nnamespace {\n");
   functionsCode.append("// functions -----\n");
-  if (!funcGen.DeclareGetSizeFuncs(functionsCode, containerTypesFuncDef,
-                                   config.features)) {
+  if (!funcGen.DeclareGetSizeFuncs(
+          functionsCode, containerTypesFuncDef, config.features)) {
     LOG(ERROR) << "declaring get size for containers failed";
     return false;
   }
@@ -3235,8 +3253,8 @@ bool OICodeGen::generateJitCode(std::string& code) {
   funcGen.DeclareEncodeData(functionsCode);
   funcGen.DeclareEncodeDataSize(functionsCode);
 
-  if (!funcGen.DefineGetSizeFuncs(functionsCode, containerTypesFuncDef,
-                                  config.features)) {
+  if (!funcGen.DefineGetSizeFuncs(
+          functionsCode, containerTypesFuncDef, config.features)) {
     LOG(ERROR) << "defining get size for containers failed";
     return false;
   }
@@ -3289,8 +3307,8 @@ bool OICodeGen::generateJitCode(std::string& code) {
     bool generateOffsetAsserts =
         (drgn_type_kind(structType) != DRGN_TYPE_UNION);
 
-    if (!addStaticAssertsForType(structType, generateOffsetAsserts,
-                                 functionsCode)) {
+    if (!addStaticAssertsForType(
+            structType, generateOffsetAsserts, functionsCode)) {
       return false;
     }
   }
@@ -3328,11 +3346,11 @@ bool OICodeGen::generateJitCode(std::string& code) {
     if (rootTypeStr.starts_with("unique_ptr") ||
         rootTypeStr.starts_with("LowPtr") ||
         rootTypeStr.starts_with("shared_ptr")) {
-      funcGen.DefineTopLevelGetSizeSmartPtr(functionsCode, rawTypeName,
-                                            config.features);
+      funcGen.DefineTopLevelGetSizeSmartPtr(
+          functionsCode, rawTypeName, config.features);
     } else {
-      funcGen.DefineTopLevelGetSizeRef(functionsCode, rawTypeName,
-                                       config.features);
+      funcGen.DefineTopLevelGetSizeRef(
+          functionsCode, rawTypeName, config.features);
     }
   }
 
@@ -3593,7 +3611,9 @@ bool OICodeGen::staticAssertMemberOffsets(
     // Operate on the underlying type for typedefs
     return staticAssertMemberOffsets(struct_name,
                                      drgn_utils::underlyingType(struct_type),
-                                     assert_str, memberNames, base_offset);
+                                     assert_str,
+                                     memberNames,
+                                     base_offset);
   }
 
   const auto* tag = drgn_type_tag(struct_type);
@@ -3606,8 +3626,11 @@ bool OICodeGen::staticAssertMemberOffsets(
     // Recurse into parents to find inherited members
     for (const auto& parent : parentClasses[struct_type]) {
       auto parentOffset = base_offset + parent.bit_offset / CHAR_BIT;
-      if (!staticAssertMemberOffsets(struct_name, parent.type, assert_str,
-                                     memberNames, parentOffset)) {
+      if (!staticAssertMemberOffsets(struct_name,
+                                     parent.type,
+                                     assert_str,
+                                     memberNames,
+                                     parentOffset)) {
         return false;
       }
     }
diff --git a/oi/OICompiler.cpp b/oi/OICompiler.cpp
index d036b33..9d3aa79 100644
--- a/oi/OICompiler.cpp
+++ b/oi/OICompiler.cpp
@@ -79,8 +79,8 @@ static struct LLVMInitializer {
     llvm::InitializeNativeTargetAsmPrinter();
     llvm::InitializeNativeTargetDisassembler();
 
-    disassemblerContext = LLVMCreateDisasm("x86_64-pc-linux", nullptr, 0,
-                                           nullptr, symbolLookupCallback);
+    disassemblerContext = LLVMCreateDisasm(
+        "x86_64-pc-linux", nullptr, 0, nullptr, symbolLookupCallback);
     if (!disassemblerContext) {
       throw std::runtime_error("Failed to initialize disassemblerContext");
     }
@@ -106,10 +106,13 @@ OICompiler::Disassembler::operator()() {
     return std::nullopt;
   }
 
-  size_t instSize = LLVMDisasmInstruction(
-      disassemblerContext, const_cast<uint8_t*>(std::data(funcText)),
-      std::size(funcText), 0, std::data(disassemblyBuffer),
-      std::size(disassemblyBuffer));
+  size_t instSize =
+      LLVMDisasmInstruction(disassemblerContext,
+                            const_cast<uint8_t*>(std::data(funcText)),
+                            std::size(funcText),
+                            0,
+                            std::data(disassemblyBuffer),
+                            std::size(disassemblyBuffer));
   if (instSize == 0) {
     return std::nullopt;
   }
@@ -150,7 +153,9 @@ class OIMemoryManager : public RTDyldMemoryManager {
       std::error_code errorCode;
       auto mem = sys::Memory::allocateMappedMemory(
           alignTo(totalSize + 256, 256),  // Extra to fit paddings added below
-          nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, errorCode);
+          nullptr,
+          sys::Memory::MF_READ | sys::Memory::MF_WRITE,
+          errorCode);
 
       /*
        * It looks like report_fatal_error() calls exit() by default. If it's
@@ -509,7 +514,9 @@ bool OICompiler::compile(const std::string& code,
 
   for (const auto& path : config.userHeaderPaths) {
     headerSearchOptions.AddPath(
-        path.c_str(), clang::frontend::IncludeDirGroup::IndexHeaderMap, false,
+        path.c_str(),
+        clang::frontend::IncludeDirGroup::IndexHeaderMap,
+        false,
         false);
   }
 
@@ -518,22 +525,25 @@ bool OICompiler::compile(const std::string& code,
         path.c_str(), clang::frontend::IncludeDirGroup::System, false, false);
   }
 
-  static const auto syntheticHeaders = std::array<
-      std::pair<Feature, std::pair<std::string_view, std::string>>, 7>{{
-      {Feature::TypedDataSegment, {headers::oi_types_st_h, "oi/types/st.h"}},
-      {Feature::TreeBuilderTypeChecking,
-       {headers::oi_types_dy_h, "oi/types/dy.h"}},
-      {Feature::TreeBuilderV2,
-       {headers::oi_exporters_inst_h, "oi/exporters/inst.h"}},
-      {Feature::TreeBuilderV2,
-       {headers::oi_exporters_ParsedData_h, "oi/exporters/ParsedData.h"}},
-      {Feature::TreeBuilderV2,
-       {headers::oi_result_Element_h, "oi/result/Element.h"}},
-      {Feature::Library,
-       {headers::oi_IntrospectionResult_h, "oi/IntrospectionResult.h"}},
-      {Feature::Library,
-       {headers::oi_IntrospectionResult_inl_h, "oi/IntrospectionResult-inl.h"}},
-  }};
+  static const auto syntheticHeaders =
+      std::array<std::pair<Feature, std::pair<std::string_view, std::string>>,
+                 7>{{
+          {Feature::TypedDataSegment,
+           {headers::oi_types_st_h, "oi/types/st.h"}},
+          {Feature::TreeBuilderTypeChecking,
+           {headers::oi_types_dy_h, "oi/types/dy.h"}},
+          {Feature::TreeBuilderV2,
+           {headers::oi_exporters_inst_h, "oi/exporters/inst.h"}},
+          {Feature::TreeBuilderV2,
+           {headers::oi_exporters_ParsedData_h, "oi/exporters/ParsedData.h"}},
+          {Feature::TreeBuilderV2,
+           {headers::oi_result_Element_h, "oi/result/Element.h"}},
+          {Feature::Library,
+           {headers::oi_IntrospectionResult_h, "oi/IntrospectionResult.h"}},
+          {Feature::Library,
+           {headers::oi_IntrospectionResult_inl_h,
+            "oi/IntrospectionResult-inl.h"}},
+      }};
   for (const auto& [k, v] : syntheticHeaders) {
     if (!config.features[k])
       continue;
@@ -546,7 +556,9 @@ bool OICompiler::compile(const std::string& code,
     if (config.features[k]) {
       headerSearchOptions.AddPath(
           "/synthetic/headers",
-          clang::frontend::IncludeDirGroup::IndexHeaderMap, false, false);
+          clang::frontend::IncludeDirGroup::IndexHeaderMap,
+          false,
+          false);
       break;
     }
   }
@@ -657,9 +669,10 @@ std::optional<OICompiler::RelocResult> OICompiler::applyRelocs(
               << currentRelocAddress + offset;
     }
 
-    res.relocInfos.push_back(RelocResult::RelocInfo{
-        (uintptr_t)slab.memBlock.base(), currentRelocAddress,
-        slab.memBlock.allocatedSize()});
+    res.relocInfos.push_back(
+        RelocResult::RelocInfo{(uintptr_t)slab.memBlock.base(),
+                               currentRelocAddress,
+                               slab.memBlock.allocatedSize()});
     currentRelocAddress =
         alignTo(currentRelocAddress + slab.memBlock.allocatedSize(), 128);
     res.newBaseRelocAddr = currentRelocAddress;
diff --git a/oi/OID.cpp b/oi/OID.cpp
index e0f810c..6403b61 100644
--- a/oi/OID.cpp
+++ b/oi/OID.cpp
@@ -108,51 +108,90 @@ enum ExitStatus {
 
 constexpr static OIOpts opts{
     OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit"},
-    OIOpt{'p', "pid", required_argument, "<pid>",
-          "Target process to attach to"},
-    OIOpt{'c', "config-file", required_argument, nullptr,
-          "</path/to/oid.toml>"},
-    OIOpt{'x', "data-buf-size", required_argument, "<bytes>",
+    OIOpt{
+        'p', "pid", required_argument, "<pid>", "Target process to attach to"},
+    OIOpt{
+        'c', "config-file", required_argument, nullptr, "</path/to/oid.toml>"},
+    OIOpt{'x',
+          "data-buf-size",
+          required_argument,
+          "<bytes>",
           "Size of data segment (default:1MB)\n"
           "Accepts multiplicative suffix: K, M, G, T, P, E"},
-    OIOpt{'d', "debug-level", required_argument, "<level>",
+    OIOpt{'d',
+          "debug-level",
+          required_argument,
+          "<level>",
           "Verbose level for logging"},
-    OIOpt{'r', "remove-mappings", no_argument, nullptr,
+    OIOpt{'r',
+          "remove-mappings",
+          no_argument,
+          nullptr,
           "Remove oid mappings from target process"},
     OIOpt{'s', "script", required_argument, nullptr, "</path/to/script.oid>"},
     OIOpt{'S', "script-source", required_argument, nullptr, "type:symbol:arg"},
-    OIOpt{'t', "timeout", required_argument, "<seconds>",
+    OIOpt{'t',
+          "timeout",
+          required_argument,
+          "<seconds>",
           "How long to probe the target process for"},
-    OIOpt{'k', "custom-code-file", required_argument, nullptr,
+    OIOpt{'k',
+          "custom-code-file",
+          required_argument,
+          nullptr,
           "</path/to/code.cpp>\n"
           "Use your own CPP file instead of CodeGen"},
-    OIOpt{'e', "compile-and-exit", no_argument, nullptr,
+    OIOpt{'e',
+          "compile-and-exit",
+          no_argument,
+          nullptr,
           "Compile only then exit"},
-    OIOpt{'o', "cache-path", required_argument, "<path>",
+    OIOpt{'o',
+          "cache-path",
+          required_argument,
+          "<path>",
           "Enable caching using the provided directory"},
-    OIOpt{'u', "cache-remote", required_argument, nullptr,
+    OIOpt{'u',
+          "cache-remote",
+          required_argument,
+          nullptr,
           "Enable upload/download of cache files\n"
           "Pick from {both,upload,download}"},
-    OIOpt{'i', "debug-path", required_argument, nullptr,
+    OIOpt{'i',
+          "debug-path",
+          required_argument,
+          nullptr,
           "</path/to/binary>\n"
           "Run oid on a executable with debug infos instead of a running "
           "process"},
     // Optional arguments are pretty nasty - it will only work as
     // "--dump-json=PATH" and not "--dump-json PATH". Try and make this take a
     // required argument at a later point
-    OIOpt{'J', "dump-json", optional_argument, "[oid_out.json]",
+    OIOpt{'J',
+          "dump-json",
+          optional_argument,
+          "[oid_out.json]",
           "File to dump the results to, as JSON\n"
           "(in addition to the default RocksDB output)"},
     OIOpt{
-        'B', "dump-data-segment", no_argument, nullptr,
+        'B',
+        "dump-data-segment",
+        no_argument,
+        nullptr,
         "Dump the data segment's content, before TreeBuilder processes it\n"
         "Each argument gets its own dump file: 'dataseg.<oid-pid>.<arg>.dump'"},
     OIOpt{'a', "log-all-structs", no_argument, nullptr, "Log all structures"},
-    OIOpt{'m', "mode", required_argument, "MODE",
+    OIOpt{'m',
+          "mode",
+          required_argument,
+          "MODE",
           "Allows to specify a mode of operation/group of settings"},
-    OIOpt{'f', "enable-feature", required_argument, "FEATURE",
-          "Enable feature"},
-    OIOpt{'F', "disable-feature", required_argument, "FEATURE",
+    OIOpt{
+        'f', "enable-feature", required_argument, "FEATURE", "Enable feature"},
+    OIOpt{'F',
+          "disable-feature",
+          required_argument,
+          "FEATURE",
           "Disable feature"},
 };
 
@@ -287,11 +326,11 @@ static ExitStatus::ExitStatus runScript(
 
   std::shared_ptr<OIDebugger> oid;  // share oid with the global signal handler
   if (oidConfig.pid != 0) {
-    oid = std::make_shared<OIDebugger>(oidConfig.pid, codeGenConfig,
-                                       compilerConfig, tbConfig);
+    oid = std::make_shared<OIDebugger>(
+        oidConfig.pid, codeGenConfig, compilerConfig, tbConfig);
   } else {
-    oid = std::make_shared<OIDebugger>(oidConfig.debugInfoFile, codeGenConfig,
-                                       compilerConfig, tbConfig);
+    oid = std::make_shared<OIDebugger>(
+        oidConfig.debugInfoFile, codeGenConfig, compilerConfig, tbConfig);
   }
   weak_oid = oid;  // set the weak_ptr for signal handlers
 
@@ -496,8 +535,8 @@ int main(int argc, char* argv[]) {
   google::SetStderrLogging(google::WARNING);
 
   int c = 0;
-  while ((c = getopt_long(argc, argv, opts.shortOpts(), opts.longOpts(),
-                          nullptr)) != -1) {
+  while ((c = getopt_long(
+              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
     switch (c) {
       case 'F':
         [[fallthrough]];
@@ -675,8 +714,8 @@ int main(int argc, char* argv[]) {
       .jsonPath = jsonPath,
   };
 
-  auto featureSet = config::processConfigFiles(oidConfig.configFiles, features,
-                                               compilerConfig, codeGenConfig);
+  auto featureSet = config::processConfigFiles(
+      oidConfig.configFiles, features, compilerConfig, codeGenConfig);
   if (!featureSet) {
     return ExitStatus::UsageError;
   }
@@ -690,15 +729,15 @@ int main(int argc, char* argv[]) {
       return ExitStatus::FileNotFoundError;
     }
     std::ifstream script(scriptFile);
-    auto status = runScript(scriptFile, script, oidConfig, codeGenConfig,
-                            compilerConfig, tbConfig);
+    auto status = runScript(
+        scriptFile, script, oidConfig, codeGenConfig, compilerConfig, tbConfig);
     if (status != ExitStatus::Success) {
       return status;
     }
   } else if (!scriptSource.empty()) {
     std::istringstream script(scriptSource);
-    auto status = runScript(scriptFile, script, oidConfig, codeGenConfig,
-                            compilerConfig, tbConfig);
+    auto status = runScript(
+        scriptFile, script, oidConfig, codeGenConfig, compilerConfig, tbConfig);
     if (status != ExitStatus::Success) {
       return status;
     }
diff --git a/oi/OIDebugger.cpp b/oi/OIDebugger.cpp
index e1e38f9..9c98fc6 100644
--- a/oi/OIDebugger.cpp
+++ b/oi/OIDebugger.cpp
@@ -67,8 +67,9 @@ namespace oi::detail {
 constexpr int oidMagicId = 0x01DE8;
 
 bool OIDebugger::isGlobalDataProbeEnabled(void) const {
-  return std::any_of(cbegin(pdata), cend(pdata),
-                     [](const auto& r) { return r.type == "global"; });
+  return std::any_of(cbegin(pdata), cend(pdata), [](const auto& r) {
+    return r.type == "global";
+  });
 }
 
 bool OIDebugger::parseScript(std::istream& script) {
@@ -214,7 +215,8 @@ bool OIDebugger::setupLogFile(void) {
      * The memory will be re-used anyway and the path will get overwritten.
      */
     if (!writeTargetMemory((void*)logFilePath.c_str(),
-                           (void*)segConfig.textSegBase, logFilePathLen)) {
+                           (void*)segConfig.textSegBase,
+                           logFilePathLen)) {
       LOG(ERROR) << "Failed to write Log File's path into target process";
       return false;
     }
@@ -381,9 +383,15 @@ void OIDebugger::deleteSegmentConfig(bool deleteSegConfigFile) {
  */
 std::string OIDebugger::taskStateToString(OIDebugger::StatusType status) {
   /* Must reflect the order of OIDebugger::StatusType enum */
-  static const std::array enumMapping{"SLEEP",   "TRACED", "RUNNING",
-                                      "ZOMBIE",  "DEAD",   "DISK SLEEP",
-                                      "STOPPED", "OTHER",  "BAD"};
+  static const std::array enumMapping{"SLEEP",
+                                      "TRACED",
+                                      "RUNNING",
+                                      "ZOMBIE",
+                                      "DEAD",
+                                      "DISK SLEEP",
+                                      "STOPPED",
+                                      "OTHER",
+                                      "BAD"};
 
   return enumMapping[static_cast<int>(status)];
 }
@@ -583,7 +591,8 @@ bool OIDebugger::locateObjectsAddresses(const trapInfo& tInfo,
     }
 
     VLOG(4) << "Entry: arg addr: " << std::hex << *addr;
-    if (!writeTargetMemory((void*)(&addr.value()), (void*)remoteObjAddr->second,
+    if (!writeTargetMemory((void*)(&addr.value()),
+                           (void*)remoteObjAddr->second,
                            sizeof(*addr))) {
       LOG(ERROR) << "Entry: writeTargetMemory remoteObjAddr failed!";
       ret = false;
@@ -882,8 +891,8 @@ bool OIDebugger::processGlobal(const std::string& varName) {
     return false;
   }
 
-  if (!writeTargetMemory((void*)&addr, (void*)remoteObjAddr->second,
-                         sizeof(addr))) {
+  if (!writeTargetMemory(
+          (void*)&addr, (void*)remoteObjAddr->second, sizeof(addr))) {
     LOG(ERROR) << "processGlobal: writeTargetMemory remoteObjAddr failed!";
   }
 
@@ -1023,7 +1032,9 @@ OIDebugger::processTrapRet OIDebugger::processTrap(pid_t pid,
         VLOG(4) << "child was stopped with: " << WSTOPSIG(tstatus);
       }
 
-      ptrace(PTRACE_SETOPTIONS, childPid, NULL,
+      ptrace(PTRACE_SETOPTIONS,
+             childPid,
+             NULL,
              PTRACE_O_TRACESYSGOOD | PTRACE_O_TRACEFORK | PTRACE_O_TRACECLONE |
                  PTRACE_O_TRACEVFORK);
 
@@ -1230,8 +1241,9 @@ OIDebugger::processTrapRet OIDebugger::processTrap(pid_t pid,
 
 std::optional<std::vector<uintptr_t>> OIDebugger::findRetLocs(FuncDesc& fd) {
   size_t maxSize = std::accumulate(
-      fd.ranges.begin(), fd.ranges.end(), size_t(0),
-      [](auto currMax, auto& r) { return std::max(currMax, r.size()); });
+      fd.ranges.begin(), fd.ranges.end(), size_t(0), [](auto currMax, auto& r) {
+        return std::max(currMax, r.size());
+      });
 
   std::vector<uintptr_t> retLocs;
   std::vector<std::byte> text(maxSize);
@@ -1384,8 +1396,9 @@ bool OIDebugger::functionPatch(const prequest& req) {
 
   /* 1. Locate all TRAP points and create a corresponding empty trapInfo in
    * tiVec */
-  bool hasArg = std::any_of(begin(req.args), end(req.args),
-                            [](auto& arg) { return arg != "retval"; });
+  bool hasArg = std::any_of(begin(req.args), end(req.args), [](auto& arg) {
+    return arg != "retval";
+  });
 
   if (req.type == "entry" || hasArg) {
     trapType tType =
@@ -1414,8 +1427,8 @@ bool OIDebugger::functionPatch(const prequest& req) {
     }
 
     for (auto addr : *retLocs) {
-      tiVec.push_back(std::make_shared<trapInfo>(OID_TRAP_VECT_RET, addr,
-                                                 segConfig.textSegBase));
+      tiVec.push_back(std::make_shared<trapInfo>(
+          OID_TRAP_VECT_RET, addr, segConfig.textSegBase));
     }
   }
 
@@ -1433,8 +1446,12 @@ bool OIDebugger::functionPatch(const prequest& req) {
   }
 
   errno = 0;
-  auto readBytes = process_vm_readv(traceePid, localIov.data(), localIov.size(),
-                                    remoteIov.data(), remoteIov.size(), 0);
+  auto readBytes = process_vm_readv(traceePid,
+                                    localIov.data(),
+                                    localIov.size(),
+                                    remoteIov.data(),
+                                    remoteIov.size(),
+                                    0);
   if (readBytes < 0) {
     LOG(ERROR) << "Failed to get original instructions: " << strerror(errno);
     return false;
@@ -1489,9 +1506,12 @@ bool OIDebugger::functionPatch(const prequest& req) {
 
   /* 4. Save the original instructions in our Replay Instruction buffer */
   errno = 0;
-  auto writtenBytes =
-      process_vm_writev(traceePid, localIov.data(), localIov.size(),
-                        remoteIov.data(), remoteIov.size(), 0);
+  auto writtenBytes = process_vm_writev(traceePid,
+                                        localIov.data(),
+                                        localIov.size(),
+                                        remoteIov.data(),
+                                        remoteIov.size(),
+                                        0);
   if (writtenBytes < 0) {
     LOG(ERROR) << "Failed to save original instructions: " << strerror(errno);
     return false;
@@ -1588,8 +1608,12 @@ std::optional<typename Sys::RetType> OIDebugger::remoteSyscall(Args... _args) {
      * x86-64        rdi   rsi   rdx   r10   r8    r9    -
      */
     const std::array<unsigned long long*, 6> argToReg = {
-        &newregs.rdi, &newregs.rsi, &newregs.rdx,
-        &newregs.r10, &newregs.r8,  &newregs.r9,
+        &newregs.rdi,
+        &newregs.rsi,
+        &newregs.rdx,
+        &newregs.r10,
+        &newregs.r8,
+        &newregs.r9,
     };
 
     unsigned long long args[] = {(unsigned long long)_args...};
@@ -1682,15 +1706,19 @@ bool OIDebugger::setupSegment(SegType seg) {
   std::optional<void*> segAddr;
   if (seg == SegType::text) {
     segAddr =
-        remoteSyscall<SysMmap>(nullptr, textSegSize,  // addr & size
+        remoteSyscall<SysMmap>(nullptr,
+                               textSegSize,  // addr & size
                                PROT_READ | PROT_WRITE | PROT_EXEC,  // prot
                                MAP_PRIVATE | MAP_ANONYMOUS,         // flags
-                               -1, 0);  // fd & offset
+                               -1,
+                               0);  // fd & offset
   } else {
-    segAddr = remoteSyscall<SysMmap>(nullptr, dataSegSize,        // addr & size
+    segAddr = remoteSyscall<SysMmap>(nullptr,
+                                     dataSegSize,                 // addr & size
                                      PROT_READ | PROT_WRITE,      // prot
                                      MAP_SHARED | MAP_ANONYMOUS,  // flags
-                                     -1, 0);                      // fd & offset
+                                     -1,
+                                     0);  // fd & offset
   }
 
   if (!segAddr.has_value()) {
@@ -1835,13 +1863,16 @@ bool OIDebugger::removeTrap(pid_t pid, const trapInfo& t) {
         break;
       }
 
-      memcpy(repatchedBytes.data() + off, it->second->patchedTextBytes,
+      memcpy(repatchedBytes.data() + off,
+             it->second->patchedTextBytes,
              windowSize - off);
     }
   }
 
   VLOG(4) << "removeTrap removing int3 at " << std::hex << t.trapAddr;
-  if (ptrace(PTRACE_POKETEXT, (!pid ? traceePid : pid), t.trapAddr,
+  if (ptrace(PTRACE_POKETEXT,
+             (!pid ? traceePid : pid),
+             t.trapAddr,
              *reinterpret_cast<uintptr_t*>(repatchedBytes.data())) < 0) {
     LOG(ERROR) << "Execute: Couldn't poke text: " << strerror(errno);
     return false;
@@ -2120,8 +2151,8 @@ bool OIDebugger::writePrologue(
 
   assert(off <= prologueLength);
 
-  return writeTargetMemory(&newInsts, (void*)segConfig.textSegBase,
-                           prologueLength);
+  return writeTargetMemory(
+      &newInsts, (void*)segConfig.textSegBase, prologueLength);
 }
 
 /*
@@ -2161,7 +2192,9 @@ bool OIDebugger::compileCode() {
         } else {
           LOG(INFO) << "Attempting to get cache request from gobs";
           ObjectIntrospection::GobsService::requestCache(
-              procpath, std::string(buf, buf_size), req.toString(),
+              procpath,
+              std::string(buf, buf_size),
+              req.toString(),
               generatorConfig.toOptions());
         }
 #endif
@@ -2236,7 +2269,8 @@ bool OIDebugger::compileCode() {
       }
 
       const auto& [rootType, typeHierarchy, paddingInfo] = typeInfos.at(req);
-      cache.store(req, OICache::Entity::TypeHierarchy,
+      cache.store(req,
+                  OICache::Entity::TypeHierarchy,
                   std::make_pair(rootType, typeHierarchy));
       cache.store(req, OICache::Entity::PaddingInfo, paddingInfo);
     }
@@ -2256,8 +2290,8 @@ bool OIDebugger::compileCode() {
     for (const auto& o : objectFiles) {
       VLOG(2) << "  * " << o;
     }
-    auto relocRes = compiler.applyRelocs(segConfig.jitCodeStart, objectFiles,
-                                         syntheticSymbols);
+    auto relocRes = compiler.applyRelocs(
+        segConfig.jitCodeStart, objectFiles, syntheticSymbols);
     if (!relocRes.has_value()) {
       LOG(ERROR) << "Failed to relocate object code";
       return false;
@@ -2292,7 +2326,8 @@ bool OIDebugger::compileCode() {
       return false;
     }
 
-    if (!writeTargetMemory(&dataSegSize, (void*)syntheticSymbols["dataSize"],
+    if (!writeTargetMemory(&dataSegSize,
+                           (void*)syntheticSymbols["dataSize"],
                            sizeof(dataSegSize))) {
       LOG(ERROR) << "Failed to write dataSegSize in probe's dataSize";
       return false;
@@ -2307,8 +2342,8 @@ bool OIDebugger::compileCode() {
 
     int logFile =
         generatorConfig.features[Feature::JitLogging] ? segConfig.logFile : 0;
-    if (!writeTargetMemory(&logFile, (void*)syntheticSymbols["logFile"],
-                           sizeof(logFile))) {
+    if (!writeTargetMemory(
+            &logFile, (void*)syntheticSymbols["logFile"], sizeof(logFile))) {
       LOG(ERROR) << "Failed to write logFile in probe's cookieValue";
       return false;
     }
@@ -2329,8 +2364,9 @@ void OIDebugger::restoreState(void) {
    * Ensure we don't have any trap in the target process still active.
    */
   const size_t activeTrapsCount = std::count_if(
-      activeTraps.cbegin(), activeTraps.cend(),
-      [](const auto& t) { return t.second->trapKind != OID_TRAP_JITCODERET; });
+      activeTraps.cbegin(), activeTraps.cend(), [](const auto& t) {
+        return t.second->trapKind != OID_TRAP_JITCODERET;
+      });
   VLOG(1) << "Active traps still within the target process: "
           << activeTrapsCount;
   assert(activeTrapsCount == 0);
@@ -2562,7 +2598,9 @@ bool OIDebugger::targetAttach() {
          * here (note: ptrace(2) overloads the ESRCH return but with a seize
          * I think it can only mean one thing).
          */
-        if (ptrace(PTRACE_SEIZE, pid, NULL,
+        if (ptrace(PTRACE_SEIZE,
+                   pid,
+                   NULL,
                    PTRACE_O_TRACECLONE | PTRACE_O_TRACEFORK |
                        PTRACE_O_TRACEVFORK | PTRACE_O_TRACEEXIT) < 0) {
           LOG(ERROR) << "Couldn't seize thread " << pid
@@ -2750,9 +2788,11 @@ bool OIDebugger::decodeTargetData(const DataHeader& dataHeader,
 static bool dumpDataSegment(const irequest& req,
                             const std::vector<uint64_t>& dataSeg) {
   char dumpPath[PATH_MAX] = {0};
-  auto dumpPathSize =
-      snprintf(dumpPath, sizeof(dumpPath), "/tmp/dataseg.%d.%s.dump", getpid(),
-               req.arg.c_str());
+  auto dumpPathSize = snprintf(dumpPath,
+                               sizeof(dumpPath),
+                               "/tmp/dataseg.%d.%s.dump",
+                               getpid(),
+                               req.arg.c_str());
   if (dumpPathSize < 0 || (size_t)dumpPathSize > sizeof(dumpPath)) {
     LOG(ERROR) << "Failed to generate data-segment path";
     return false;
@@ -2781,7 +2821,8 @@ bool OIDebugger::processTargetData() {
 
   std::vector<std::byte> buf{dataSegSize};
   if (!readTargetMemory(reinterpret_cast<void*>(segConfig.dataSegBase),
-                        buf.data(), dataSegSize)) {
+                        buf.data(),
+                        dataSegSize)) {
     LOG(ERROR) << "Failed to read data segment from target process";
     return false;
   }
@@ -2838,8 +2879,8 @@ bool OIDebugger::processTargetData() {
     }
 
     try {
-      typeTree.build(outVec, rootType.varName, rootType.type.type,
-                     typeHierarchy);
+      typeTree.build(
+          outVec, rootType.varName, rootType.type.type, typeHierarchy);
     } catch (std::exception& e) {
       LOG(ERROR) << "Failed to run TreeBuilder for " << req.arg;
       LOG(ERROR) << e.what();
@@ -2901,7 +2942,8 @@ std::optional<std::string> OIDebugger::generateCode(const irequest& req) {
   typeInfos.emplace(
       req,
       std::make_tuple(RootInfo{rootInfo.varName, codegen->getRootType()},
-                      codegen->getTypeHierarchy(), codegen->getPaddingInfo()));
+                      codegen->getTypeHierarchy(),
+                      codegen->getPaddingInfo()));
 
   if (generatorConfig.features[Feature::TypeGraph]) {
     CodeGen codegen2{generatorConfig, *symbols};
diff --git a/oi/OIDebugger.h b/oi/OIDebugger.h
index 934e9f3..804d2cd 100644
--- a/oi/OIDebugger.h
+++ b/oi/OIDebugger.h
@@ -113,7 +113,8 @@ class OIDebugger {
     return std::all_of(
         std::begin(pdata), std::end(pdata), [this](const auto& req) {
           return std::all_of(
-              std::begin(req.args), std::end(req.args),
+              std::begin(req.args),
+              std::end(req.args),
               [this, &req](const auto& arg) {
                 return cache.upload(irequest{req.type, req.func, arg});
               });
@@ -123,7 +124,8 @@ class OIDebugger {
     return std::all_of(
         std::begin(pdata), std::end(pdata), [this](const auto& req) {
           return std::all_of(
-              std::begin(req.args), std::end(req.args),
+              std::begin(req.args),
+              std::end(req.args),
               [this, &req](const auto& arg) {
                 return cache.download(irequest{req.type, req.func, arg});
               });
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index d98e7b8..6c4590f 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -169,9 +169,12 @@ int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
 
   {
     std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(drgn_program_load_debug_info(
-            prog.get(), std::data(objectPaths), std::size(objectPaths), false,
-            false))) {
+    if (auto err = drgnplusplus::error(
+            drgn_program_load_debug_info(prog.get(),
+                                         std::data(objectPaths),
+                                         std::size(objectPaths),
+                                         false,
+                                         false))) {
       LOG(ERROR) << "error loading debug info program: " << err;
       throw err;
     }
@@ -193,8 +196,8 @@ int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
   OICompiler::Config compilerConfig{};
   compilerConfig.usePIC = pic;
 
-  auto features = config::processConfigFiles(configFilePaths, featuresMap,
-                                             compilerConfig, generatorConfig);
+  auto features = config::processConfigFiles(
+      configFilePaths, featuresMap, compilerConfig, generatorConfig);
   if (!features) {
     LOG(ERROR) << "failed to process config file";
     return -1;
@@ -204,8 +207,8 @@ int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
 
   size_t failures = 0;
   for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(generatorConfig, compilerConfig, type,
-                                   linkageName, symbols);
+    if (auto obj = generateForType(
+            generatorConfig, compilerConfig, type, linkageName, symbols);
         !obj.empty()) {
       std::cout << obj.string() << std::endl;
     } else {
diff --git a/oi/OILibraryImpl.cpp b/oi/OILibraryImpl.cpp
index bade97c..0f33356 100644
--- a/oi/OILibraryImpl.cpp
+++ b/oi/OILibraryImpl.cpp
@@ -40,8 +40,12 @@ drgn_qualified_type getTypeFromAtomicHole(drgn_program* prog, void* hole);
 }  // namespace
 
 OILibraryImpl::LocalTextSegment::LocalTextSegment(size_t size) {
-  void* base = mmap(NULL, size, PROT_EXEC | PROT_READ | PROT_WRITE,
-                    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  void* base = mmap(NULL,
+                    size,
+                    PROT_EXEC | PROT_READ | PROT_WRITE,
+                    MAP_PRIVATE | MAP_ANONYMOUS,
+                    -1,
+                    0);
   if (base == MAP_FAILED)
     throw std::runtime_error(std::string("segment map failed: ") +
                              std::strerror(errno));
@@ -93,9 +97,10 @@ std::pair<void*, const exporters::inst::Inst&> OILibraryImpl::init() {
 }
 
 void OILibraryImpl::processConfigFile() {
-  auto features =
-      config::processConfigFiles(opts_.configFilePaths, requestedFeatures_,
-                                 compilerConfig_, generatorConfig_);
+  auto features = config::processConfigFiles(opts_.configFilePaths,
+                                             requestedFeatures_,
+                                             compilerConfig_,
+                                             generatorConfig_);
   if (!features)
     throw std::runtime_error("failed to process configuration");
 
@@ -164,7 +169,8 @@ std::pair<void*, const exporters::inst::Inst&> OILibraryImpl::compileCode() {
 
   for (const auto& [baseAddr, relocAddr, size] : segments)
     std::memcpy(reinterpret_cast<void*>(relocAddr),
-                reinterpret_cast<void*>(baseAddr), size);
+                reinterpret_cast<void*>(baseAddr),
+                size);
 
   textSeg.release();  // don't munmap() the region containing the code
   return {fp, *ty};
diff --git a/oi/OIParser.h b/oi/OIParser.h
index 2c4e242..b0f94fd 100644
--- a/oi/OIParser.h
+++ b/oi/OIParser.h
@@ -80,7 +80,8 @@ class ParseData {
  public:
   void addReq(std::string type, std::string func, std::list<std::string> args) {
     // Convert the args std::list into a more efficient std::vector
-    reqs.emplace_back(std::move(type), std::move(func),
+    reqs.emplace_back(std::move(type),
+                      std::move(func),
                       std::vector(std::make_move_iterator(args.begin()),
                                   std::make_move_iterator(args.end())));
   }
diff --git a/oi/PaddingHunter.cpp b/oi/PaddingHunter.cpp
index 41606a3..28b6511 100644
--- a/oi/PaddingHunter.cpp
+++ b/oi/PaddingHunter.cpp
@@ -49,7 +49,8 @@ void PaddingHunter::outputPaddingInfo() {
 
   paddingStatsFile << "Total Saving Opportunity: " << sum << "\n\n\n";
 
-  std::sort(paddedStructsVec.begin(), paddedStructsVec.end(),
+  std::sort(paddedStructsVec.begin(),
+            paddedStructsVec.end(),
             [](const std::pair<std::string, PaddingInfo>& left,
                const std::pair<std::string, PaddingInfo>& right) {
               return left.second.instancesCnt * left.second.savingSize >
diff --git a/oi/SymbolService.cpp b/oi/SymbolService.cpp
index d34ce79..15d19d2 100644
--- a/oi/SymbolService.cpp
+++ b/oi/SymbolService.cpp
@@ -66,7 +66,13 @@ static bool LoadExecutableAddressRange(
     while (std::getline(f, line)) {
       if (sscanf(line.c_str(),
                  "%" PRIx64 "-%" PRIx64 " %s %" PRIx64 " %x:%x %" PRIu64 " %n",
-                 &start, &end, perm, &offset, &dmajor, &dminor, &inode,
+                 &start,
+                 &end,
+                 perm,
+                 &offset,
+                 &dmajor,
+                 &dminor,
+                 &inode,
                  &nread) < 7 ||
           nread <= 0) {
         return false;
@@ -92,7 +98,9 @@ static bool isExecutableAddr(
 
   // Find the smallest exeAddrs range where addr < range.end
   auto it = std::upper_bound(
-      begin(exeAddrs), end(exeAddrs), std::make_pair(addr, addr),
+      begin(exeAddrs),
+      end(exeAddrs),
+      std::make_pair(addr, addr),
       [](const auto& r1, const auto& r2) { return r1.second < r2.second; });
 
   return it != end(exeAddrs) && addr >= it->first;
@@ -246,8 +254,9 @@ bool SymbolService::loadModulesFromPath(const fs::path& targetPath) {
 
   Dwarf_Addr start = 0;
   Dwarf_Addr end = 0;
-  if (dwfl_module_info(mod, nullptr, &start, &end, nullptr, nullptr, nullptr,
-                       nullptr) == nullptr) {
+  if (dwfl_module_info(
+          mod, nullptr, &start, &end, nullptr, nullptr, nullptr, nullptr) ==
+      nullptr) {
     LOG(ERROR) << "dwfl_module_info: " << dwfl_errmsg(dwfl_errno());
     return false;
   }
@@ -442,8 +451,8 @@ struct drgn_program* SymbolService::getDrgnProgram() {
       auto executable = fs::read_symlink(
           "/proc/" + std::to_string(std::get<pid_t>(target)) + "/exe");
       const auto* executableCStr = executable.c_str();
-      if (auto* err = drgn_program_load_debug_info(prog, &executableCStr, 1,
-                                                   false, false)) {
+      if (auto* err = drgn_program_load_debug_info(
+              prog, &executableCStr, 1, false, false)) {
         LOG(ERROR) << "Error loading debug info: " << err->message;
         return nullptr;
       }
@@ -769,8 +778,11 @@ std::shared_ptr<GlobalDesc> SymbolService::findGlobalDesc(
     drgn_object_deinit(&globalObj);
   };
 
-  if (auto* err = drgn_program_find_object(drgnProg, global.c_str(), nullptr,
-                                           DRGN_FIND_OBJECT_ANY, &globalObj)) {
+  if (auto* err = drgn_program_find_object(drgnProg,
+                                           global.c_str(),
+                                           nullptr,
+                                           DRGN_FIND_OBJECT_ANY,
+                                           &globalObj)) {
     LOG(ERROR) << "Failed to lookup global variable '" << global
                << "': " << err->code << " " << err->message;
 
diff --git a/oi/TreeBuilder.cpp b/oi/TreeBuilder.cpp
index 9f934e4..954b293 100644
--- a/oi/TreeBuilder.cpp
+++ b/oi/TreeBuilder.cpp
@@ -455,7 +455,8 @@ TreeBuilder::Node TreeBuilder::process(NodeID id, Variable variable) {
             auto childID = nextNodeID++;
             auto child = process(childID, Variable{entry->second, "", ""});
             node.children = {childID, childID + 1};
-            setSize(node, child.staticSize + child.dynamicSize,
+            setSize(node,
+                    child.staticSize + child.dynamicSize,
                     child.staticSize + child.dynamicSize);
           }
         }
@@ -477,8 +478,8 @@ TreeBuilder::Node TreeBuilder::process(NodeID id, Variable variable) {
           auto childID = nextNodeID++;
           auto child = process(childID, Variable{entry->second, "", ""});
           node.children = {childID, childID + 1};
-          setSize(node, child.dynamicSize,
-                  child.dynamicSize + child.staticSize);
+          setSize(
+              node, child.dynamicSize, child.dynamicSize + child.staticSize);
         }
       } break;
       case DRGN_TYPE_CLASS:
@@ -532,10 +533,12 @@ TreeBuilder::Node TreeBuilder::process(NodeID id, Variable variable) {
               }
             }
             const auto& member = members[i];
-            auto child =
-                process(childID++,
-                        Variable{member.type, member.member_name,
-                                 member.member_name, isset, member.isStubbed});
+            auto child = process(childID++,
+                                 Variable{member.type,
+                                          member.member_name,
+                                          member.member_name,
+                                          isset,
+                                          member.isStubbed});
             node.dynamicSize += child.dynamicSize;
             memberSizes += child.dynamicSize + child.staticSize;
           }
@@ -582,8 +585,8 @@ void TreeBuilder::processContainer(const Variable& variable, Node& node) {
       arrayElementType = drgn_type_type(variable.type).type;
       numElems = drgn_type_length(variable.type);
     } else {
-      drgn_utils::getDrgnArrayElementType(variable.type, &arrayElementType,
-                                          numElems);
+      drgn_utils::getDrgnArrayElementType(
+          variable.type, &arrayElementType, numElems);
     }
     assert(numElems > 0);
     elementTypes.push_back(
@@ -675,10 +678,11 @@ void TreeBuilder::processContainer(const Variable& variable, Node& node) {
       // elementTypes is only populated with the underlying container type for
       // container adapters
       auto containerType = elementTypes[0];
-      auto child = process(
-          childID++, {.type = containerType.type,
-                      .name = "",
-                      .typePath = drgnTypeToName(containerType.type) + "[]"});
+      auto child =
+          process(childID++,
+                  {.type = containerType.type,
+                   .name = "",
+                   .typePath = drgnTypeToName(containerType.type) + "[]"});
 
       setSize(node, child.dynamicSize, child.dynamicSize + child.staticSize);
       node.containerStats = child.containerStats;
@@ -715,10 +719,11 @@ void TreeBuilder::processContainer(const Variable& variable, Node& node) {
         auto childID = node.children->first;
 
         auto elementType = elementTypes[index];
-        auto child = process(
-            childID++, {.type = elementType.type,
-                        .name = "",
-                        .typePath = drgnTypeToName(elementType.type) + "[]"});
+        auto child =
+            process(childID++,
+                    {.type = elementType.type,
+                     .name = "",
+                     .typePath = drgnTypeToName(elementType.type) + "[]"});
 
         setSize(node, child.dynamicSize, child.dynamicSize + child.staticSize);
       }
@@ -879,8 +884,9 @@ void TreeBuilder::processContainer(const Variable& variable, Node& node) {
         "uninitialized data in the target process");
   }
   if (std::ranges::all_of(
-          elementTypes.cbegin(), elementTypes.cend(),
-          [this](auto& type) { return isPrimitive(type.type); })) {
+          elementTypes.cbegin(), elementTypes.cend(), [this](auto& type) {
+            return isPrimitive(type.type);
+          })) {
     VLOG(1)
         << "Container [" << node.id
         << "] contains only primitive types, skipping processing its members";
@@ -900,10 +906,10 @@ void TreeBuilder::processContainer(const Variable& variable, Node& node) {
   uint64_t memberSizes = 0;
   for (size_t i = 0; i < containerStats.length; i++) {
     for (auto& type : elementTypes) {
-      auto child =
-          process(childID++, {.type = type.type,
-                              .name = "",
-                              .typePath = drgnTypeToName(type.type) + "[]"});
+      auto child = process(childID++,
+                           {.type = type.type,
+                            .name = "",
+                            .typePath = drgnTypeToName(type.type) + "[]"});
       node.dynamicSize += child.dynamicSize;
       memberSizes += child.dynamicSize + child.staticSize;
     }
diff --git a/oi/exporters/Json.cpp b/oi/exporters/Json.cpp
index ad2590e..8a4d33c 100644
--- a/oi/exporters/Json.cpp
+++ b/oi/exporters/Json.cpp
@@ -87,8 +87,8 @@ void Json::print(IntrospectionResult::const_iterator& it,
     out_ << (pretty_ ? ",\n" : ",") << indent;
 
     out_ << tab << "\"typeNames\"" << space << ':' << space;
-    printStringList(out_, it->type_names.begin(), it->type_names.end(),
-                    pretty_);
+    printStringList(
+        out_, it->type_names.begin(), it->type_names.end(), pretty_);
     out_ << ',' << endl << indent;
 
     out_ << tab << "\"staticSize\":" << space << it->static_size << ',' << endl
diff --git a/oi/exporters/test/TypeCheckingWalkerTest.cpp b/oi/exporters/test/TypeCheckingWalkerTest.cpp
index 3383c82..89b3fef 100644
--- a/oi/exporters/test/TypeCheckingWalkerTest.cpp
+++ b/oi/exporters/test/TypeCheckingWalkerTest.cpp
@@ -145,8 +145,8 @@ TEST(TypeCheckingWalker, TestListEmpty) {
 TEST(TypeCheckingWalker, TestListSome) {
   // ASSIGN
   std::array<uint64_t, 3> listElements{59942, 44126, 64525};
-  std::vector<uint64_t> data{listElements.size(), listElements[0],
-                             listElements[1], listElements[2]};
+  std::vector<uint64_t> data{
+      listElements.size(), listElements[0], listElements[1], listElements[2]};
 
   types::dy::VarInt varint;
   types::dy::List rootType{varint};
diff --git a/oi/type_graph/AddPadding.cpp b/oi/type_graph/AddPadding.cpp
index 79fbd53..b4474d5 100644
--- a/oi/type_graph/AddPadding.cpp
+++ b/oi/type_graph/AddPadding.cpp
@@ -123,8 +123,8 @@ void AddPadding::addPadding(uint64_t paddingStartBits,
 
   if (paddingBits % 8 != 0) {
     // Pad with a bitfield up to the next byte
-    paddedMembers.emplace_back(primitive, MemberPrefix, paddingStartBits,
-                               paddingBits % 8);
+    paddedMembers.emplace_back(
+        primitive, MemberPrefix, paddingStartBits, paddingBits % 8);
   }
 
   uint64_t paddingBytes = paddingBits / 8;
diff --git a/oi/type_graph/DrgnParser.cpp b/oi/type_graph/DrgnParser.cpp
index 42a5033..0116edf 100644
--- a/oi/type_graph/DrgnParser.cpp
+++ b/oi/type_graph/DrgnParser.cpp
@@ -180,8 +180,8 @@ Class& DrgnParser::enumerateClass(struct drgn_type* type) {
                             std::to_string(drgn_type_kind(type))};
   }
 
-  auto& c = makeType<Class>(type, kind, std::move(name), std::move(fqName),
-                            size, virtuality);
+  auto& c = makeType<Class>(
+      type, kind, std::move(name), std::move(fqName), size, virtuality);
 
   enumerateClassTemplateParams(type, c.templateParams);
   enumerateClassParents(type, c.parents);
@@ -402,7 +402,8 @@ void DrgnParser::enumerateClassFunctions(struct drgn_type* type,
     drgn_qualified_type t{};
     if (auto* err = drgn_member_function_type(&drgn_functions[i], &t)) {
       warnForDrgnError(
-          type, "Error looking up member function (" + std::to_string(i) + ")",
+          type,
+          "Error looking up member function (" + std::to_string(i) + ")",
           err);
       continue;
     }
diff --git a/oi/type_graph/Flattener.cpp b/oi/type_graph/Flattener.cpp
index f8ee3af..57e3748 100644
--- a/oi/type_graph/Flattener.cpp
+++ b/oi/type_graph/Flattener.cpp
@@ -54,8 +54,8 @@ void flattenParent(const Parent& parent,
     }
   } else if (auto* parentContainer = dynamic_cast<Container*>(&parentType)) {
     // Create a new member to represent this parent container
-    flattenedMembers.emplace_back(*parentContainer, Flattener::ParentPrefix,
-                                  parent.bitOffset);
+    flattenedMembers.emplace_back(
+        *parentContainer, Flattener::ParentPrefix, parent.bitOffset);
   } else if (auto* parentPrimitive = dynamic_cast<Incomplete*>(&parentType)) {
     // Bad DWARF can lead to us seeing incomplete parent types. Just ignore
     // these as there is nothing we can do to recover the missing info.
@@ -151,7 +151,8 @@ void Flattener::visit(Class& c) {
   for (const auto& parent : c.parents) {
     Type& parentType = stripTypedefs(parent.type());
     if (Class* parentClass = dynamic_cast<Class*>(&parentType)) {
-      c.functions.insert(c.functions.end(), parentClass->functions.begin(),
+      c.functions.insert(c.functions.end(),
+                         parentClass->functions.begin(),
                          parentClass->functions.end());
     }
   }
@@ -201,8 +202,8 @@ void Flattener::visit(Class& c) {
   // Pull in children from flattened children
   // This may result in duplicates, but that shouldn't be a big deal
   for (const Class& child : c.children) {
-    c.children.insert(c.children.end(), child.children.begin(),
-                      child.children.end());
+    c.children.insert(
+        c.children.end(), child.children.begin(), child.children.end());
   }
 }
 
diff --git a/oi/type_graph/TypeIdentifier.cpp b/oi/type_graph/TypeIdentifier.cpp
index 0fd8696..17fa43a 100644
--- a/oi/type_graph/TypeIdentifier.cpp
+++ b/oi/type_graph/TypeIdentifier.cpp
@@ -112,8 +112,8 @@ void TypeIdentifier::visit(Container& c) {
             typeToAllocate, size, param.type().align(), allocator->name());
         c.templateParams[i] = dummy;
       } else {
-        auto& dummy = typeGraph_.makeType<Dummy>(size, param.type().align(),
-                                                 param.type().name());
+        auto& dummy = typeGraph_.makeType<Dummy>(
+            size, param.type().align(), param.type().name());
         c.templateParams[i] = dummy;
       }
     }
diff --git a/test/TypeGraphParser.cpp b/test/TypeGraphParser.cpp
index 1eca163..7b6f425 100644
--- a/test/TypeGraphParser.cpp
+++ b/test/TypeGraphParser.cpp
@@ -310,8 +310,8 @@ Type& TypeGraphParser::parseType(std::string_view& input, size_t rootIndent) {
     auto size = parseNumericAttribute(line, nodeTypeName, "size: ");
     std::string inputName{*tryParseInputName(line)};
     auto& typeToAlloc = parseType(input, indent + 2);
-    type = &typeGraph_.makeType<DummyAllocator>(id, typeToAlloc, size, 0,
-                                                inputName);
+    type = &typeGraph_.makeType<DummyAllocator>(
+        id, typeToAlloc, size, 0, inputName);
   } else {
     throw TypeGraphParserError{"Unsupported node type: " +
                                std::string{nodeTypeName}};
diff --git a/test/integration/runner_common.cpp b/test/integration/runner_common.cpp
index fcdbc66..c86bd59 100644
--- a/test/integration/runner_common.cpp
+++ b/test/integration/runner_common.cpp
@@ -35,17 +35,35 @@ std::vector<std::string> global_oid_args{};
 
 constexpr static OIOpts cliOpts{
     OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit"},
-    OIOpt{'p', "preserve", no_argument, nullptr,
+    OIOpt{'p',
+          "preserve",
+          no_argument,
+          nullptr,
           "Do not clean up files generated by OID after tests are finished"},
-    OIOpt{'P', "preserve-on-failure", no_argument, nullptr,
+    OIOpt{'P',
+          "preserve-on-failure",
+          no_argument,
+          nullptr,
           "Do not clean up files generated by OID for failed tests"},
-    OIOpt{'v', "verbose", no_argument, nullptr,
+    OIOpt{'v',
+          "verbose",
+          no_argument,
+          nullptr,
           "Verbose output. Show OID's stdout and stderr on test failure"},
-    OIOpt{'f', "force", no_argument, nullptr,
+    OIOpt{'f',
+          "force",
+          no_argument,
+          nullptr,
           "Force running tests, even if they are marked as skipped"},
-    OIOpt{'x', "oid", required_argument, nullptr,
+    OIOpt{'x',
+          "oid",
+          required_argument,
+          nullptr,
           "Path to OID executable to test"},
-    OIOpt{'\0', "enable-feature", required_argument, nullptr,
+    OIOpt{'\0',
+          "enable-feature",
+          required_argument,
+          nullptr,
           "Enable extra OID feature."},
 };
 
@@ -64,8 +82,9 @@ int main(int argc, char* argv[]) {
   }
 
   int c;
-  while ((c = getopt_long(argc, argv, cliOpts.shortOpts(), cliOpts.longOpts(),
-                          nullptr)) != -1) {
+  while ((c = getopt_long(
+              argc, argv, cliOpts.shortOpts(), cliOpts.longOpts(), nullptr)) !=
+         -1) {
     switch (c) {
       case 'p':
         preserve = true;
@@ -193,8 +212,8 @@ OidProc OidIntegration::runOidOnProcess(OidOpts opts,
 
   // The arguments are appended in ascending order of precedence (low -> high)
   std::vector<std::string> oid_args;
-  oid_args.insert(oid_args.end(), global_oid_args.begin(),
-                  global_oid_args.end());
+  oid_args.insert(
+      oid_args.end(), global_oid_args.begin(), global_oid_args.end());
   oid_args.insert(oid_args.end(), extra_args.begin(), extra_args.end());
   oid_args.insert(oid_args.end(), default_args.begin(), default_args.end());
 
@@ -269,7 +288,9 @@ OidProc OidIntegration::runOidOnProcess(OidOpts opts,
 
   return OidProc{
       .target = Proc{opts.ctx, std::move(targetProcess), {}, {}},
-      .oid = Proc{opts.ctx, std::move(oidProcess), std::move(std_out),
+      .oid = Proc{opts.ctx,
+                  std::move(oidProcess),
+                  std::move(std_out),
                   std::move(std_err)},
   };
 }
@@ -298,8 +319,10 @@ void IntegrationBase::compare_json(const bpt::ptree& expected_json,
     for (auto e_it = expected_json.begin(), a_it = actual_json.begin();
          e_it != expected_json.end() && a_it != actual_json.end();
          e_it++, a_it++) {
-      compare_json(e_it->second, a_it->second,
-                   full_key + "[" + std::to_string(i) + "]", expect_eq);
+      compare_json(e_it->second,
+                   a_it->second,
+                   full_key + "[" + std::to_string(i) + "]",
+                   expect_eq);
       i++;
     }
     return;
@@ -417,6 +440,8 @@ Proc OilIntegration::runOilTarget(OilOpts opts,
       opts.ctx);
   // clang-format on
 
-  return Proc{opts.ctx, std::move(targetProcess), std::move(std_out),
+  return Proc{opts.ctx,
+              std::move(targetProcess),
+              std::move(std_out),
               std::move(std_err)};
 }
diff --git a/test/test_add_children.cpp b/test/test_add_children.cpp
index 5d563cb..aca046e 100644
--- a/test/test_add_children.cpp
+++ b/test/test_add_children.cpp
@@ -66,7 +66,8 @@ TEST_F(AddChildrenTest, InheritanceStatic) {
 }
 
 TEST_F(AddChildrenTest, InheritancePolymorphic) {
-  testMultiCompilerGlob("oid_test_case_inheritance_polymorphic_a_as_a", R"(
+  testMultiCompilerGlob("oid_test_case_inheritance_polymorphic_a_as_a",
+                        R"(
 [1]  Pointer
 [0]    Class: A (size: 16)
          Member: _vptr$A (offset: 0)
diff --git a/test/test_add_padding.cpp b/test/test_add_padding.cpp
index 21e47ef..293a965 100644
--- a/test/test_add_padding.cpp
+++ b/test/test_add_padding.cpp
@@ -7,7 +7,8 @@
 using namespace type_graph;
 
 TEST(AddPaddingTest, BetweenMembers) {
-  test(AddPadding::createPass(), R"(
+  test(AddPadding::createPass(),
+       R"(
 [0] Class: MyClass (size: 16)
       Member: n1 (offset: 0)
         Primitive: int8_t
@@ -27,7 +28,8 @@ TEST(AddPaddingTest, BetweenMembers) {
 }
 
 TEST(AddPaddingTest, AtBeginning) {
-  test(AddPadding::createPass(), R"(
+  test(AddPadding::createPass(),
+       R"(
 [0] Struct: MyStruct (size: 16)
       Member: n1 (offset: 8)
         Primitive: int64_t
@@ -43,7 +45,8 @@ TEST(AddPaddingTest, AtBeginning) {
 }
 
 TEST(AddPaddingTest, AtEnd) {
-  test(AddPadding::createPass(), R"(
+  test(AddPadding::createPass(),
+       R"(
 [0] Struct: MyStruct (size: 16)
       Member: n1 (offset: 0)
         Primitive: int64_t
@@ -63,7 +66,8 @@ TEST(AddPaddingTest, AtEnd) {
 }
 
 TEST(AddPaddingTest, UnionBetweenMembers) {
-  test(AddPadding::createPass(), R"(
+  test(AddPadding::createPass(),
+       R"(
 [0] Union: MyUnion (size: 8)
       Member: n1 (offset: 0)
         Primitive: int64_t
@@ -80,7 +84,8 @@ TEST(AddPaddingTest, UnionBetweenMembers) {
 }
 
 TEST(AddPaddingTest, UnionAtEnd) {
-  test(AddPadding::createPass(), R"(
+  test(AddPadding::createPass(),
+       R"(
 [0] Union: MyUnion (size: 16)
       Member: n1 (offset: 0)
         Primitive: int64_t
@@ -100,7 +105,8 @@ TEST(AddPaddingTest, UnionAtEnd) {
 }
 
 TEST(AddPaddingTest, Bitfields) {
-  test(AddPadding::createPass(), R"(
+  test(AddPadding::createPass(),
+       R"(
 [0] Class: MyClass (size: 16)
       Member: b1 (offset: 0, bitsize: 3)
         Primitive: int64_t
@@ -148,7 +154,8 @@ TEST(AddPaddingTest, EmptyClass) {
 }
 
 TEST(AddPaddingTest, MemberlessClass) {
-  test(AddPadding::createPass(), R"(
+  test(AddPadding::createPass(),
+       R"(
 [0] Class: MyClass (size: 12)
 )",
        R"(
@@ -160,7 +167,8 @@ TEST(AddPaddingTest, MemberlessClass) {
 }
 
 TEST(AddPaddingTest, MemberlessUnion) {
-  test(AddPadding::createPass(), R"(
+  test(AddPadding::createPass(),
+       R"(
 [0] Union: MyUnion (size: 16)
 )",
        R"(
diff --git a/test/test_alignment_calc.cpp b/test/test_alignment_calc.cpp
index c4f9671..5bc4573 100644
--- a/test/test_alignment_calc.cpp
+++ b/test/test_alignment_calc.cpp
@@ -6,7 +6,8 @@
 using namespace type_graph;
 
 TEST(AlignmentCalcTest, PrimitiveMembers) {
-  test(AlignmentCalc::createPass(), R"(
+  test(AlignmentCalc::createPass(),
+       R"(
 [0] Class: MyClass (size: 16)
       Member: n (offset: 0)
         Primitive: int8_t
@@ -23,7 +24,8 @@ TEST(AlignmentCalcTest, PrimitiveMembers) {
 }
 
 TEST(AlignmentCalcTest, StructMembers) {
-  test(AlignmentCalc::createPass(), R"(
+  test(AlignmentCalc::createPass(),
+       R"(
 [0] Class: MyClass (size: 12)
       Member: n (offset: 0)
         Primitive: int8_t
@@ -48,7 +50,8 @@ TEST(AlignmentCalcTest, StructMembers) {
 }
 
 TEST(AlignmentCalcTest, StructInContainer) {
-  test(AlignmentCalc::createPass(), R"(
+  test(AlignmentCalc::createPass(),
+       R"(
 [0] Container: std::vector (size: 8)
       Param
 [1]     Class: MyClass (size: 16)
@@ -69,7 +72,8 @@ TEST(AlignmentCalcTest, StructInContainer) {
 }
 
 TEST(AlignmentCalcTest, PackedMembers) {
-  test(AlignmentCalc::createPass(), R"(
+  test(AlignmentCalc::createPass(),
+       R"(
 [0] Struct: MyStruct (size: 8)
       Member: n1 (offset: 0)
         Primitive: int8_t
@@ -98,7 +102,8 @@ TEST(AlignmentCalcTest, PackedMembers) {
 }
 
 TEST(AlignmentCalcTest, PackedTailPadding) {
-  test(AlignmentCalc::createPass(), R"(
+  test(AlignmentCalc::createPass(),
+       R"(
 [0] Struct: MyStruct (size: 5)
       Member: n1 (offset: 0)
         Primitive: int32_t
@@ -115,7 +120,8 @@ TEST(AlignmentCalcTest, PackedTailPadding) {
 }
 
 TEST(AlignmentCalcTest, RecurseClassParam) {
-  test(AlignmentCalc::createPass(), R"(
+  test(AlignmentCalc::createPass(),
+       R"(
 [0] Class: MyClass (size: 0)
       Param
 [1]     Class: ClassA (size: 16)
@@ -136,7 +142,8 @@ TEST(AlignmentCalcTest, RecurseClassParam) {
 }
 
 TEST(AlignmentCalcTest, RecurseClassParent) {
-  test(AlignmentCalc::createPass(), R"(
+  test(AlignmentCalc::createPass(),
+       R"(
 [0] Class: MyClass (size: 0)
       Parent (offset: 0)
 [1]     Class: ClassA (size: 16)
@@ -157,7 +164,8 @@ TEST(AlignmentCalcTest, RecurseClassParent) {
 }
 
 TEST(AlignmentCalcTest, RecurseClassMember) {
-  test(AlignmentCalc::createPass(), R"(
+  test(AlignmentCalc::createPass(),
+       R"(
 [0] Class: MyClass (size: 0)
       Member: xxx (offset: 0)
 [1]     Class: ClassA (size: 16)
@@ -178,7 +186,8 @@ TEST(AlignmentCalcTest, RecurseClassMember) {
 }
 
 TEST(AlignmentCalcTest, RecurseClassChild) {
-  test(AlignmentCalc::createPass(), R"(
+  test(AlignmentCalc::createPass(),
+       R"(
 [0] Class: MyClass (size: 0)
       Child
 [1]     Class: ClassA (size: 16)
@@ -199,7 +208,8 @@ TEST(AlignmentCalcTest, RecurseClassChild) {
 }
 
 TEST(AlignmentCalcTest, Bitfields) {
-  test(AlignmentCalc::createPass(), R"(
+  test(AlignmentCalc::createPass(),
+       R"(
 [0] Class: MyClass (size: 8)
       Member: a (offset: 0, bitsize: 2)
         Primitive: int8_t
@@ -216,7 +226,8 @@ TEST(AlignmentCalcTest, Bitfields) {
 }
 
 TEST(AlignmentCalcTest, Array) {
-  test(AlignmentCalc::createPass(), R"(
+  test(AlignmentCalc::createPass(),
+       R"(
 [0] Class: MyClass (size: 1)
       Member: a (offset: 0)
 [1]     Array: (length: 1)
@@ -235,7 +246,8 @@ TEST(AlignmentCalcTest, Array) {
 }
 
 TEST(AlignmentCalcTest, Typedef) {
-  test(AlignmentCalc::createPass(), R"(
+  test(AlignmentCalc::createPass(),
+       R"(
 [0] Class: MyClass (size: 1)
       Member: a (offset: 0)
 [1]     Typedef: MyTypedef
diff --git a/test/test_codegen.cpp b/test/test_codegen.cpp
index 87ec437..660c411 100644
--- a/test/test_codegen.cpp
+++ b/test/test_codegen.cpp
@@ -114,7 +114,8 @@ TEST(CodeGenTest, TransformContainerAllocatorParamInParent) {
 TEST(CodeGenTest, RemovedMemberAlignment) {
   OICodeGen::Config config;
   config.membersToStub = {{"MyClass", "b"}};
-  testTransform(config, R"(
+  testTransform(config,
+                R"(
 [0] Class: MyClass (size: 24)
       Member: a (offset: 0)
         Primitive: int8_t
diff --git a/test/test_compiler.cpp b/test/test_compiler.cpp
index 3172fc6..cf9f48d 100644
--- a/test/test_compiler.cpp
+++ b/test/test_compiler.cpp
@@ -50,9 +50,12 @@ TEST(CompilerTest, CompileAndRelocate) {
   EXPECT_TRUE(compiler.compile(code, sourcePath, objectPath));
 
   const size_t relocSlabSize = 4096;
-  void* relocSlab =
-      mmap(nullptr, relocSlabSize, PROT_READ | PROT_WRITE | PROT_EXEC,
-           MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+  void* relocSlab = mmap(nullptr,
+                         relocSlabSize,
+                         PROT_READ | PROT_WRITE | PROT_EXEC,
+                         MAP_ANONYMOUS | MAP_PRIVATE,
+                         -1,
+                         0);
   EXPECT_NE(relocSlab, nullptr);
 
   auto relocResult =
@@ -128,13 +131,16 @@ TEST(CompilerTest, CompileAndRelocateMultipleObjs) {
   EXPECT_TRUE(compiler.compile(codeY, sourceYPath, objectYPath));
 
   const size_t relocSlabSize = 8192;
-  void* relocSlab =
-      mmap(nullptr, relocSlabSize, PROT_READ | PROT_WRITE | PROT_EXEC,
-           MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+  void* relocSlab = mmap(nullptr,
+                         relocSlabSize,
+                         PROT_READ | PROT_WRITE | PROT_EXEC,
+                         MAP_ANONYMOUS | MAP_PRIVATE,
+                         -1,
+                         0);
   EXPECT_NE(relocSlab, nullptr);
 
-  auto relocResult = compiler.applyRelocs((uintptr_t)relocSlab,
-                                          {objectXPath, objectYPath}, {});
+  auto relocResult = compiler.applyRelocs(
+      (uintptr_t)relocSlab, {objectXPath, objectYPath}, {});
   EXPECT_TRUE(relocResult.has_value());
 
   auto& [_, segs, jitSymbols] = relocResult.value();
@@ -254,10 +260,10 @@ TEST(CompilerTest, LocateOpcodes) {
   { /* Large range of differently sized needles */
     const std::array needles = {
         std::vector{0x41_b, 0x54_b}, /* push r12:     1 instance */
-        std::vector{0x48_b, 0x83_b, 0xec_b,
-                    0x18_b},         /* sub rsp,0x18: 1 instance */
-        std::vector(1, 0xe8_b),      /* call:         4 instances */
-        std::vector{0x41_b, 0x5c_b}, /* pop r12:      1 instance */
+        std::vector{
+            0x48_b, 0x83_b, 0xec_b, 0x18_b}, /* sub rsp,0x18: 1 instance */
+        std::vector(1, 0xe8_b),              /* call:         4 instances */
+        std::vector{0x41_b, 0x5c_b},         /* pop r12:      1 instance */
     };
     auto locs = OICompiler::locateOpcodes(insts, needles);
     ASSERT_TRUE(locs.has_value());
diff --git a/test/test_drgn_parser.cpp b/test/test_drgn_parser.cpp
index d5b230d..a5ebe30 100644
--- a/test/test_drgn_parser.cpp
+++ b/test/test_drgn_parser.cpp
@@ -253,7 +253,8 @@ TEST_F(DrgnParserTest, InheritanceMultiple) {
 }
 
 TEST_F(DrgnParserTest, Container) {
-  testMultiCompilerGlob("oid_test_case_std_vector_int_empty", R"(
+  testMultiCompilerGlob("oid_test_case_std_vector_int_empty",
+                        R"(
 [13] Pointer
 [0]    Class: vector<int, std::allocator<int> > (size: 24)
          Param
@@ -340,7 +341,8 @@ TEST_F(DrgnParserTest, EnumNoValues) {
   DrgnParserOptions options{
       .readEnumValues = false,
   };
-  test("oid_test_case_enums_scoped", R"(
+  test("oid_test_case_enums_scoped",
+       R"(
     Enum: ScopedEnum (size: 4)
 )",
        options);
@@ -408,7 +410,8 @@ TEST_F(DrgnParserTest, PointerNoFollow) {
   DrgnParserOptions options{
       .chaseRawPointers = false,
   };
-  test("oid_test_case_pointers_struct_primitive_ptrs", R"(
+  test("oid_test_case_pointers_struct_primitive_ptrs",
+       R"(
 [1] Pointer
 [0]   Struct: PrimitivePtrs (size: 24)
         Member: a (offset: 0)
@@ -582,7 +585,8 @@ TEST_F(DrgnParserTest, MemberAlignment) {
 }
 
 TEST_F(DrgnParserTest, VirtualFunctions) {
-  testMultiCompiler("oid_test_case_inheritance_polymorphic_a_as_a", R"(
+  testMultiCompiler("oid_test_case_inheritance_polymorphic_a_as_a",
+                    R"(
 [1] Pointer
 [0]   Class: A (size: 16)
         Member: _vptr$A (offset: 0)
diff --git a/test/test_enforce_compatibility.cpp b/test/test_enforce_compatibility.cpp
index 89b92d7..2632823 100644
--- a/test/test_enforce_compatibility.cpp
+++ b/test/test_enforce_compatibility.cpp
@@ -6,7 +6,8 @@
 using namespace type_graph;
 
 TEST(EnforceCompatibilityTest, ParentContainers) {
-  test(EnforceCompatibility::createPass(), R"(
+  test(EnforceCompatibility::createPass(),
+       R"(
 [0] Class: MyClass (size: 24)
       Member: __oi_parent (offset: 0)
 [1]     Container: std::vector (size: 24)
@@ -19,7 +20,8 @@ TEST(EnforceCompatibilityTest, ParentContainers) {
 }
 
 TEST(EnforceCompatibilityTest, TypesToStub) {
-  test(EnforceCompatibility::createPass(), R"(
+  test(EnforceCompatibility::createPass(),
+       R"(
 [0] Class: EnumMap (size: 8)
       Member: a (offset: 0)
         Primitive: int32_t
@@ -32,7 +34,8 @@ TEST(EnforceCompatibilityTest, TypesToStub) {
 }
 
 TEST(EnforceCompatibilityTest, VoidPointer) {
-  test(EnforceCompatibility::createPass(), R"(
+  test(EnforceCompatibility::createPass(),
+       R"(
 [0] Class: MyClass (size: 8)
       Member: p (offset: 0)
 [1]     Pointer
diff --git a/test/test_flattener.cpp b/test/test_flattener.cpp
index ae6900b..eab3cf8 100644
--- a/test/test_flattener.cpp
+++ b/test/test_flattener.cpp
@@ -39,7 +39,8 @@ TEST(FlattenerTest, OnlyParents) {
   //     int b;
   //     int c;
   //   };
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassA (size: 8)
       Parent (offset: 0)
 [1]     Class: ClassB (size: 4)
@@ -72,7 +73,8 @@ TEST(FlattenerTest, ParentsFirst) {
   //     int a;
   //   };
 
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassA (size: 12)
       Parent (offset: 0)
 [1]     Class: ClassB (size: 4)
@@ -109,7 +111,8 @@ TEST(FlattenerTest, MembersFirst) {
   //     int c;
   //   };
 
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassA (size: 12)
       Parent (offset: 4)
 [1]     Class: ClassB (size: 4)
@@ -147,7 +150,8 @@ TEST(FlattenerTest, MixedMembersAndParents) {
   //     int c;
   //   };
 
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassA (size: 16)
       Parent (offset: 0)
 [1]     Class: ClassB (size: 4)
@@ -188,7 +192,8 @@ TEST(FlattenerTest, EmptyParent) {
   //     int a2;
   //   };
 
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassA (size: 12)
       Parent (offset: 0)
 [1]     Class: ClassB (size: 0)
@@ -227,7 +232,8 @@ TEST(FlattenerTest, TwoDeep) {
   //     int a;
   //   };
 
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassA (size: 16)
       Parent (offset: 0)
 [1]     Class: ClassB (size: 8)
@@ -271,7 +277,8 @@ TEST(FlattenerTest, DiamondInheritance) {
   //     int a;
   //   };
 
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassA (size: 16)
       Parent (offset: 0)
 [1]     Class: ClassB (size: 8)
@@ -309,7 +316,8 @@ TEST(FlattenerTest, Member) {
   //   class B { int c; int b; };
   //   Class A { int a; B b; };
 
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassA (size: 12)
       Member: a (offset: 0)
         Primitive: int32_t
@@ -345,7 +353,8 @@ TEST(FlattenerTest, MemberOfParent) {
   //   class C { int c; };
   //   class A { int b; C c; int a; };
 
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassA (size: 12)
       Parent (offset: 0)
 [1]     Class: ClassB (size: 8)
@@ -381,7 +390,8 @@ TEST(FlattenerTest, ContainerParam) {
   //   class A { int b; int a; };
   //   std::vector<A, int>
 
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Container: std::vector (size: 24)
       Param
 [1]     Class: ClassA (size: 8)
@@ -413,7 +423,8 @@ TEST(FlattenerTest, Array) {
   //   class A : B { int a; };
   //   A[5]
 
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Array: (length: 5)
 [1]   Class: ClassA (size: 8)
         Parent (offset: 0)
@@ -439,7 +450,8 @@ TEST(FlattenerTest, Typedef) {
   //   class A : B { int a; };
   //   using aliasA = A;
 
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Typedef: aliasA
 [1]   Class: ClassA (size: 8)
         Parent (offset: 0)
@@ -465,7 +477,8 @@ TEST(FlattenerTest, TypedefParent) {
   //   using aliasB = B;
   //   class A : aliasB { int a; };
 
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassA (size: 8)
       Parent (offset: 0)
 [1]     Typedef: aliasB
@@ -502,7 +515,8 @@ TEST(FlattenerTest, Pointer) {
   auto classC = Class{0, Class::Kind::Class, "ClassC", 8};
   classC.members.push_back(Member{ptrA, "a", 0});
 
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassC (size: 8)
       Member: a (offset: 0)
 [1]     Pointer
@@ -539,7 +553,8 @@ TEST(FlattenerTest, PointerCycle) {
   classA.members.push_back(Member{classB, "b", 0});
   classB.members.push_back(Member{ptrA, "a", 0});
 
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassA (size: 69)
       Member: b (offset: 0)
 [1]     Class: ClassB (size: 69)
@@ -565,7 +580,8 @@ TEST(FlattenerTest, Alignment) {
   //   class B { alignas(8) int b; };
   //   class A : B, C { int a; };
 
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassA (size: 12)
       Parent (offset: 0)
 [1]     Class: ClassB (size: 4)
@@ -605,7 +621,8 @@ TEST(FlattenerTest, Functions) {
   classB.functions.push_back(Function{"funcB"});
   classC.functions.push_back(Function{"funcC"});
 
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassA (size: 0)
       Parent (offset: 0)
 [1]     Class: ClassB (size: 0)
@@ -629,7 +646,8 @@ TEST(FlattenerTest, Children) {
   //   class B { int b; };
   //   class A : B, C { };
 
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassB (size: 4)
       Member: b (offset: 0)
         Primitive: int32_t
@@ -664,7 +682,8 @@ TEST(FlattenerTest, ChildrenTwoDeep) {
   //   class B : D { int b; };
   //   class A : B, C { int a; };
 
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassD (size: 4)
       Member: d (offset: 0)
         Primitive: int32_t
@@ -713,7 +732,8 @@ TEST(FlattenerTest, ChildrenTwoDeep) {
 }
 
 TEST(FlattenerTest, ParentContainer) {
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassA (size: 32)
       Parent (offset: 0)
 [1]     Container: std::vector (size: 24)
@@ -734,7 +754,8 @@ TEST(FlattenerTest, ParentContainer) {
 }
 
 TEST(FlattenerTest, ParentTwoContainers) {
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassA (size: 48)
       Parent (offset: 0)
 [1]     Container: std::vector (size: 24)
@@ -755,7 +776,8 @@ TEST(FlattenerTest, ParentTwoContainers) {
 }
 
 TEST(FlattenerTest, ParentClassAndContainer) {
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: ClassA (size: 32)
       Parent (offset: 0)
 [1]     Class: ClassB (size: 4)
@@ -778,7 +800,8 @@ TEST(FlattenerTest, ParentClassAndContainer) {
 }
 
 TEST(FlattenerTest, AllocatorParamInParent) {
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Container: std::map (size: 24)
       Param
         Primitive: int32_t
@@ -836,7 +859,8 @@ TEST(FlattenerTest, AllocatorUnfixableNoParent) {
 
 TEST(FlattenerTest, AllocatorUnfixableParentNotClass) {
   // This could be supported if need-be, we just don't do it yet
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Container: std::vector (size: 24)
       Param
         Primitive: int32_t
@@ -875,7 +899,8 @@ TEST(FlattenerTest, AllocatorUnfixableParentNotClass) {
 }
 
 TEST(FlattenerTest, AllocatorUnfixableParentNoParams) {
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Container: std::vector (size: 24)
       Param
         Primitive: int32_t
@@ -902,7 +927,8 @@ TEST(FlattenerTest, AllocatorUnfixableParentNoParams) {
 }
 
 TEST(FlattenerTest, AllocatorUnfixableParentParamIsValue) {
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Container: std::map (size: 24)
       Param
         Primitive: int32_t
@@ -936,7 +962,8 @@ TEST(FlattenerTest, AllocatorUnfixableParentParamIsValue) {
 }
 
 TEST(FlattenerTest, ClassParam) {
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: MyClass (size: 4)
       Param
 [1]     Class: MyChild (size: 4)
@@ -955,7 +982,8 @@ TEST(FlattenerTest, ClassParam) {
 }
 
 TEST(FlattenerTest, IncompleteParent) {
-  test(Flattener::createPass(), R"(
+  test(Flattener::createPass(),
+       R"(
 [0] Class: MyClass (size: 4)
       Parent (offset: 0)
         Incomplete: [IncompleteParent]
diff --git a/test/test_identify_containers.cpp b/test/test_identify_containers.cpp
index 7a2eab1..4c359eb 100644
--- a/test/test_identify_containers.cpp
+++ b/test/test_identify_containers.cpp
@@ -9,7 +9,8 @@ using namespace type_graph;
 
 namespace {
 void test(std::string_view input, std::string_view expectedAfter) {
-  ::test(IdentifyContainers::createPass(getContainerInfos()), input,
+  ::test(IdentifyContainers::createPass(getContainerInfos()),
+         input,
          expectedAfter);
 }
 };  // namespace
diff --git a/test/test_key_capture.cpp b/test/test_key_capture.cpp
index b676340..deb3699 100644
--- a/test/test_key_capture.cpp
+++ b/test/test_key_capture.cpp
@@ -11,7 +11,8 @@ TEST(KeyCaptureTest, InClass) {
       {"MyClass", "b"},
   };
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos;
-  test(KeyCapture::createPass(keysToCapture, containerInfos), R"(
+  test(KeyCapture::createPass(keysToCapture, containerInfos),
+       R"(
 [0] Class: MyClass (size: 12)
       Member: a (offset: 0)
         Primitive: int32_t
@@ -45,7 +46,8 @@ TEST(KeyCaptureTest, MapInMap) {
       {"MyClass", "a"},
   };
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos;
-  test(KeyCapture::createPass(keysToCapture, containerInfos), R"(
+  test(KeyCapture::createPass(keysToCapture, containerInfos),
+       R"(
 [0] Class: MyClass (size: 12)
       Member: a (offset: 8)
 [1]     Container: std::map (size: 24)
@@ -79,7 +81,8 @@ TEST(KeyCaptureTest, TopLevel) {
       {{}, {}, true},
   };
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos;
-  test(KeyCapture::createPass(keysToCapture, containerInfos), R"(
+  test(KeyCapture::createPass(keysToCapture, containerInfos),
+       R"(
 [0] Container: std::map (size: 24)
       Param
         Primitive: int32_t
diff --git a/test/test_parser.cpp b/test/test_parser.cpp
index b9d5be9..8b10dc5 100644
--- a/test/test_parser.cpp
+++ b/test/test_parser.cpp
@@ -92,7 +92,8 @@ TEST(ParserTest, MangledFunc) {
         "EEESaISF_EERS1_IS8_SaIS8_EERS1_ISB_IS8_dESaISM_EE:arg9");
     EXPECT_EQ(pdata.numReqs(), 1);
     EXPECT_REQ_EQ(
-        pdata.getReq(0), "entry",
+        pdata.getReq(0),
+        "entry",
         "_Z7doStuffR3FooRSt6vectorISt3mapINSt7__cxx1112basic_stringIcSt11char_"
         "traitsIcESaIcEEES8_St4lessIS8_ESaISt4pairIKS8_S8_EEESaISF_EERS1_IS8_"
         "SaIS8_EERS1_ISB_IS8_dESaISM_EE",
diff --git a/test/test_prune.cpp b/test/test_prune.cpp
index 0ab05ae..3ae5d61 100644
--- a/test/test_prune.cpp
+++ b/test/test_prune.cpp
@@ -6,7 +6,8 @@
 using type_graph::Prune;
 
 TEST(PruneTest, PruneClass) {
-  test(Prune::createPass(), R"(
+  test(Prune::createPass(),
+       R"(
 [0] Class: MyClass (size: 8)
       Param
         Primitive: int32_t
@@ -34,7 +35,8 @@ TEST(PruneTest, PruneClass) {
 }
 
 TEST(PruneTest, RecurseClassMember) {
-  test(Prune::createPass(), R"(
+  test(Prune::createPass(),
+       R"(
 [0] Class: MyClass (size: 0)
       Member: xxx (offset: 0)
 [1]     Class: ClassA (size: 12)
@@ -48,7 +50,8 @@ TEST(PruneTest, RecurseClassMember) {
 }
 
 TEST(PruneTest, RecurseClassChild) {
-  test(Prune::createPass(), R"(
+  test(Prune::createPass(),
+       R"(
 [0] Class: MyClass (size: 0)
       Child
 [1]     Class: ClassA (size: 12)
diff --git a/test/test_remove_members.cpp b/test/test_remove_members.cpp
index b77da6c..4d6a641 100644
--- a/test/test_remove_members.cpp
+++ b/test/test_remove_members.cpp
@@ -11,7 +11,8 @@ TEST(RemoveMembersTest, Match) {
       {"ClassA", "b"},
   };
 
-  test(RemoveMembers::createPass(membersToIgnore), R"(
+  test(RemoveMembers::createPass(membersToIgnore),
+       R"(
 [0] Class: ClassA (size: 12)
       Member: a (offset: 0)
 [1]     Class: ClassB (size: 4)
@@ -65,7 +66,8 @@ TEST(RemoveMembersTest, RecurseClassParam) {
   const std::vector<std::pair<std::string, std::string>>& membersToIgnore = {
       {"ClassA", "b"},
   };
-  test(RemoveMembers::createPass(membersToIgnore), R"(
+  test(RemoveMembers::createPass(membersToIgnore),
+       R"(
 [0] Class: MyClass (size: 0)
       Param
 [1]     Class: ClassA (size: 12)
@@ -91,7 +93,8 @@ TEST(RemoveMembersTest, RecurseClassParent) {
   const std::vector<std::pair<std::string, std::string>>& membersToIgnore = {
       {"ClassA", "b"},
   };
-  test(RemoveMembers::createPass(membersToIgnore), R"(
+  test(RemoveMembers::createPass(membersToIgnore),
+       R"(
 [0] Class: MyClass (size: 0)
       Parent (offset: 0)
 [1]     Class: ClassA (size: 12)
@@ -117,7 +120,8 @@ TEST(RemoveMembersTest, RecurseClassMember) {
   const std::vector<std::pair<std::string, std::string>>& membersToIgnore = {
       {"ClassA", "b"},
   };
-  test(RemoveMembers::createPass(membersToIgnore), R"(
+  test(RemoveMembers::createPass(membersToIgnore),
+       R"(
 [0] Class: MyClass (size: 0)
       Member: xxx (offset: 0)
 [1]     Class: ClassA (size: 12)
@@ -143,7 +147,8 @@ TEST(RemoveMembersTest, RecurseClassChild) {
   const std::vector<std::pair<std::string, std::string>>& membersToIgnore = {
       {"ClassA", "b"},
   };
-  test(RemoveMembers::createPass(membersToIgnore), R"(
+  test(RemoveMembers::createPass(membersToIgnore),
+       R"(
 [0] Class: MyClass (size: 0)
       Child
 [1]     Class: ClassA (size: 12)
@@ -166,7 +171,8 @@ TEST(RemoveMembersTest, RecurseClassChild) {
 }
 
 TEST(RemoveMembersTest, Union) {
-  test(RemoveMembers::createPass({}), R"(
+  test(RemoveMembers::createPass({}),
+       R"(
 [0] Union: MyUnion (size: 4)
       Member: a (offset: 0)
         Primitive: int32_t
diff --git a/test/test_remove_top_level_pointer.cpp b/test/test_remove_top_level_pointer.cpp
index 6e6b4f9..119e051 100644
--- a/test/test_remove_top_level_pointer.cpp
+++ b/test/test_remove_top_level_pointer.cpp
@@ -7,7 +7,8 @@
 using namespace type_graph;
 
 TEST(RemoveTopLevelPointerTest, TopLevelPointerRemoved) {
-  test(RemoveTopLevelPointer::createPass(), R"(
+  test(RemoveTopLevelPointer::createPass(),
+       R"(
 [0] Pointer
 [1]   Class: MyClass (size: 4)
         Member: n (offset: 0)
diff --git a/test/test_type_identifier.cpp b/test/test_type_identifier.cpp
index 965da9a..b2b8c8c 100644
--- a/test/test_type_identifier.cpp
+++ b/test/test_type_identifier.cpp
@@ -7,7 +7,8 @@
 using namespace type_graph;
 
 TEST(TypeIdentifierTest, StubbedParam) {
-  test(TypeIdentifier::createPass({}), R"(
+  test(TypeIdentifier::createPass({}),
+       R"(
 [0] Container: std::vector (size: 24)
       Param
         Primitive: int32_t
@@ -30,7 +31,8 @@ TEST(TypeIdentifierTest, StubbedParam) {
 }
 
 TEST(TypeIdentifierTest, Allocator) {
-  test(TypeIdentifier::createPass({}), R"(
+  test(TypeIdentifier::createPass({}),
+       R"(
 [0] Container: std::vector (size: 24)
       Param
         Primitive: int32_t
@@ -56,7 +58,8 @@ TEST(TypeIdentifierTest, Allocator) {
 }
 
 TEST(TypeIdentifierTest, AllocatorSize1) {
-  test(TypeIdentifier::createPass({}), R"(
+  test(TypeIdentifier::createPass({}),
+       R"(
 [0] Container: std::vector (size: 24)
       Param
         Primitive: int32_t
@@ -85,7 +88,8 @@ TEST(TypeIdentifierTest, PassThroughTypes) {
   std::vector<ContainerInfo> passThroughTypes;
   passThroughTypes.emplace_back("std::allocator", DUMMY_TYPE, "memory");
 
-  test(TypeIdentifier::createPass(passThroughTypes), R"(
+  test(TypeIdentifier::createPass(passThroughTypes),
+       R"(
 [0] Container: std::vector (size: 24)
       Param
         Primitive: int32_t
@@ -111,7 +115,8 @@ TEST(TypeIdentifierTest, PassThroughSameType) {
   std::vector<ContainerInfo> passThroughTypes;
   passThroughTypes.emplace_back("std::allocator", DUMMY_TYPE, "memory");
 
-  test(TypeIdentifier::createPass(passThroughTypes), R"(
+  test(TypeIdentifier::createPass(passThroughTypes),
+       R"(
 [0] Container: std::vector (size: 24)
       Param
         Primitive: int32_t
@@ -138,7 +143,8 @@ TEST(TypeIdentifierTest, PassThroughSameType) {
 }
 
 TEST(TypeIdentifierTest, ContainerNotReplaced) {
-  test(TypeIdentifier::createPass({}), R"(
+  test(TypeIdentifier::createPass({}),
+       R"(
 [0] Container: std::vector (size: 24)
       Param
         Primitive: int32_t
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index 6e84dfa..d1e345d 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -31,17 +31,35 @@ using namespace oi::detail;
 
 constexpr static OIOpts opts{
     OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o', "output", required_argument, "<file>",
+    OIOpt{'o',
+          "output",
+          required_argument,
+          "<file>",
           "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c', "config-file", required_argument, "<oid.toml>",
+    OIOpt{'c',
+          "config-file",
+          required_argument,
+          "<oid.toml>",
           "Path to OI configuration file."},
-    OIOpt{'d', "debug-level", required_argument, "<level>",
+    OIOpt{'d',
+          "debug-level",
+          required_argument,
+          "<level>",
           "Verbose level for logging"},
-    OIOpt{'j', "dump-jit", optional_argument, "<jit.cpp>",
+    OIOpt{'j',
+          "dump-jit",
+          optional_argument,
+          "<jit.cpp>",
           "Write generated code to a file (for debugging)."},
-    OIOpt{'e', "exit-code", no_argument, nullptr,
+    OIOpt{'e',
+          "exit-code",
+          no_argument,
+          nullptr,
           "Return a bad exit code if nothing is generated."},
-    OIOpt{'p', "pic", no_argument, nullptr,
+    OIOpt{'p',
+          "pic",
+          no_argument,
+          nullptr,
           "Generate position independent code."},
 };
 
@@ -68,8 +86,8 @@ int main(int argc, char* argv[]) {
   bool pic = false;
 
   int c;
-  while ((c = getopt_long(argc, argv, opts.shortOpts(), opts.longOpts(),
-                          nullptr)) != -1) {
+  while ((c = getopt_long(
+              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
     switch (c) {
       case 'h':
         usage();
diff --git a/tools/OIP.cpp b/tools/OIP.cpp
index 03df960..66c986e 100644
--- a/tools/OIP.cpp
+++ b/tools/OIP.cpp
@@ -208,7 +208,9 @@ void printFuncArg(const std::shared_ptr<FuncDesc::TargetObject>& funcObj) {
       const auto& location = funcArg->locator.locations[i];
       printf(
           "{\"start\":\"0x%zx\",\"end\":\"0x%zx\",\"expr_size\":%zu,\"expr\":[",
-          location.start, location.end, location.expr_size);
+          location.start,
+          location.end,
+          location.expr_size);
       for (size_t j = 0; j < location.expr_size; j++) {
         if (j > 0) {
           printf(",");
@@ -232,8 +234,8 @@ void printFuncDesc(const std::shared_ptr<FuncDesc>& funcDesc) {
     if (!isFirstRange) {
       printf(",");
     }
-    printf("{\"start\": \"0x%zx\", \"end\": \"0x%zx\"}", range.start,
-           range.end);
+    printf(
+        "{\"start\": \"0x%zx\", \"end\": \"0x%zx\"}", range.start, range.end);
     isFirstRange = false;
   }
   printf("],");
diff --git a/tools/OIRP.cpp b/tools/OIRP.cpp
index f15637b..2421e84 100644
--- a/tools/OIRP.cpp
+++ b/tools/OIRP.cpp
@@ -227,8 +227,10 @@ int main(int argc, const char** argv) {
     rocksdb::DB* _db = nullptr;
     if (auto status = rocksdb::DB::Open(options, dbpath.string(), &_db);
         !status.ok()) {
-      fprintf(stderr, "Failed to open DB '%s' with error %s\n",
-              dbpath.string().c_str(), status.ToString().c_str());
+      fprintf(stderr,
+              "Failed to open DB '%s' with error %s\n",
+              dbpath.string().c_str(),
+              status.ToString().c_str());
       return 1;
     }
     db.reset(_db);
diff --git a/tools/OITB.cpp b/tools/OITB.cpp
index d016c29..2570aef 100644
--- a/tools/OITB.cpp
+++ b/tools/OITB.cpp
@@ -36,15 +36,24 @@ using namespace oi::detail;
 
 constexpr static OIOpts opts{
     OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit"},
-    OIOpt{'a', "log-all-structs", no_argument, nullptr,
+    OIOpt{'a',
+          "log-all-structs",
+          no_argument,
+          nullptr,
           "Enable TreeBuilder::Config::logAllStructs (=true)\n"
           "Note: this option is already enabled, this is a no-op"},
-    OIOpt{'J', "dump-json", optional_argument, "[oid_out.json]",
+    OIOpt{'J',
+          "dump-json",
+          optional_argument,
+          "[oid_out.json]",
           "File to dump the results to, as JSON\n"
           "(in addition to the default RocksDB output)"},
-    OIOpt{'f', "enable-feature", required_argument, "FEATURE",
-          "Enable feature"},
-    OIOpt{'F', "disable-feature", required_argument, "FEATURE",
+    OIOpt{
+        'f', "enable-feature", required_argument, "FEATURE", "Enable feature"},
+    OIOpt{'F',
+          "disable-feature",
+          required_argument,
+          "FEATURE",
           "Disable feature"},
 };
 
@@ -146,8 +155,8 @@ int main(int argc, char* argv[]) {
   };
 
   int c = '\0';
-  while ((c = getopt_long(argc, argv, opts.shortOpts(), opts.longOpts(),
-                          nullptr)) != -1) {
+  while ((c = getopt_long(
+              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
     switch (c) {
       case 'h':
         usage(std::cout);

From 0b8f8732827e8cffdd6ecfb19909a1fe74a5d7bd Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Mon, 13 Nov 2023 10:27:17 -0800
Subject: [PATCH 013/188] add range-v3 library

Adds the range-v3 library which supports features that otherwise wouldn't be
available until C++23 or C++26. I caught a couple of uses that suit it but this
will allow us to use more in future.

Test Plan:
- CI
---
 CMakeLists.txt               | 10 ++++++++++
 oi/OICompiler.h              | 18 +++++-------------
 oi/type_graph/KeyCapture.cpp |  3 +--
 3 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 28ccdf4..d7f9ee7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -95,6 +95,15 @@ FetchContent_Declare(
 )
 FetchContent_Populate(folly)
 
+### range-v3
+FetchContent_Declare(
+  range-v3
+  GIT_REPOSITORY https://github.com/ericniebler/range-v3.git
+  GIT_TAG        a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
+  GIT_PROGRESS   TRUE
+)
+FetchContent_MakeAvailable(range-v3)
+
 set_project_warnings()
 
 if (ASAN)
@@ -291,6 +300,7 @@ target_link_libraries(oicore
   ${Boost_LIBRARIES}
   Boost::headers
   glog::glog
+  range-v3
   resources
 )
 if (FORCE_LLVM_STATIC)
diff --git a/oi/OICompiler.h b/oi/OICompiler.h
index b08528c..2ce4544 100644
--- a/oi/OICompiler.h
+++ b/oi/OICompiler.h
@@ -20,6 +20,8 @@
 #include <filesystem>
 #include <memory>
 #include <optional>
+#include <range/v3/algorithm/find_if.hpp>
+#include <range/v3/algorithm/starts_with.hpp>
 #include <ranges>
 #include <set>
 #include <span>
@@ -204,19 +206,9 @@ std::optional<std::vector<uintptr_t>> OICompiler::locateOpcodes(
 
   std::vector<uintptr_t> locs;
   while (auto inst = DG()) {
-    auto it = std::find_if(
-        std::begin(needles), std::end(needles), [&](const auto& needle) {
-          // std::ranges::starts_with(inst->opcodes, needle)
-          if (std::ranges::size(needle) > std::ranges::size(inst->opcodes))
-            return false;
-          auto it1 = std::ranges::begin(inst->opcodes);
-          auto it2 = std::ranges::begin(needle);
-          for (; it2 != std::ranges::end(needle); ++it1, ++it2) {
-            if (*it1 != *it2)
-              return false;
-          }
-          return true;
-        });
+    auto it = ranges::find_if(needles, [&](const auto& needle) {
+      return ranges::starts_with(inst->opcodes, needle);
+    });
 
     if (it != std::end(needles)) {
       locs.push_back(inst->offset);
diff --git a/oi/type_graph/KeyCapture.cpp b/oi/type_graph/KeyCapture.cpp
index 5e09020..e3f7d11 100644
--- a/oi/type_graph/KeyCapture.cpp
+++ b/oi/type_graph/KeyCapture.cpp
@@ -66,8 +66,7 @@ void KeyCapture::visit(Class& c) {
       continue;
     if (!keyToCapture.member.has_value())
       continue;
-    for (size_t i = 0; i < c.members.size(); i++) {
-      auto& member = c.members[i];
+    for (auto& member : c.members) {
       if (member.name != *keyToCapture.member)
         continue;
 

From e0e81f5876e8db9f2cb01138ae4b575eb554288a Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Mon, 13 Nov 2023 10:31:52 -0800
Subject: [PATCH 014/188] collapse TreeBuilderV2 features

Summary:

Currently there are two features between CodeGen v2 (TypeGraph) and TreeBuilder
v2. These are TypedDataSegment and TreeBuilderTypeChecking. Each of these
features currently has a full set of tests run in the CI and each have specific
exclusions.

Collapse these features into TreeBuilder v2. This allows for significantly
simplified testing as any OIL tests run under TreeBuilder v2 and any OID tests
run under TreeBuilder v1.

The reasoning behind this is I no longer intend to partially roll out this
feature. Full TreeBuilder v2 applies different conditions to containers than
the intermediate states, and writing these only to have them never deployed is
a waste of time.

Test Plan:
- it builds
- CI
---
 .circleci/config.yml         |  22 -------
 oi/CodeGen.cpp               |  64 ++++++--------------
 oi/ContainerInfo.cpp         |  10 +--
 oi/ContainerInfo.h           |   1 -
 oi/Features.cpp              |  13 +---
 oi/Features.h                |  26 ++++----
 oi/FuncGen.cpp               | 114 -----------------------------------
 oi/FuncGen.h                 |   5 --
 oi/OICompiler.cpp            |   6 +-
 oi/OIGenerator.cpp           |   2 -
 oi/OILibraryImpl.cpp         |   2 -
 test/integration/arrays.toml |   2 +-
 types/README.md              |  10 +--
 13 files changed, 39 insertions(+), 238 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 286f759..7e805ad 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -21,20 +21,6 @@ workflows:
           oid_test_args: "-ftype-graph"
           tests_regex: "OidIntegration\\..*"
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
-      - test:
-          name: test-typed-data-segment-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-ftyped-data-segment"
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
-      - test:
-          name: test-tree-builder-type-checking-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-ftree-builder-type-checking"
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
       - coverage:
           name: coverage
           requires:
@@ -43,14 +29,6 @@ workflows:
           name: coverage-type-graph
           requires:
             - test-type-graph-gcc
-      - coverage:
-          name: coverage-typed-data-segment
-          requires:
-            - test-typed-data-segment-gcc
-      - coverage:
-          name: coverage-tree-builder-type-checking
-          requires:
-            - test-tree-builder-type-checking-gcc
 
       - build:
           name: build-clang
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 2fe0566..73725e4 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -119,17 +119,14 @@ void addIncludes(const TypeGraph& typeGraph,
                  FeatureSet features,
                  std::string& code) {
   std::set<std::string_view> includes{"cstddef"};
-  if (features[Feature::TypedDataSegment]) {
+  if (features[Feature::TreeBuilderV2]) {
+    code += "#define DEFINE_DESCRIBE 1\n";  // added before all includes
+
     includes.emplace("functional");
+    includes.emplace("oi/exporters/inst.h");
+    includes.emplace("oi/types/dy.h");
     includes.emplace("oi/types/st.h");
   }
-  if (features[Feature::TreeBuilderTypeChecking]) {
-    includes.emplace("oi/types/dy.h");
-
-    code += "#define DEFINE_DESCRIBE 1\n";  // added before all includes
-  }
-  if (features[Feature::TreeBuilderV2])
-    includes.emplace("oi/exporters/inst.h");
   if (features[Feature::Library]) {
     includes.emplace("vector");
     includes.emplace("oi/IntrospectionResult.h");
@@ -832,35 +829,20 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) {
   code += "  using type = ";
   genClassStaticType(c, code);
   code += ";\n";
-  if (config_.features[Feature::TreeBuilderV2])
-    genClassTreeBuilderInstructions(c, code);
+  genClassTreeBuilderInstructions(c, code);
   genClassTraversalFunction(c, code);
   code += "};\n";
 }
 
 namespace {
 
-void genContainerTypeHandler(FeatureSet features,
-                             std::unordered_set<const ContainerInfo*>& used,
+void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
                              const ContainerInfo& c,
                              std::span<const TemplateParam> templateParams,
                              std::string& code) {
   if (!used.insert(&c).second)
     return;
 
-  if (!features[Feature::TreeBuilderV2]) {
-    const auto& handler = c.codegen.handler;
-    if (handler.empty()) {
-      LOG(ERROR) << "`codegen.handler` must be specified for all containers "
-                    "under \"-ftyped-data-segment\", not specified for \"" +
-                        c.typeName + "\"";
-      throw std::runtime_error("missing `codegen.handler`");
-    }
-    auto fmt = boost::format(c.codegen.handler) % c.typeName;
-    code += fmt.str();
-    return;
-  }
-
   code += c.codegen.extra;
 
   // TODO: Move this check into the ContainerInfo parsing once always enabled.
@@ -1088,7 +1070,7 @@ constexpr inst::Field make_field(std::string_view name) {
                     "0"},
   };
   genContainerTypeHandler(
-      features, used, FuncGen::GetOiArrayContainerInfo(), arrayParams, code);
+      used, FuncGen::GetOiArrayContainerInfo(), arrayParams, code);
 }
 
 }  // namespace
@@ -1098,14 +1080,10 @@ void CodeGen::addTypeHandlers(const TypeGraph& typeGraph, std::string& code) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
       genClassTypeHandler(*c, code);
     } else if (const auto* con = dynamic_cast<const Container*>(&t)) {
-      genContainerTypeHandler(config_.features,
-                              definedContainers_,
-                              con->containerInfo_,
-                              con->templateParams,
-                              code);
+      genContainerTypeHandler(
+          definedContainers_, con->containerInfo_, con->templateParams, code);
     } else if (const auto* cap = dynamic_cast<const CaptureKeys*>(&t)) {
-      genContainerTypeHandler(config_.features,
-                              definedContainers_,
+      genContainerTypeHandler(definedContainers_,
                               cap->containerInfo(),
                               cap->container().templateParams,
                               code);
@@ -1227,14 +1205,14 @@ void CodeGen::generate(
   if (!config_.features[Feature::Library]) {
     FuncGen::DeclareExterns(code);
   }
-  if (!config_.features[Feature::TypedDataSegment]) {
+  if (!config_.features[Feature::TreeBuilderV2]) {
     defineMacros(code);
   }
   addIncludes(typeGraph, config_.features, code);
   defineInternalTypes(code);
   FuncGen::DefineJitLog(code, config_.features);
 
-  if (config_.features[Feature::TypedDataSegment]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     if (config_.features[Feature::Library]) {
       FuncGen::DefineBackInserterDataBuffer(code);
     } else {
@@ -1242,10 +1220,8 @@ void CodeGen::generate(
     }
     code += "using namespace oi;\n";
     code += "using namespace oi::detail;\n";
-    if (config_.features[Feature::TreeBuilderV2]) {
-      code += "using oi::exporters::ParsedData;\n";
-      code += "using namespace oi::exporters;\n";
-    }
+    code += "using oi::exporters::ParsedData;\n";
+    code += "using namespace oi::exporters;\n";
     code += "namespace OIInternal {\nnamespace {\n";
     FuncGen::DefineBasicTypeHandlers(code, config_.features);
     code += "} // namespace\n} // namespace OIInternal\n";
@@ -1265,7 +1241,7 @@ void CodeGen::generate(
    * process faster.
    */
   code += "namespace OIInternal {\nnamespace {\n";
-  if (!config_.features[Feature::TypedDataSegment]) {
+  if (!config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineEncodeData(code);
     FuncGen::DefineEncodeDataSize(code);
     FuncGen::DefineStoreData(code);
@@ -1280,7 +1256,7 @@ void CodeGen::generate(
     genExclusiveSizes(typeGraph, code);
   }
 
-  if (config_.features[Feature::TypedDataSegment]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     addStandardTypeHandlers(typeGraph, config_.features, code);
     addTypeHandlers(typeGraph, code);
   } else {
@@ -1297,10 +1273,8 @@ void CodeGen::generate(
   code += "} // namespace\n} // namespace OIInternal\n";
 
   const auto typeName = SymbolService::getTypeName(drgnType);
-  if (config_.features[Feature::Library]) {
+  if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
-  } else if (config_.features[Feature::TypedDataSegment]) {
-    FuncGen::DefineTopLevelGetSizeRefTyped(code, typeName, config_.features);
   } else {
     FuncGen::DefineTopLevelGetSizeRef(code, typeName, config_.features);
   }
@@ -1310,8 +1284,6 @@ void CodeGen::generate(
                                            typeName,
                                            calculateExclusiveSize(rootType),
                                            enumerateTypeNames(rootType));
-  } else if (config_.features[Feature::TreeBuilderTypeChecking]) {
-    FuncGen::DefineOutputType(code, typeName);
   }
 
   if (!linkageName_.empty())
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 068da95..603c3c3 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -274,10 +274,6 @@ ContainerInfo::ContainerInfo(const fs::path& path) {
   } else {
     throw ContainerInfoError(path, "`codegen.decl` is a required field");
   }
-  if (std::optional<std::string> str =
-          codegenToml["handler"].value<std::string>()) {
-    codegen.handler = std::move(*str);
-  }
   if (std::optional<std::string> str =
           codegenToml["traversal_func"].value<std::string>()) {
     codegen.traversalFunc = std::move(*str);
@@ -323,8 +319,6 @@ ContainerInfo::ContainerInfo(std::string typeName_,
       matcher(getMatcher(typeName)),
       ctype(ctype_),
       header(std::move(header_)),
-      codegen(Codegen{"// DummyDecl %1%\n",
-                      "// DummyFunc %1%\n",
-                      "// DummyHandler %1%\n",
-                      "// DummyFunc\n"}) {
+      codegen(Codegen{
+          "// DummyDecl %1%\n", "// DummyFunc %1%\n", "// DummyFunc\n"}) {
 }
diff --git a/oi/ContainerInfo.h b/oi/ContainerInfo.h
index 0ad9da9..376274d 100644
--- a/oi/ContainerInfo.h
+++ b/oi/ContainerInfo.h
@@ -36,7 +36,6 @@ struct ContainerInfo {
   struct Codegen {
     std::string decl;
     std::string func;
-    std::string handler = "";
     std::string traversalFunc = "";
     std::string extra = "";
     std::vector<Processor> processors{};
diff --git a/oi/Features.cpp b/oi/Features.cpp
index baf9344..9d82b17 100644
--- a/oi/Features.cpp
+++ b/oi/Features.cpp
@@ -40,11 +40,6 @@ std::optional<std::string_view> featureHelp(Feature f) {
       return "Use Type Graph for code generation (CodeGen v2).";
     case Feature::PruneTypeGraph:
       return "Prune unreachable nodes from the type graph";
-    case Feature::TypedDataSegment:
-      return "Use Typed Data Segment in generated code.";
-    case Feature::TreeBuilderTypeChecking:
-      return "Use Typed Data Segment to perform runtime Type Checking in "
-             "TreeBuilder.";
     case Feature::Library:
       return std::nullopt;  // Hide in OID help
     case Feature::TreeBuilderV2:
@@ -65,14 +60,8 @@ std::optional<std::string_view> featureHelp(Feature f) {
 
 std::span<const Feature> requirements(Feature f) {
   switch (f) {
-    case Feature::TypedDataSegment:
-      static constexpr std::array tds = {Feature::TypeGraph};
-      return tds;
-    case Feature::TreeBuilderTypeChecking:
-      static constexpr std::array tc = {Feature::TypedDataSegment};
-      return tc;
     case Feature::TreeBuilderV2:
-      static constexpr std::array tb2 = {Feature::TreeBuilderTypeChecking};
+      static constexpr std::array tb2 = {Feature::TypeGraph};
       return tb2;
     case Feature::Library:
       static constexpr std::array lib = {Feature::TreeBuilderV2};
diff --git a/oi/Features.h b/oi/Features.h
index 2e92ba0..959a576 100644
--- a/oi/Features.h
+++ b/oi/Features.h
@@ -22,20 +22,18 @@
 
 #include "oi/EnumBitset.h"
 
-#define OI_FEATURE_LIST                                    \
-  X(ChaseRawPointers, "chase-raw-pointers")                \
-  X(PackStructs, "pack-structs")                           \
-  X(GenPaddingStats, "gen-padding-stats")                  \
-  X(CaptureThriftIsset, "capture-thrift-isset")            \
-  X(TypeGraph, "type-graph")                               \
-  X(PruneTypeGraph, "prune-type-graph")                    \
-  X(TypedDataSegment, "typed-data-segment")                \
-  X(TreeBuilderTypeChecking, "tree-builder-type-checking") \
-  X(Library, "library")                                    \
-  X(TreeBuilderV2, "tree-builder-v2")                      \
-  X(GenJitDebug, "gen-jit-debug")                          \
-  X(JitLogging, "jit-logging")                             \
-  X(JitTiming, "jit-timing")                               \
+#define OI_FEATURE_LIST                         \
+  X(ChaseRawPointers, "chase-raw-pointers")     \
+  X(PackStructs, "pack-structs")                \
+  X(GenPaddingStats, "gen-padding-stats")       \
+  X(CaptureThriftIsset, "capture-thrift-isset") \
+  X(TypeGraph, "type-graph")                    \
+  X(PruneTypeGraph, "prune-type-graph")         \
+  X(Library, "library")                         \
+  X(TreeBuilderV2, "tree-builder-v2")           \
+  X(GenJitDebug, "gen-jit-debug")               \
+  X(JitLogging, "jit-logging")                  \
+  X(JitTiming, "jit-timing")                    \
   X(PolymorphicInheritance, "polymorphic-inheritance")
 
 namespace oi::detail {
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 9d6805b..7a1dd09 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -352,103 +352,6 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
   testCode.append(fmt.str());
 }
 
-/*
- * DefineTopLevelGetSizeRefTyped
- *
- * Top level function to run OI on a type utilising static types and enabled
- * with feature '-ftyped-data-segment'.
- */
-void FuncGen::DefineTopLevelGetSizeRefTyped(std::string& testCode,
-                                            const std::string& rawType,
-                                            FeatureSet features) {
-  std::string func = R"(
-    #pragma GCC diagnostic push
-    #pragma GCC diagnostic ignored "-Wunknown-attributes"
-    /* RawType: %1% */
-    void __attribute__((used, retain)) getSize_%2$016x(const OIInternal::__ROOT_TYPE__& t)
-    #pragma GCC diagnostic pop
-    {
-    )";
-  if (features[Feature::JitTiming]) {
-    func += "      const auto startTime = std::chrono::steady_clock::now();\n";
-  }
-  func += R"(
-      pointers.initialize();
-      pointers.add((uintptr_t)&t);
-      auto data = reinterpret_cast<uintptr_t*>(dataBase);
-
-      // TODO: Replace these with types::st::Uint64 once the VarInt decoding
-      // logic is moved out of OIDebugger and into new TreeBuilder.
-      size_t dataSegOffset = 0;
-      data[dataSegOffset++] = oidMagicId;
-      data[dataSegOffset++] = cookieValue;
-      uintptr_t& writtenSize = data[dataSegOffset++];
-      writtenSize = 0;
-      uintptr_t& timeTakenNs = data[dataSegOffset++];
-
-      dataSegOffset *= sizeof(uintptr_t);
-      JLOG("%1% @");
-      JLOGPTR(&t);
-
-      using ContentType = OIInternal::TypeHandler<DataBuffer::DataSegment, OIInternal::__ROOT_TYPE__>::type;
-      using SuffixType = types::st::Pair<
-        DataBuffer::DataSegment,
-        types::st::VarInt<DataBuffer::DataSegment>,
-        types::st::VarInt<DataBuffer::DataSegment>
-      >;
-      using DataBufferType = types::st::Pair<
-        DataBuffer::DataSegment,
-        ContentType,
-        SuffixType
-      >;
-
-      DataBufferType db = DataBuffer::DataSegment(dataSegOffset);
-      SuffixType suffix = db.delegate([&t](auto ret) {
-        return OIInternal::getSizeType<DataBuffer::DataSegment>(t, ret);
-      });
-      types::st::Unit<DataBuffer::DataSegment> end = suffix
-        .write(123456789)
-        .write(123456789);
-
-      dataSegOffset = end.offset();
-      writtenSize = dataSegOffset;
-      dataBase += dataSegOffset;
-    )";
-  if (features[Feature::JitTiming]) {
-    func += R"(
-      timeTakenNs = std::chrono::duration_cast<std::chrono::nanoseconds>(
-        std::chrono::steady_clock::now() - startTime).count();
-      )";
-  }
-  func += R"(
-    }
-    )";
-
-  boost::format fmt =
-      boost::format(func) % rawType % std::hash<std::string>{}(rawType);
-  testCode.append(fmt.str());
-}
-
-/*
- * DefineOutputType
- *
- * Present the dynamic type of an object for OID/OIL/OITB to link against.
- */
-void FuncGen::DefineOutputType(std::string& code, const std::string& rawType) {
-  std::string func = R"(
-    #pragma GCC diagnostic push
-    #pragma GCC diagnostic ignored "-Wunknown-attributes"
-    /* RawType: %1% */
-    extern const types::dy::Dynamic __attribute__((used, retain)) outputType%2$016x =
-          OIInternal::TypeHandler<DataBuffer::DataSegment, OIInternal::__ROOT_TYPE__>::type::describe;
-    #pragma GCC diagnostic pop
-)";
-
-  boost::format fmt =
-      boost::format(func) % rawType % std::hash<std::string>{}(rawType);
-  code.append(fmt.str());
-}
-
 void FuncGen::DefineTreeBuilderInstructions(
     std::string& code,
     const std::string& rawType,
@@ -804,23 +707,6 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
                         UNKNOWN_TYPE,
                         "cstdint"};  // TODO: remove the need for a dummy header
 
-  oiArray.codegen.handler = R"(
-template<typename DB, typename T0, long unsigned int N>
-struct TypeHandler<DB, %1%<T0, N>> {
-  using type = types::st::List<DB, typename TypeHandler<DB, T0>::type>;
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, N> &container,
-      typename TypeHandler<DB, %1%<T0,N>>::type returnArg) {
-    auto tail = returnArg.write(N);
-    for (size_t i=0; i<N; i++) {
-      tail = tail.delegate([&container, i](auto ret) {
-          return TypeHandler<DB, T0>::getSizeType(container.vals[i], ret);
-      });
-    }
-    return tail.finish();
-  }
-};
-)";
   oiArray.codegen.traversalFunc = R"(
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
diff --git a/oi/FuncGen.h b/oi/FuncGen.h
index ff21841..33aaf9c 100644
--- a/oi/FuncGen.h
+++ b/oi/FuncGen.h
@@ -61,11 +61,6 @@ class FuncGen {
   static void DefineTopLevelGetSizeRef(std::string& testCode,
                                        const std::string& rawType,
                                        FeatureSet features);
-  static void DefineTopLevelGetSizeRefTyped(std::string& testCode,
-                                            const std::string& rawType,
-                                            FeatureSet features);
-  static void DefineOutputType(std::string& testCode,
-                               const std::string& rawType);
   static void DefineTreeBuilderInstructions(
       std::string& testCode,
       const std::string& rawType,
diff --git a/oi/OICompiler.cpp b/oi/OICompiler.cpp
index 9d3aa79..cd67c68 100644
--- a/oi/OICompiler.cpp
+++ b/oi/OICompiler.cpp
@@ -528,10 +528,8 @@ bool OICompiler::compile(const std::string& code,
   static const auto syntheticHeaders =
       std::array<std::pair<Feature, std::pair<std::string_view, std::string>>,
                  7>{{
-          {Feature::TypedDataSegment,
-           {headers::oi_types_st_h, "oi/types/st.h"}},
-          {Feature::TreeBuilderTypeChecking,
-           {headers::oi_types_dy_h, "oi/types/dy.h"}},
+          {Feature::TreeBuilderV2, {headers::oi_types_st_h, "oi/types/st.h"}},
+          {Feature::TreeBuilderV2, {headers::oi_types_dy_h, "oi/types/dy.h"}},
           {Feature::TreeBuilderV2,
            {headers::oi_exporters_inst_h, "oi/exporters/inst.h"}},
           {Feature::TreeBuilderV2,
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 6c4590f..1cb2a49 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -184,8 +184,6 @@ int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
 
   std::map<Feature, bool> featuresMap = {
       {Feature::TypeGraph, true},
-      {Feature::TypedDataSegment, true},
-      {Feature::TreeBuilderTypeChecking, true},
       {Feature::TreeBuilderV2, true},
       {Feature::Library, true},
       {Feature::PackStructs, true},
diff --git a/oi/OILibraryImpl.cpp b/oi/OILibraryImpl.cpp
index 0f33356..6442e9a 100644
--- a/oi/OILibraryImpl.cpp
+++ b/oi/OILibraryImpl.cpp
@@ -180,8 +180,6 @@ namespace {
 std::map<Feature, bool> convertFeatures(std::unordered_set<oi::Feature> fs) {
   std::map<Feature, bool> out{
       {Feature::TypeGraph, true},
-      {Feature::TypedDataSegment, true},
-      {Feature::TreeBuilderTypeChecking, true},
       {Feature::TreeBuilderV2, true},
       {Feature::Library, true},
       {Feature::PackStructs, true},
diff --git a/test/integration/arrays.toml b/test/integration/arrays.toml
index 3efb2d2..4a06b13 100644
--- a/test/integration/arrays.toml
+++ b/test/integration/arrays.toml
@@ -62,7 +62,7 @@ definitions = '''
       }]}]'''
   [cases.multidim_legacy] # Test for legacy behaviour. Remove with OICodeGen
     oil_disable = 'oil only runs on codegen v2'
-    cli_options = ["-Ftype-graph", "-Ftyped-data-segment", "-Ftree-builder-type-checking", "-Ftree-builder-v2"]
+    cli_options = ["-Ftype-graph", "-Ftree-builder-v2"]
     param_types = ["const MultiDim&"]
     setup = "return {};"
     expect_json = '''[{
diff --git a/types/README.md b/types/README.md
index ebde86c..e2b70fd 100644
--- a/types/README.md
+++ b/types/README.md
@@ -62,14 +62,10 @@ This document describes the format of the container definition files contained i
   will collide if they share the same name.
 
 
-## Changes introduced with Typed Data Segment
-- `decl` and `func` fields are ignored when using `-ftyped-data-segment` and the
-  `handler` field is used instead.
-
 ## Changes introduced with TreeBuilder V2
-- `decl`, `func`, and `handler` fields are ignored when using `-ftree-builder-v2`.
-   The `TypeHandler` is instead constructed from `traversal_func` and `processor`
-   entries.
+- `decl` and `func` fields are ignored when using `-ftree-builder-v2`. The
+  `TypeHandler` is constructed from `traversal_func` field and `processor`
+  entries.
 
 ## Changes introduced with TypeGraph
 - `typeName` and `matcher` fields have been merged into the single field `type_name`.

From 2f1bf4e0c2254bf229a4e7fd37f49dd0610c641d Mon Sep 17 00:00:00 2001
From: Jake Hillion <jake@hillion.co.uk>
Date: Mon, 13 Nov 2023 10:39:36 -0800
Subject: [PATCH 015/188] ci: move formatting checks to nix

---
 .circleci/config.yml | 31 ++++-------------
 flake.lock           | 82 ++++++++++++++++++++++++++++++++++++++++++++
 flake.nix            | 30 ++++++++++++++++
 3 files changed, 119 insertions(+), 24 deletions(-)
 create mode 100644 flake.lock
 create mode 100644 flake.nix

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 286f759..94451db 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -73,6 +73,10 @@ workflows:
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_a"
 
 executors:
+  nix-docker:
+    docker:
+      - image: nixos/nix:latest
+    resource_class: small
   ubuntu-docker:
     docker:
       - image: ubuntu:jammy
@@ -84,33 +88,12 @@ executors:
 
 jobs:
   lint:
-    executor: ubuntu-docker
+    executor: nix-docker
     steps:
-      - run:
-          name: Install dependencies
-          command: |
-            apt-get update
-            apt-get install -y \
-              clang-format \
-              git \
-              python3-pip
-            # click broke semver with 8.1.0, causing issues for black
-            pip install click==8.0.0 black isort
-          environment:
-            DEBIAN_FRONTEND: noninteractive
       - checkout
       - run:
-          name: clang-format
-          command: |
-            git ls-files '*.cpp' '*.h' | xargs clang-format --fallback-style=Google -i
-            git ls-files '*.py' | xargs black
-            git ls-files '*.py' | xargs isort
-            git diff --exit-code
-      - run:
-          name: python linting
-          command: |
-            black --check --diff test/
-            isort --check --diff test/
+          name: Flake check
+          command: nix --experimental-features 'nix-command flakes' flake check
 
   build:
     # TODO this job could be run in Docker
diff --git a/flake.lock b/flake.lock
new file mode 100644
index 0000000..927748e
--- /dev/null
+++ b/flake.lock
@@ -0,0 +1,82 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1694529238,
+        "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "ff7b65b44d01cf9ba6a71320833626af21126384",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1696375444,
+        "narHash": "sha256-Sv0ICt/pXfpnFhTGYTsX6lUr1SljnuXWejYTI2ZqHa4=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "81e8f48ebdecf07aab321182011b067aafc78896",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs",
+        "treefmt-nix": "treefmt-nix"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1695822946,
+        "narHash": "sha256-IQU3fYo0H+oGlqX5YrgZU3VRhbt2Oqe6KmslQKUO4II=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "720bd006d855b08e60664e4683ccddb7a9ff614a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
diff --git a/flake.nix b/flake.nix
new file mode 100644
index 0000000..b35d91e
--- /dev/null
+++ b/flake.nix
@@ -0,0 +1,30 @@
+{
+  description = "A flake for building Object Introspection.";
+
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
+
+    flake-utils.url = "github:numtide/flake-utils";
+
+    treefmt-nix.url = "github:numtide/treefmt-nix";
+    treefmt-nix.inputs.nixpkgs.follows = "nixpkgs";
+  };
+
+  outputs = { self, nixpkgs, flake-utils, treefmt-nix, ... }@inputs:
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        pkgs = nixpkgs.legacyPackages.${system};
+        treefmtEval = treefmt-nix.lib.evalModule pkgs (pkgs: {
+          projectRootFile = "flake.nix";
+          settings.global.excludes = [ "./extern/**" ];
+
+          programs.nixfmt.enable = true;
+          programs.clang-format.enable = true;
+          programs.black.enable = true;
+          programs.isort.enable = true;
+        });
+      in {
+        formatter = treefmtEval.config.build.wrapper;
+        checks.formatting = treefmtEval.config.build.check self;
+      });
+}

From 0a89e9b580ef2523ec80b988ef3b5225693f7181 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 14 Nov 2023 08:23:59 -0800
Subject: [PATCH 016/188] oil: add support for std::list

Summary:

Remove the now useless `handler` and adds the `traversal_func` and `processor`
entries for `std::list`. This type is a bit weird as most of our sequential
containers don't have any overhead on storing the element. I went for the same
approach we take for maps where we have a shared `[]` element covering the map
overhead and below that a `key` & `value`. As we only have a single element
under it which doesn't have a logical name I went for `*`.

Closes #315.

Test Plan:
- CI
- Copied the relevant `std::vector` tests and updated the existing one.
---
 test/integration/std_list.toml               | 59 +++++++++++++++++++
 test/integration/std_list_del_allocator.toml |  8 ++-
 types/cxx11_list_type.toml                   | 62 ++++++++++++++++----
 3 files changed, 118 insertions(+), 11 deletions(-)
 create mode 100644 test/integration/std_list.toml

diff --git a/test/integration/std_list.toml b/test/integration/std_list.toml
new file mode 100644
index 0000000..dfe2d8f
--- /dev/null
+++ b/test/integration/std_list.toml
@@ -0,0 +1,59 @@
+includes = ["list"]
+
+definitions = '''
+  struct SimpleStruct {
+    int a;
+    char b;
+    long long c;
+  };
+'''
+
+[cases]
+  [cases.int_empty]
+    param_types = ["const std::list<int>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+  [cases.int_some]
+    param_types = ["const std::list<int>&"]
+    setup = "return {{1,2,3}};"
+    expect_json = '[{"staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}]'
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
+      {"staticSize":4, "exclusiveSize":4},
+      {"staticSize":4, "exclusiveSize":4},
+      {"staticSize":4, "exclusiveSize":4}
+    ]}]'''
+  [cases.struct_some]
+    param_types = ["const std::list<SimpleStruct>&"]
+    setup = "return {{{}, {}, {}}};"
+    expect_json = '[{"staticSize":24, "dynamicSize":48, "length":3, "capacity":3, "elementStaticSize":16}]'
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
+      {"staticSize":16, "exclusiveSize":3},
+      {"staticSize":16, "exclusiveSize":3},
+      {"staticSize":16, "exclusiveSize":3}
+    ]}]'''
+  [cases.list_int_empty]
+    param_types = ["const std::list<std::list<int>>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":24}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+  [cases.list_int_some]
+    param_types = ["const std::list<std::list<int>>&"]
+    setup = "return {{{1,2,3},{4},{5,6}}};"
+    expect_json = '''[{
+      "staticSize":24,
+      "dynamicSize":96,
+      "exclusiveSize":24,
+      "length":3,
+      "capacity":3,
+      "elementStaticSize":24,
+      "members":[
+        {"staticSize":24, "dynamicSize":12, "exclusiveSize":36, "length":3, "capacity":3, "elementStaticSize":4},
+        {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4},
+        {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
+      ]}]'''
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[
+      {"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[]},
+      {"staticSize":24, "exclusiveSize":24, "length":1, "capacity": 1, "members":[]},
+      {"staticSize":24, "exclusiveSize":24, "length":2, "capacity": 2, "members":[]}
+    ]}]'''
diff --git a/test/integration/std_list_del_allocator.toml b/test/integration/std_list_del_allocator.toml
index a85668d..57e7b07 100644
--- a/test/integration/std_list_del_allocator.toml
+++ b/test/integration/std_list_del_allocator.toml
@@ -27,7 +27,6 @@ includes = ["list"]
 
 [cases]
   [cases.a]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/315
     param_types = ["const Foo&"]
     setup = '''
       Foo foo;
@@ -58,3 +57,10 @@ includes = ["list"]
               ]}
           ]}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize": 48,
+      "exclusiveSize": 0,
+      "members": [
+        {"name": "v1", "staticSize": 24, "exclusiveSize": 24, "length": 1, "capacity": 1},
+        {"name": "v2", "staticSize": 24, "exclusiveSize": 24, "length": 2, "capacity": 2}
+      ]}]'''
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index 437846c..c88848b 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -1,10 +1,10 @@
 [info]
 type_name = "std::__cxx11::list"
 stub_template_params = [1]
-ctype = "LIST_TYPE"
 header = "list"
 
 # Old:
+ctype = "LIST_TYPE"
 typeName = "std::__cxx11::list"
 ns = ["namespace std"]
 numTemplateParams = 1
@@ -43,18 +43,60 @@ struct TypeHandler<DB, %1% <T0, T1>> {
   static types::st::Unit<DB> getSizeType(
       const %1% <T0, T1> & container,
       typename TypeHandler<DB, %1% <T0, T1>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container)
-                    .write(container.size());
 
     // The double ampersand is needed otherwise this loop doesn't work with
     // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it, ret);
-      });
-    }
-
-    return tail.finish();
   }
 };
 """
+
+traversal_func = """
+auto tail = returnArg.write((uintptr_t)&container)
+                .write(container.size());
+
+for (auto&& it : container) {
+  tail = tail.delegate([&it](auto ret) {
+    return OIInternal::getSizeType<DB>(it, ret);
+  });
+}
+
+return tail.finish();
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.pointer = std::get<ParsedData::VarInt>(d.val).value;
+"""
+
+[[codegen.processor]]
+type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+func = """
+#ifdef __GLIBCXX__
+static constexpr size_t element_size = sizeof(std::_List_node<T0>);
+#else
+static_assert(false && "No known element_size for list. See types/cxx11_list_type.toml");
+#endif
+
+static constexpr std::array<inst::Field, 1> child_field{
+  make_field<DB, T0>("*"),
+};
+static constexpr inst::Field element{
+  element_size,
+  element_size - sizeof(T0),
+  "[]",
+  std::array<std::string_view, 0>{},
+  child_field,
+  std::array<inst::ProcessorInst, 0>{},
+};
+static constexpr auto childField = make_field<DB, T0>("[]");
+
+auto list = std::get<ParsedData::List>(d.val);
+el.container_stats.emplace(result::Element::ContainerStats{
+  .capacity = list.length,
+  .length = list.length,
+});
+el.exclusive_size += (el.container_stats->capacity - el.container_stats->length) * sizeof(T0);
+
+stack_ins(inst::Repeat{ list.length, childField });
+"""

From 6925380ff2b7638a3aeb5ebcd567c9a4c171a781 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 14 Nov 2023 08:34:17 -0800
Subject: [PATCH 017/188] oil: add support for std::list

Summary:

Remove the now useless `handler` and adds the `traversal_func` and `processor`
entries for `std::list`. This type is a bit weird as most of our sequential
containers don't have any overhead on storing the element. I went for the same
approach we take for maps where we have a shared `[]` element covering the map
overhead and below that a `key` & `value`. As we only have a single element
under it which doesn't have a logical name I went for `*`.

Closes #315.

Test Plan:
- CI
- Copied the relevant `std::vector` tests and updated the existing one.
---
 test/integration/std_list.toml               | 59 +++++++++++++++++++
 test/integration/std_list_del_allocator.toml |  8 ++-
 types/cxx11_list_type.toml                   | 62 ++++++++++++++++----
 3 files changed, 118 insertions(+), 11 deletions(-)
 create mode 100644 test/integration/std_list.toml

diff --git a/test/integration/std_list.toml b/test/integration/std_list.toml
new file mode 100644
index 0000000..dfe2d8f
--- /dev/null
+++ b/test/integration/std_list.toml
@@ -0,0 +1,59 @@
+includes = ["list"]
+
+definitions = '''
+  struct SimpleStruct {
+    int a;
+    char b;
+    long long c;
+  };
+'''
+
+[cases]
+  [cases.int_empty]
+    param_types = ["const std::list<int>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+  [cases.int_some]
+    param_types = ["const std::list<int>&"]
+    setup = "return {{1,2,3}};"
+    expect_json = '[{"staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}]'
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
+      {"staticSize":4, "exclusiveSize":4},
+      {"staticSize":4, "exclusiveSize":4},
+      {"staticSize":4, "exclusiveSize":4}
+    ]}]'''
+  [cases.struct_some]
+    param_types = ["const std::list<SimpleStruct>&"]
+    setup = "return {{{}, {}, {}}};"
+    expect_json = '[{"staticSize":24, "dynamicSize":48, "length":3, "capacity":3, "elementStaticSize":16}]'
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
+      {"staticSize":16, "exclusiveSize":3},
+      {"staticSize":16, "exclusiveSize":3},
+      {"staticSize":16, "exclusiveSize":3}
+    ]}]'''
+  [cases.list_int_empty]
+    param_types = ["const std::list<std::list<int>>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":24}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+  [cases.list_int_some]
+    param_types = ["const std::list<std::list<int>>&"]
+    setup = "return {{{1,2,3},{4},{5,6}}};"
+    expect_json = '''[{
+      "staticSize":24,
+      "dynamicSize":96,
+      "exclusiveSize":24,
+      "length":3,
+      "capacity":3,
+      "elementStaticSize":24,
+      "members":[
+        {"staticSize":24, "dynamicSize":12, "exclusiveSize":36, "length":3, "capacity":3, "elementStaticSize":4},
+        {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4},
+        {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
+      ]}]'''
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[
+      {"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[]},
+      {"staticSize":24, "exclusiveSize":24, "length":1, "capacity": 1, "members":[]},
+      {"staticSize":24, "exclusiveSize":24, "length":2, "capacity": 2, "members":[]}
+    ]}]'''
diff --git a/test/integration/std_list_del_allocator.toml b/test/integration/std_list_del_allocator.toml
index a85668d..57e7b07 100644
--- a/test/integration/std_list_del_allocator.toml
+++ b/test/integration/std_list_del_allocator.toml
@@ -27,7 +27,6 @@ includes = ["list"]
 
 [cases]
   [cases.a]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/315
     param_types = ["const Foo&"]
     setup = '''
       Foo foo;
@@ -58,3 +57,10 @@ includes = ["list"]
               ]}
           ]}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize": 48,
+      "exclusiveSize": 0,
+      "members": [
+        {"name": "v1", "staticSize": 24, "exclusiveSize": 24, "length": 1, "capacity": 1},
+        {"name": "v2", "staticSize": 24, "exclusiveSize": 24, "length": 2, "capacity": 2}
+      ]}]'''
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index 437846c..c88848b 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -1,10 +1,10 @@
 [info]
 type_name = "std::__cxx11::list"
 stub_template_params = [1]
-ctype = "LIST_TYPE"
 header = "list"
 
 # Old:
+ctype = "LIST_TYPE"
 typeName = "std::__cxx11::list"
 ns = ["namespace std"]
 numTemplateParams = 1
@@ -43,18 +43,60 @@ struct TypeHandler<DB, %1% <T0, T1>> {
   static types::st::Unit<DB> getSizeType(
       const %1% <T0, T1> & container,
       typename TypeHandler<DB, %1% <T0, T1>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container)
-                    .write(container.size());
 
     // The double ampersand is needed otherwise this loop doesn't work with
     // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it, ret);
-      });
-    }
-
-    return tail.finish();
   }
 };
 """
+
+traversal_func = """
+auto tail = returnArg.write((uintptr_t)&container)
+                .write(container.size());
+
+for (auto&& it : container) {
+  tail = tail.delegate([&it](auto ret) {
+    return OIInternal::getSizeType<DB>(it, ret);
+  });
+}
+
+return tail.finish();
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.pointer = std::get<ParsedData::VarInt>(d.val).value;
+"""
+
+[[codegen.processor]]
+type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+func = """
+#ifdef __GLIBCXX__
+static constexpr size_t element_size = sizeof(std::_List_node<T0>);
+#else
+static_assert(false && "No known element_size for list. See types/cxx11_list_type.toml");
+#endif
+
+static constexpr std::array<inst::Field, 1> child_field{
+  make_field<DB, T0>("*"),
+};
+static constexpr inst::Field element{
+  element_size,
+  element_size - sizeof(T0),
+  "[]",
+  std::array<std::string_view, 0>{},
+  child_field,
+  std::array<inst::ProcessorInst, 0>{},
+};
+static constexpr auto childField = make_field<DB, T0>("[]");
+
+auto list = std::get<ParsedData::List>(d.val);
+el.container_stats.emplace(result::Element::ContainerStats{
+  .capacity = list.length,
+  .length = list.length,
+});
+el.exclusive_size += (el.container_stats->capacity - el.container_stats->length) * sizeof(T0);
+
+stack_ins(inst::Repeat{ list.length, childField });
+"""

From 4c31652c7a10f9816b14ec3938195611d5248d0d Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 14 Nov 2023 08:56:13 -0800
Subject: [PATCH 018/188] oil: add support for std::list

Summary:

Remove the now useless `handler` and adds the `traversal_func` and `processor`
entries for `std::list`. This type is a bit weird as most of our sequential
containers don't have any overhead on storing the element. I went for the same
approach we take for maps where we have a shared `[]` element covering the map
overhead and below that a `key` & `value`. As we only have a single element
under it which doesn't have a logical name I went for `*`.

Closes #315.

Test Plan:
- CI
- Copied the relevant `std::vector` tests and updated the existing one.
---
 test/integration/std_list.toml               | 59 ++++++++++++++++
 test/integration/std_list_del_allocator.toml |  8 ++-
 types/cxx11_list_type.toml                   | 71 +++++++++++++-------
 3 files changed, 114 insertions(+), 24 deletions(-)
 create mode 100644 test/integration/std_list.toml

diff --git a/test/integration/std_list.toml b/test/integration/std_list.toml
new file mode 100644
index 0000000..dfe2d8f
--- /dev/null
+++ b/test/integration/std_list.toml
@@ -0,0 +1,59 @@
+includes = ["list"]
+
+definitions = '''
+  struct SimpleStruct {
+    int a;
+    char b;
+    long long c;
+  };
+'''
+
+[cases]
+  [cases.int_empty]
+    param_types = ["const std::list<int>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+  [cases.int_some]
+    param_types = ["const std::list<int>&"]
+    setup = "return {{1,2,3}};"
+    expect_json = '[{"staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}]'
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
+      {"staticSize":4, "exclusiveSize":4},
+      {"staticSize":4, "exclusiveSize":4},
+      {"staticSize":4, "exclusiveSize":4}
+    ]}]'''
+  [cases.struct_some]
+    param_types = ["const std::list<SimpleStruct>&"]
+    setup = "return {{{}, {}, {}}};"
+    expect_json = '[{"staticSize":24, "dynamicSize":48, "length":3, "capacity":3, "elementStaticSize":16}]'
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
+      {"staticSize":16, "exclusiveSize":3},
+      {"staticSize":16, "exclusiveSize":3},
+      {"staticSize":16, "exclusiveSize":3}
+    ]}]'''
+  [cases.list_int_empty]
+    param_types = ["const std::list<std::list<int>>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":24}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+  [cases.list_int_some]
+    param_types = ["const std::list<std::list<int>>&"]
+    setup = "return {{{1,2,3},{4},{5,6}}};"
+    expect_json = '''[{
+      "staticSize":24,
+      "dynamicSize":96,
+      "exclusiveSize":24,
+      "length":3,
+      "capacity":3,
+      "elementStaticSize":24,
+      "members":[
+        {"staticSize":24, "dynamicSize":12, "exclusiveSize":36, "length":3, "capacity":3, "elementStaticSize":4},
+        {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4},
+        {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
+      ]}]'''
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[
+      {"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[]},
+      {"staticSize":24, "exclusiveSize":24, "length":1, "capacity": 1, "members":[]},
+      {"staticSize":24, "exclusiveSize":24, "length":2, "capacity": 2, "members":[]}
+    ]}]'''
diff --git a/test/integration/std_list_del_allocator.toml b/test/integration/std_list_del_allocator.toml
index a85668d..57e7b07 100644
--- a/test/integration/std_list_del_allocator.toml
+++ b/test/integration/std_list_del_allocator.toml
@@ -27,7 +27,6 @@ includes = ["list"]
 
 [cases]
   [cases.a]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/315
     param_types = ["const Foo&"]
     setup = '''
       Foo foo;
@@ -58,3 +57,10 @@ includes = ["list"]
               ]}
           ]}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize": 48,
+      "exclusiveSize": 0,
+      "members": [
+        {"name": "v1", "staticSize": 24, "exclusiveSize": 24, "length": 1, "capacity": 1},
+        {"name": "v2", "staticSize": 24, "exclusiveSize": 24, "length": 2, "capacity": 2}
+      ]}]'''
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index 437846c..fb54d22 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -1,10 +1,10 @@
 [info]
 type_name = "std::__cxx11::list"
 stub_template_params = [1]
-ctype = "LIST_TYPE"
 header = "list"
 
 # Old:
+ctype = "LIST_TYPE"
 typeName = "std::__cxx11::list"
 ns = ["namespace std"]
 numTemplateParams = 1
@@ -33,28 +33,53 @@ void getSizeType(const %1%<T, Allocator> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1>
-struct TypeHandler<DB, %1% <T0, T1>> {
-  using type = types::st::Pair<DB,
-        types::st::VarInt<DB>,
-        types::st::List<DB, typename TypeHandler<DB, T0>::type>>;
+traversal_func = """
+auto tail = returnArg.write((uintptr_t)&container)
+                .write(container.size());
 
-  static types::st::Unit<DB> getSizeType(
-      const %1% <T0, T1> & container,
-      typename TypeHandler<DB, %1% <T0, T1>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container)
-                    .write(container.size());
+for (auto&& it : container) {
+  tail = tail.delegate([&it](auto ret) {
+    return OIInternal::getSizeType<DB>(it, ret);
+  });
+}
 
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
+return tail.finish();
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.pointer = std::get<ParsedData::VarInt>(d.val).value;
+"""
+
+[[codegen.processor]]
+type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+func = """
+#ifdef __GLIBCXX__
+static constexpr size_t element_size = sizeof(std::_List_node<T0>);
+#else
+static_assert(false && "No known element_size for list. See types/cxx11_list_type.toml");
+#endif
+
+static constexpr std::array<inst::Field, 1> child_field{
+  make_field<DB, T0>("*"),
+};
+static constexpr inst::Field element{
+  element_size,
+  element_size - sizeof(T0),
+  "[]",
+  std::array<std::string_view, 0>{},
+  child_field,
+  std::array<inst::ProcessorInst, 0>{},
+};
+static constexpr auto childField = make_field<DB, T0>("[]");
+
+auto list = std::get<ParsedData::List>(d.val);
+el.container_stats.emplace(result::Element::ContainerStats{
+  .capacity = list.length,
+  .length = list.length,
+});
+el.exclusive_size += (el.container_stats->capacity - el.container_stats->length) * sizeof(T0);
+
+stack_ins(inst::Repeat{ list.length, childField });
 """

From ef1e5dac3bfaf61c3833781ac5491b69940ec678 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 14 Nov 2023 08:56:13 -0800
Subject: [PATCH 019/188] oil: add support for std::list

Summary:

Remove the now useless `handler` and adds the `traversal_func` and `processor`
entries for `std::list`. This type is a bit weird as most of our sequential
containers don't have any overhead on storing the element. I went for the same
approach we take for maps where we have a shared `[]` element covering the map
overhead and below that a `key` & `value`. As we only have a single element
under it which doesn't have a logical name I went for `*`.

Closes #315.

Test Plan:
- CI
- Copied the relevant `std::vector` tests and updated the existing one.
---
 test/integration/std_list.toml               | 59 ++++++++++++++++
 test/integration/std_list_del_allocator.toml |  8 ++-
 types/cxx11_list_type.toml                   | 71 +++++++++++++-------
 types/list_type.toml                         | 69 +++++++++++++------
 4 files changed, 161 insertions(+), 46 deletions(-)
 create mode 100644 test/integration/std_list.toml

diff --git a/test/integration/std_list.toml b/test/integration/std_list.toml
new file mode 100644
index 0000000..dfe2d8f
--- /dev/null
+++ b/test/integration/std_list.toml
@@ -0,0 +1,59 @@
+includes = ["list"]
+
+definitions = '''
+  struct SimpleStruct {
+    int a;
+    char b;
+    long long c;
+  };
+'''
+
+[cases]
+  [cases.int_empty]
+    param_types = ["const std::list<int>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+  [cases.int_some]
+    param_types = ["const std::list<int>&"]
+    setup = "return {{1,2,3}};"
+    expect_json = '[{"staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}]'
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
+      {"staticSize":4, "exclusiveSize":4},
+      {"staticSize":4, "exclusiveSize":4},
+      {"staticSize":4, "exclusiveSize":4}
+    ]}]'''
+  [cases.struct_some]
+    param_types = ["const std::list<SimpleStruct>&"]
+    setup = "return {{{}, {}, {}}};"
+    expect_json = '[{"staticSize":24, "dynamicSize":48, "length":3, "capacity":3, "elementStaticSize":16}]'
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
+      {"staticSize":16, "exclusiveSize":3},
+      {"staticSize":16, "exclusiveSize":3},
+      {"staticSize":16, "exclusiveSize":3}
+    ]}]'''
+  [cases.list_int_empty]
+    param_types = ["const std::list<std::list<int>>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":24}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+  [cases.list_int_some]
+    param_types = ["const std::list<std::list<int>>&"]
+    setup = "return {{{1,2,3},{4},{5,6}}};"
+    expect_json = '''[{
+      "staticSize":24,
+      "dynamicSize":96,
+      "exclusiveSize":24,
+      "length":3,
+      "capacity":3,
+      "elementStaticSize":24,
+      "members":[
+        {"staticSize":24, "dynamicSize":12, "exclusiveSize":36, "length":3, "capacity":3, "elementStaticSize":4},
+        {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4},
+        {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
+      ]}]'''
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[
+      {"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[]},
+      {"staticSize":24, "exclusiveSize":24, "length":1, "capacity": 1, "members":[]},
+      {"staticSize":24, "exclusiveSize":24, "length":2, "capacity": 2, "members":[]}
+    ]}]'''
diff --git a/test/integration/std_list_del_allocator.toml b/test/integration/std_list_del_allocator.toml
index a85668d..57e7b07 100644
--- a/test/integration/std_list_del_allocator.toml
+++ b/test/integration/std_list_del_allocator.toml
@@ -27,7 +27,6 @@ includes = ["list"]
 
 [cases]
   [cases.a]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/315
     param_types = ["const Foo&"]
     setup = '''
       Foo foo;
@@ -58,3 +57,10 @@ includes = ["list"]
               ]}
           ]}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize": 48,
+      "exclusiveSize": 0,
+      "members": [
+        {"name": "v1", "staticSize": 24, "exclusiveSize": 24, "length": 1, "capacity": 1},
+        {"name": "v2", "staticSize": 24, "exclusiveSize": 24, "length": 2, "capacity": 2}
+      ]}]'''
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index 437846c..fb54d22 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -1,10 +1,10 @@
 [info]
 type_name = "std::__cxx11::list"
 stub_template_params = [1]
-ctype = "LIST_TYPE"
 header = "list"
 
 # Old:
+ctype = "LIST_TYPE"
 typeName = "std::__cxx11::list"
 ns = ["namespace std"]
 numTemplateParams = 1
@@ -33,28 +33,53 @@ void getSizeType(const %1%<T, Allocator> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1>
-struct TypeHandler<DB, %1% <T0, T1>> {
-  using type = types::st::Pair<DB,
-        types::st::VarInt<DB>,
-        types::st::List<DB, typename TypeHandler<DB, T0>::type>>;
+traversal_func = """
+auto tail = returnArg.write((uintptr_t)&container)
+                .write(container.size());
 
-  static types::st::Unit<DB> getSizeType(
-      const %1% <T0, T1> & container,
-      typename TypeHandler<DB, %1% <T0, T1>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container)
-                    .write(container.size());
+for (auto&& it : container) {
+  tail = tail.delegate([&it](auto ret) {
+    return OIInternal::getSizeType<DB>(it, ret);
+  });
+}
 
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
+return tail.finish();
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.pointer = std::get<ParsedData::VarInt>(d.val).value;
+"""
+
+[[codegen.processor]]
+type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+func = """
+#ifdef __GLIBCXX__
+static constexpr size_t element_size = sizeof(std::_List_node<T0>);
+#else
+static_assert(false && "No known element_size for list. See types/cxx11_list_type.toml");
+#endif
+
+static constexpr std::array<inst::Field, 1> child_field{
+  make_field<DB, T0>("*"),
+};
+static constexpr inst::Field element{
+  element_size,
+  element_size - sizeof(T0),
+  "[]",
+  std::array<std::string_view, 0>{},
+  child_field,
+  std::array<inst::ProcessorInst, 0>{},
+};
+static constexpr auto childField = make_field<DB, T0>("[]");
+
+auto list = std::get<ParsedData::List>(d.val);
+el.container_stats.emplace(result::Element::ContainerStats{
+  .capacity = list.length,
+  .length = list.length,
+});
+el.exclusive_size += (el.container_stats->capacity - el.container_stats->length) * sizeof(T0);
+
+stack_ins(inst::Repeat{ list.length, childField });
 """
diff --git a/types/list_type.toml b/types/list_type.toml
index 78ef5c5..9ea8ef7 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -33,28 +33,53 @@ void getSizeType(const %1%<T, Allocator> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1>
-struct TypeHandler<DB, %1% <T0, T1>> {
-  using type = types::st::Pair<DB,
-        types::st::VarInt<DB>,
-        types::st::List<DB, typename TypeHandler<DB, T0>::type>>;
+traversal_func = """
+auto tail = returnArg.write((uintptr_t)&container)
+                .write(container.size());
 
-  static types::st::Unit<DB> getSizeType(
-      const %1% <T0, T1> & container,
-      typename TypeHandler<DB, %1% <T0, T1>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container)
-                    .write(container.size());
+for (auto&& it : container) {
+  tail = tail.delegate([&it](auto ret) {
+    return OIInternal::getSizeType<DB>(it, ret);
+  });
+}
 
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
+return tail.finish();
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.pointer = std::get<ParsedData::VarInt>(d.val).value;
+"""
+
+[[codegen.processor]]
+type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+func = """
+#ifdef __GLIBCXX__
+static constexpr size_t element_size = sizeof(std::_List_node<T0>);
+#else
+static_assert(false && "No known element_size for list. See types/cxx11_list_type.toml");
+#endif
+
+static constexpr std::array<inst::Field, 1> child_field{
+  make_field<DB, T0>("*"),
+};
+static constexpr inst::Field element{
+  element_size,
+  element_size - sizeof(T0),
+  "[]",
+  std::array<std::string_view, 0>{},
+  child_field,
+  std::array<inst::ProcessorInst, 0>{},
+};
+static constexpr auto childField = make_field<DB, T0>("[]");
+
+auto list = std::get<ParsedData::List>(d.val);
+el.container_stats.emplace(result::Element::ContainerStats{
+  .capacity = list.length,
+  .length = list.length,
+});
+el.exclusive_size += (el.container_stats->capacity - el.container_stats->length) * sizeof(T0);
+
+stack_ins(inst::Repeat{ list.length, childField });
 """

From 6c972e6f3145f05541519e1f12b1ab593cebd000 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 14 Nov 2023 09:12:14 -0800
Subject: [PATCH 020/188] types: remove now unused handlers

Summary:

Handlers were added in an intermediate form of tbv2 but those intermediate
forms have now been removed. Remove all the handlers to make grepping/find and
replaces easier across the types.

Test Plan:

- CI
---
 types/array_type.toml                         | 21 -----------
 types/cxx11_string_type.toml                  | 18 ----------
 types/deque_list_type.toml                    | 25 -------------
 types/f14_fast_map.toml                       | 34 ------------------
 types/f14_fast_set.toml                       | 30 ----------------
 types/f14_node_map.toml                       | 34 ------------------
 types/f14_node_set.toml                       | 30 ----------------
 types/f14_value_map.toml                      | 34 ------------------
 types/f14_value_set.toml                      | 30 ----------------
 types/f14_vector_map.toml                     | 34 ------------------
 types/f14_vector_set.toml                     | 30 ----------------
 types/fb_string_type.toml                     | 32 -----------------
 types/map_seq_type.toml                       | 31 ----------------
 types/multi_map_type.toml                     | 28 ---------------
 types/multi_set_type.toml                     | 25 -------------
 types/optional_type.toml                      | 22 ------------
 types/pair_type.toml                          | 20 -----------
 ...priority_queue_container_adapter_type.toml | 18 ----------
 types/queue_container_adapter_type.toml       | 17 ---------
 types/ref_wrapper_type.toml                   | 26 --------------
 types/seq_type.toml                           | 29 ---------------
 types/set_type.toml                           | 28 ---------------
 types/shrd_ptr_type.toml                      | 32 -----------------
 types/small_vec_type.toml                     | 27 --------------
 types/sorted_vec_set_type.toml                | 17 ---------
 types/stack_container_adapter_type.toml       | 18 ----------
 types/std_map_type.toml                       | 32 -----------------
 types/std_unordered_map_type.toml             | 36 -------------------
 types/std_unordered_multimap_type.toml        | 36 -------------------
 types/std_variant.toml                        | 32 ++---------------
 types/thrift_isset_type.toml                  |  3 --
 types/uniq_ptr_type.toml                      | 32 -----------------
 types/unordered_multiset_type.toml            | 31 ----------------
 types/unordered_set_type.toml                 | 31 ----------------
 types/weak_ptr_type.toml                      | 13 -------
 35 files changed, 2 insertions(+), 934 deletions(-)

diff --git a/types/array_type.toml b/types/array_type.toml
index 92e72ca..6d73322 100644
--- a/types/array_type.toml
+++ b/types/array_type.toml
@@ -29,27 +29,6 @@ void getSizeType(const %1%<T,N> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template<typename DB, typename T0, long unsigned int N>
-struct TypeHandler<DB, %1%<T0, N>> {
-  using type = types::st::List<DB, typename TypeHandler<DB, T0>::type>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, N> &container,
-      typename TypeHandler<DB, %1%<T0,N>>::type returnArg) {
-    auto tail = returnArg.write(container.size());
-
-    for (auto & it: container) {
-        tail = tail.delegate([&it](auto ret) {
-            return TypeHandler<DB, T0>::getSizeType(it, ret);
-        });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
     auto tail = returnArg.write(container.size());
 
diff --git a/types/cxx11_string_type.toml b/types/cxx11_string_type.toml
index 1ecc931..94e7183 100644
--- a/types/cxx11_string_type.toml
+++ b/types/cxx11_string_type.toml
@@ -36,24 +36,6 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0>
-struct TypeHandler<DB, %1% <T0>> {
-  using type =
-      types::st::Pair<DB, types::st::VarInt<DB>, types::st::VarInt<DB>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1% <T0> & container,
-      typename TypeHandler<DB, %1% <T0>>::type returnArg) {
-    bool sso = ((uintptr_t)container.data() <
-                (uintptr_t)(&container + sizeof(%1% <T0>))) &&
-               ((uintptr_t)container.data() >= (uintptr_t)&container);
-
-    return returnArg.write(container.capacity()).write(container.size());
-  }
-};
-"""
-
 extra = """
 template <typename DB, typename CharT, typename Traits, typename Allocator>
 class CaptureKeyHandler<DB, std::__cxx11::basic_string<CharT, Traits, Allocator>> {
diff --git a/types/deque_list_type.toml b/types/deque_list_type.toml
index eb0015d..a59742a 100644
--- a/types/deque_list_type.toml
+++ b/types/deque_list_type.toml
@@ -33,28 +33,3 @@ void getSizeType(const %1%<T, Allocator> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1>
-struct TypeHandler<DB, %1%<T0, T1>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB, typename TypeHandler<DB, T0>::type>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1>& container,
-      typename TypeHandler<DB, %1%<T0, T1>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container)
-                    .write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return TypeHandler<DB, T0>::getSizeType(it, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index 87d8d87..c8c84f2 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -36,40 +36,6 @@ void getSizeType(const %1%<Key, Mapped, Hasher, KeyEqual, Alloc> &container, siz
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3, typename T4>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3, T4>> {
-  using type = types::st::Pair<DB,
-    types::st::VarInt<DB>,
-    types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB, types::st::Pair<DB,
-        typename TypeHandler<DB, T0>::type,
-        typename TypeHandler<DB, T1>::type>>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3, T4>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3, T4>>::type returnArg) {
-    size_t memorySize = container.getAllocatedMemorySize();
-    auto tail = returnArg
-      .write(memorySize)
-      .write(container.bucket_count())
-      .write(container.size());
-
-    for (auto &&entry: container) {
-      tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-        auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-          return OIInternal::getSizeType<DB>(key, ret);
-        });
-        return OIInternal::getSizeType<DB>(value, next);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)container.getAllocatedMemorySize())
diff --git a/types/f14_fast_set.toml b/types/f14_fast_set.toml
index 63262c5..8516f8c 100644
--- a/types/f14_fast_set.toml
+++ b/types/f14_fast_set.toml
@@ -35,36 +35,6 @@ void getSizeType(const %1%<Key, Hasher, KeyEqual, Alloc> &container, size_t& ret
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
-  using type = types::st::Pair<DB,
-    types::st::VarInt<DB>,
-    types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB,
-        typename TypeHandler<DB, T0>::type>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3>>::type returnArg) {
-    size_t memorySize = container.getAllocatedMemorySize();
-    auto tail = returnArg
-      .write(memorySize)
-      .write(container.bucket_count())
-      .write(container.size());
-
-    for (auto &&entry: container) {
-      tail = tail.delegate([&entry](auto ret) {
-        return OIInternal::getSizeType<DB>(entry, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)container.getAllocatedMemorySize())
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index 4d483e6..93b53e9 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -36,40 +36,6 @@ void getSizeType(const %1%<Key, Mapped, Hasher, KeyEqual, Alloc> &container, siz
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3, typename T4>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3, T4>> {
-  using type = types::st::Pair<DB,
-    types::st::VarInt<DB>,
-    types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB, types::st::Pair<DB,
-        typename TypeHandler<DB, T0>::type,
-        typename TypeHandler<DB, T1>::type>>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3, T4>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3, T4>>::type returnArg) {
-    size_t memorySize = container.getAllocatedMemorySize();
-    auto tail = returnArg
-      .write(memorySize)
-      .write(container.bucket_count())
-      .write(container.size());
-
-    for (auto &&entry: container) {
-      tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-        auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-          return OIInternal::getSizeType<DB>(key, ret);
-        });
-        return OIInternal::getSizeType<DB>(value, next);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)container.getAllocatedMemorySize())
diff --git a/types/f14_node_set.toml b/types/f14_node_set.toml
index 8413cf0..511c2f8 100644
--- a/types/f14_node_set.toml
+++ b/types/f14_node_set.toml
@@ -35,36 +35,6 @@ void getSizeType(const %1%<Key, Hasher, KeyEqual, Alloc> &container, size_t& ret
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
-  using type = types::st::Pair<DB,
-    types::st::VarInt<DB>,
-    types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB,
-        typename TypeHandler<DB, T0>::type>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3>>::type returnArg) {
-    size_t memorySize = container.getAllocatedMemorySize();
-    auto tail = returnArg
-      .write(memorySize)
-      .write(container.bucket_count())
-      .write(container.size());
-
-    for (auto &&entry: container) {
-      tail = tail.delegate([&entry](auto ret) {
-        return OIInternal::getSizeType<DB>(entry, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)container.getAllocatedMemorySize())
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index ed569de..ec4f187 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -36,40 +36,6 @@ void getSizeType(const %1%<Key, Mapped, Hasher, KeyEqual, Alloc> &container, siz
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3, typename T4>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3, T4>> {
-  using type = types::st::Pair<DB,
-    types::st::VarInt<DB>,
-    types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB, types::st::Pair<DB,
-        typename TypeHandler<DB, T0>::type,
-        typename TypeHandler<DB, T1>::type>>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3, T4>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3, T4>>::type returnArg) {
-    size_t memorySize = container.getAllocatedMemorySize();
-    auto tail = returnArg
-      .write(memorySize)
-      .write(container.bucket_count())
-      .write(container.size());
-
-    for (auto &&entry: container) {
-      tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-        auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-          return OIInternal::getSizeType<DB>(key, ret);
-        });
-        return OIInternal::getSizeType<DB>(value, next);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)container.getAllocatedMemorySize())
diff --git a/types/f14_value_set.toml b/types/f14_value_set.toml
index 07887e7..20aae30 100644
--- a/types/f14_value_set.toml
+++ b/types/f14_value_set.toml
@@ -35,36 +35,6 @@ void getSizeType(const %1%<Key, Hasher, KeyEqual, Alloc> &container, size_t& ret
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
-  using type = types::st::Pair<DB,
-    types::st::VarInt<DB>,
-    types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB,
-        typename TypeHandler<DB, T0>::type>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3>>::type returnArg) {
-    size_t memorySize = container.getAllocatedMemorySize();
-    auto tail = returnArg
-      .write(memorySize)
-      .write(container.bucket_count())
-      .write(container.size());
-
-    for (auto &&entry: container) {
-      tail = tail.delegate([&entry](auto ret) {
-        return OIInternal::getSizeType<DB>(entry, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)container.getAllocatedMemorySize())
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index 9287d1b..5e29d82 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -36,40 +36,6 @@ void getSizeType(const %1%<Key, Mapped, Hasher, KeyEqual, Alloc> &container, siz
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3, typename T4>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3, T4>> {
-  using type = types::st::Pair<DB,
-    types::st::VarInt<DB>,
-    types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB, types::st::Pair<DB,
-        typename TypeHandler<DB, T0>::type,
-        typename TypeHandler<DB, T1>::type>>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3, T4>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3, T4>>::type returnArg) {
-    size_t memorySize = container.getAllocatedMemorySize();
-    auto tail = returnArg
-      .write(memorySize)
-      .write(container.bucket_count())
-      .write(container.size());
-
-    for (auto &&entry: container) {
-      tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-        auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-          return OIInternal::getSizeType<DB>(key, ret);
-        });
-        return OIInternal::getSizeType<DB>(value, next);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)container.getAllocatedMemorySize())
diff --git a/types/f14_vector_set.toml b/types/f14_vector_set.toml
index e94b8b9..1aed210 100644
--- a/types/f14_vector_set.toml
+++ b/types/f14_vector_set.toml
@@ -35,36 +35,6 @@ void getSizeType(const %1%<Key, Hasher, KeyEqual, Alloc> &container, size_t& ret
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
-  using type = types::st::Pair<DB,
-    types::st::VarInt<DB>,
-    types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB,
-        typename TypeHandler<DB, T0>::type>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3>>::type returnArg) {
-    size_t memorySize = container.getAllocatedMemorySize();
-    auto tail = returnArg
-      .write(memorySize)
-      .write(container.bucket_count())
-      .write(container.size());
-
-    for (auto &&entry: container) {
-      tail = tail.delegate([&entry](auto ret) {
-        return OIInternal::getSizeType<DB>(entry, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)container.getAllocatedMemorySize())
diff --git a/types/fb_string_type.toml b/types/fb_string_type.toml
index 19b9f34..e0679d3 100644
--- a/types/fb_string_type.toml
+++ b/types/fb_string_type.toml
@@ -38,38 +38,6 @@ void getSizeType(const %1%<E, T, A, Storage> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::Pair<DB,
-        types::st::VarInt<DB>,
-        types::st::Pair<DB,
-          types::st::VarInt<DB>,
-          types::st::VarInt<DB>
-      >>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3>>::type returnArg) {
-    auto last = returnArg.write((uintptr_t)container.data())
-                    .write(container.capacity())
-                    .write(container.size());
-
-    bool inlined = ((uintptr_t)container.data() < (uintptr_t)(&container + sizeof(%1%<T0, T1, T2, T3>)))
-        &&
-      ((uintptr_t)container.data() >= (uintptr_t)&container);
-
-    if (!inlined && pointers.add((uintptr_t)container.data())) {
-      return last.write(1);
-    } else {
-      return last.write(0);
-    }
-  }
-};
-"""
-
 traversal_func = """
 // fbstring has inlining (SSO) and allocates large strings as
 // reference counted strings. Reference counted strings have an
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index 16b9615..90d424d 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -35,37 +35,6 @@ void getSizeType(const %1%<Key, Value, Compare, Allocator, GrowthPolicy, Contain
     }
 }
 """
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3, typename T4, typename T5>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3, T4, T5>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::Pair<DB,
-        types::st::VarInt<DB>,
-        types::st::List<DB, types::st::Pair<DB,
-          typename TypeHandler<DB, T0>::type,
-          typename TypeHandler<DB, T1>::type
-    >>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3, T4, T5>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3, T4, T5>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container)
-                    .write(container.capacity())
-                    .write(container.size());
-
-    for (const auto& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it.second, ret.delegate([&it](auto ret) {
-            return OIInternal::getSizeType<DB>(it.first, ret);
-        }));
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
 
 traversal_func = '''
     auto tail = returnArg.write((uintptr_t)&container)
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index f29eb8d..7b1e622 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -33,34 +33,6 @@ void getSizeType(const %1%<Key,T,Compare,Allocator> &container, size_t& returnAr
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
-  using type = types::st::List<DB, types::st::Pair<DB,
-      typename TypeHandler<DB, T0>::type,
-      typename TypeHandler<DB, T1>::type
-    >>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3>>::type returnArg) {
-    auto tail = returnArg.write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it.second, ret.delegate([&it](auto ret) {
-            return OIInternal::getSizeType<DB>(it.first, ret);
-        }));
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)&container)
diff --git a/types/multi_set_type.toml b/types/multi_set_type.toml
index aab9fd7..34163e8 100644
--- a/types/multi_set_type.toml
+++ b/types/multi_set_type.toml
@@ -35,31 +35,6 @@ void getSizeType(const %1%<Key, Compare, Alloc> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2>
-struct TypeHandler<DB, %1% <T0, T1, T2>> {
-  using type = types::st::List<DB, typename TypeHandler<DB, T0>::type>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2>>::type returnArg) {
-    constexpr size_t nodeSize = sizeof(typename %1%<T0, T1, T2>::node_type);
-
-    auto tail = returnArg.write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
diff --git a/types/optional_type.toml b/types/optional_type.toml
index d153537..2178743 100644
--- a/types/optional_type.toml
+++ b/types/optional_type.toml
@@ -28,28 +28,6 @@ void getSizeType(const %1%<T>& container, size_t& returnArg) {
 }
 """
 
-handler = """
-template <typename DB, typename T0>
-struct TypeHandler<DB, %1%<T0>> {
-    using type = types::st::Sum<DB,
-      types::st::Unit<DB>,
-      typename TypeHandler<DB, T0>::type
-    >;
-
-    static types::st::Unit<DB> getSizeType(
-        const %1%<T0>& container,
-        typename TypeHandler<DB, %1%<T0>>::type returnArg) {
-      if (container) {
-        return returnArg.template delegate<1>([&container](auto ret) {
-          return OIInternal::getSizeType<DB>(*container, ret);
-        });
-      } else {
-        return returnArg.template delegate<0>(std::identity());
-      }
-    }
-};
-"""
-
 traversal_func = """
 if (container.has_value()) {
   return returnArg.template delegate<1>([&container](auto ret) {
diff --git a/types/pair_type.toml b/types/pair_type.toml
index e904db8..570d289 100644
--- a/types/pair_type.toml
+++ b/types/pair_type.toml
@@ -26,26 +26,6 @@ void getSizeType(const %1%<P,Q> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1>
-struct TypeHandler<DB, %1%<T0, T1>> {
-  using type = types::st::Pair<DB,
-      typename TypeHandler<DB, T0>::type,
-      typename TypeHandler<DB, T1>::type>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1> & container,
-      typename TypeHandler<DB, %1%<T0, T1>>::type returnArg) {
-    return OIInternal::getSizeType<DB>(
-        container.second,
-        returnArg.delegate([&container](auto ret) {
-            return OIInternal::getSizeType<DB>(container.first, ret);
-        })
-    );
-  }
-};
-"""
-
 traversal_func = """
     return OIInternal::getSizeType<DB>(
         container.second,
diff --git a/types/priority_queue_container_adapter_type.toml b/types/priority_queue_container_adapter_type.toml
index cc0b33c..7bdebb6 100644
--- a/types/priority_queue_container_adapter_type.toml
+++ b/types/priority_queue_container_adapter_type.toml
@@ -32,21 +32,3 @@ void getSizeType(const %1%<T, Container, Cmp> &containerAdapter, size_t& returnA
     getSizeType(container, returnArg);
 }
 """
-
-handler = """
-template <typename DB, typename T0, typename T1, typename T2>
-struct TypeHandler<DB, %1%<T0, T1, T2>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      typename TypeHandler<DB, T1>::type>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container);
-
-    const T1 &underlyingContainer = get_container(container);
-    return OIInternal::getSizeType<DB>(underlyingContainer, tail);
-  }
-};
-"""
diff --git a/types/queue_container_adapter_type.toml b/types/queue_container_adapter_type.toml
index 10891f8..7870901 100644
--- a/types/queue_container_adapter_type.toml
+++ b/types/queue_container_adapter_type.toml
@@ -32,20 +32,3 @@ void getSizeType(const %1%<T, Container> &containerAdapter, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1>
-struct TypeHandler<DB, %1%<T0, T1>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      typename TypeHandler<DB, T1>::type>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1% <T0, T1> & container,
-      typename TypeHandler<DB, %1% <T0, T1>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container);
-
-    const T1 &underlyingContainer = get_container(container);
-    return OIInternal::getSizeType<DB>(underlyingContainer, tail);
-  }
-};
-"""
diff --git a/types/ref_wrapper_type.toml b/types/ref_wrapper_type.toml
index 9c87b2a..949b385 100644
--- a/types/ref_wrapper_type.toml
+++ b/types/ref_wrapper_type.toml
@@ -29,29 +29,3 @@ void getSizeType(const %1%<T> &ref, size_t& returnArg)
     }
 }
 """
-
-handler = """
-template <typename DB, typename T0>
-struct TypeHandler<DB, %1%<T0>> {
-    using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::Sum<DB,
-        types::st::Unit<DB>,
-        typename TypeHandler<DB, T0>::type
-    >>;
-
-    static types::st::Unit<DB> getSizeType(
-        const %1%<T0>& container,
-        typename TypeHandler<DB, %1%<T0>>::type returnArg) {
-      auto r0 = returnArg.write((uintptr_t)&(container.get()));
-
-      if (pointers.add((uintptr_t)&container.get())) {
-        return r0.template delegate<1>([&container](auto ret) {
-          return OIInternal::getSizeType<DB>(container.get(), ret);
-        });
-      } else {
-        return r0.template delegate<0>(std::identity());
-      }
-    }
-};
-"""
diff --git a/types/seq_type.toml b/types/seq_type.toml
index 1da8fd6..0ccfee8 100644
--- a/types/seq_type.toml
+++ b/types/seq_type.toml
@@ -35,35 +35,6 @@ void getSizeType(const %1%<T, Allocator> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1>
-struct TypeHandler<DB, %1%<T0, T1>> {
-  using type = types::st::Pair<
-      DB, types::st::VarInt<DB>,
-      types::st::Pair<
-          DB, types::st::VarInt<DB>,
-          types::st::List<DB, typename TypeHandler<DB, T0>::type>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1> & container,
-      typename TypeHandler<DB, %1%<T0, T1>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container)
-                    .write(container.capacity())
-                    .write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.capacity())
diff --git a/types/set_type.toml b/types/set_type.toml
index bd22ad6..43206f6 100644
--- a/types/set_type.toml
+++ b/types/set_type.toml
@@ -36,34 +36,6 @@ void getSizeType(const %1%<Key, Compare, Alloc> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2>
-struct TypeHandler<DB, %1% <T0, T1, T2>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB, typename TypeHandler<DB, T0>::type>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2>>::type returnArg) {
-    constexpr size_t nodeSize = sizeof(typename %1%<T0, T1, T2>::node_type);
-
-    auto tail = returnArg.write(nodeSize)
-                    .write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 3154fe4..693f871 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -34,38 +34,6 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0>
-struct TypeHandler<DB, %1%<T0>> {
-    using type = typename std::conditional<
-      std::is_void<T0>::value,
-      types::st::Unit<DB>,
-      types::st::Pair<DB,
-        types::st::VarInt<DB>,
-        types::st::Sum<DB,
-          types::st::Unit<DB>,
-          typename TypeHandler<DB, T0>::type
-      >>>::type;
-
-    static types::st::Unit<DB> getSizeType(
-        const %1%<T0>& container,
-        typename TypeHandler<DB, %1%<T0>>::type returnArg) {
-      if constexpr (!std::is_void<T0>::value) {
-        auto r0 = returnArg.write((uintptr_t)(container.get()));
-        if (container && pointers.add((uintptr_t)(container.get()))) {
-          return r0.template delegate<1>([&container](auto ret) {
-            return OIInternal::getSizeType<DB>(*(container.get()), ret);
-          });
-        } else {
-          return r0.template delegate<0>(std::identity());
-        }
-      } else {
-        return returnArg;
-      }
-    }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
diff --git a/types/small_vec_type.toml b/types/small_vec_type.toml
index 0bdd1dd..aca3e79 100644
--- a/types/small_vec_type.toml
+++ b/types/small_vec_type.toml
@@ -42,33 +42,6 @@ void getSizeType(const %1%<V, N, P> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, std::size_t N0, typename T1>
-struct TypeHandler<DB, %1%<T0, N0, T1>> {
-  using type = types::st::Pair<
-      DB, types::st::VarInt<DB>,
-      types::st::Pair<
-          DB, types::st::VarInt<DB>,
-          types::st::List<DB, typename TypeHandler<DB, T0>::type>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, N0, T1> & container,
-      typename TypeHandler<DB, %1%<T0, N0, T1>>::type returnArg) {
-    auto tail = returnArg.write(N0)
-                    .write(container.capacity())
-                    .write(container.size());
-
-    for (auto& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 // If `container.data()` pointer is within the container struct,
 // then the container's storage is inlined and doesn't uses the heap.
diff --git a/types/sorted_vec_set_type.toml b/types/sorted_vec_set_type.toml
index bc44219..f17a0f7 100644
--- a/types/sorted_vec_set_type.toml
+++ b/types/sorted_vec_set_type.toml
@@ -31,23 +31,6 @@ void getSizeType(const %1%<T,Compare, Allocator, GrowthPolicy, Container> &conta
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3, typename T4>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3, T4>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      typename TypeHandler<DB, T4>::type>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3, T4>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3, T4>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container);
-    const T4 &underlyingContainer = container.get_container();
-    return OIInternal::getSizeType<DB>(underlyingContainer, tail);
-  }
-};
-"""
-
 traversal_func = '''
 auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.capacity())
diff --git a/types/stack_container_adapter_type.toml b/types/stack_container_adapter_type.toml
index 67187bb..db91880 100644
--- a/types/stack_container_adapter_type.toml
+++ b/types/stack_container_adapter_type.toml
@@ -31,21 +31,3 @@ void getSizeType(const %1%<T, Container> &containerAdapter, size_t& returnArg)
     getSizeType(container, returnArg);
 }
 """
-
-handler = """
-template <typename DB, typename T0, typename T1>
-struct TypeHandler<DB, %1%<T0, T1>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      typename TypeHandler<DB, T1>::type>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1% <T0, T1> & container,
-      typename TypeHandler<DB, %1% <T0, T1>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container);
-
-    const T1 &underlyingContainer = get_container(container);
-    return OIInternal::getSizeType<DB>(underlyingContainer, tail);
-  }
-};
-"""
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index ecceb36..61be6be 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -37,38 +37,6 @@ void getSizeType(const %1%<K, T, C, A> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB, types::st::Pair<DB,
-        typename TypeHandler<DB, T0>::type,
-        typename TypeHandler<DB, T1>::type
-    >>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3>>::type returnArg) {
-    constexpr size_t nodeSize = sizeof(typename %1%<T0, T1, T2, T3>::node_type);
-
-    auto tail = returnArg.write(nodeSize).write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (const auto& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it.second, ret.delegate([&it](auto ret) {
-            return OIInternal::getSizeType<DB>(it.first, ret);
-        }));
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)&container)
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index b7f99d7..541576d 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -39,42 +39,6 @@ void getSizeType(const %1%<K, T, H, KE, A> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3, typename T4>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3, T4>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::Pair<DB,
-        types::st::VarInt<DB>,
-        types::st::List<DB, types::st::Pair<DB,
-          typename TypeHandler<DB, T0>::type,
-          typename TypeHandler<DB, T1>::type
-    >>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3, T4>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3, T4>>::type returnArg) {
-    constexpr size_t nodeSize = sizeof(typename %1%<T0, T1, T2, T3, T4>::node_type);
-
-    auto tail = returnArg.write(nodeSize)
-                    .write(container.bucket_count())
-                    .write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (const auto& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it.second, ret.delegate([&it](auto ret) {
-            return OIInternal::getSizeType<DB>(it.first, ret);
-        }));
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)&container)
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index 2782118..a548623 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -39,42 +39,6 @@ void getSizeType(const %1%<K, T, H, KE, A> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3, typename T4>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3, T4>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::Pair<DB,
-        types::st::VarInt<DB>,
-        types::st::List<DB, types::st::Pair<DB,
-          typename TypeHandler<DB, T0>::type,
-          typename TypeHandler<DB, T1>::type
-    >>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3, T4>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3, T4>>::type returnArg) {
-    constexpr size_t nodeSize = sizeof(typename %1%<T0, T1, T2, T3, T4>::node_type);
-
-    auto tail = returnArg.write(nodeSize)
-                    .write(container.bucket_count())
-                    .write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (const auto& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it.second, ret.delegate([&it](auto ret) {
-            return OIInternal::getSizeType<DB>(it.first, ret);
-        }));
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)&container)
diff --git a/types/std_variant.toml b/types/std_variant.toml
index d8d5f9b..c7b9dcc 100644
--- a/types/std_variant.toml
+++ b/types/std_variant.toml
@@ -36,33 +36,5 @@ void getSizeType(const %1%<Types...> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename... Types>
-struct TypeHandler<DB, %1%<Types...>> {
-    using type = types::st::Sum<DB, typename TypeHandler<DB, Types>::type..., types::st::Unit<DB>>;
-
-    static types::st::Unit<DB> getSizeType(
-        const %1%<Types...>& container,
-        typename TypeHandler<DB, %1%<Types...>>::type returnArg) {
-      return getSizeTypeRecursive(container, returnArg);
-    }
-
-  private:
-    template <size_t I = 0>
-    static types::st::Unit<DB> getSizeTypeRecursive(
-        const %1%<Types...>& container,
-        typename TypeHandler<DB, %1%<Types...>>::type returnArg) {
-      if constexpr (I < sizeof...(Types)) {
-        if (I == container.index()) {
-          return returnArg.template delegate<I>([&container](auto ret) {
-            return OIInternal::getSizeType<DB>(std::get<I>(container), ret);
-          });
-        } else {
-          return getSizeTypeRecursive<I+1>(container, returnArg);
-        }
-      } else {
-        return returnArg.template delegate<sizeof...(Types)>(std::identity());
-      }
-    }
-};
-"""
+# TODO: Add tbv2 definitions. The removed intermediate handler is a good
+# template for this, find it in the git logs.
diff --git a/types/thrift_isset_type.toml b/types/thrift_isset_type.toml
index 213cb58..ce60791 100644
--- a/types/thrift_isset_type.toml
+++ b/types/thrift_isset_type.toml
@@ -17,9 +17,6 @@ decl = """
 func = """
 // DummyFunc %1%
 """
-handler = """
-// DummyHandler %1%
-"""
 
 traversal_func = """
 return returnArg;
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index eb3be12..35155f9 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -35,38 +35,6 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1>
-struct TypeHandler<DB, %1%<T0,T1>> {
-    using type = typename std::conditional<
-      std::is_void<T0>::value,
-      types::st::Unit<DB>,
-      types::st::Pair<DB,
-        types::st::VarInt<DB>,
-        types::st::Sum<DB,
-          types::st::Unit<DB>,
-          typename TypeHandler<DB, T0>::type
-      >>>::type;
-
-    static types::st::Unit<DB> getSizeType(
-        const %1%<T0,T1>& container,
-        typename TypeHandler<DB, %1%<T0,T1>>::type returnArg) {
-      if constexpr (!std::is_void<T0>::value) {
-        auto r0 = returnArg.write((uintptr_t)(container.get()));
-        if (container && pointers.add((uintptr_t)(container.get()))) {
-          return r0.template delegate<1>([&container](auto ret) {
-            return OIInternal::getSizeType<DB>(*(container.get()), ret);
-          });
-        } else {
-          return r0.template delegate<0>(std::identity());
-        }
-      } else {
-        return returnArg;
-      }
-    }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
diff --git a/types/unordered_multiset_type.toml b/types/unordered_multiset_type.toml
index bf6e86f..9d1449a 100644
--- a/types/unordered_multiset_type.toml
+++ b/types/unordered_multiset_type.toml
@@ -37,37 +37,6 @@ void getSizeType(const %1%<Key, Hasher, KeyEqual, Alloc> &container, size_t& ret
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
-  using type = types::st::Pair<
-      DB, types::st::VarInt<DB>,
-      types::st::Pair<
-          DB, types::st::VarInt<DB>,
-          types::st::List<DB, typename TypeHandler<DB, T0>::type>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3>>::type returnArg) {
-    constexpr size_t nodeSize = sizeof(typename %1%<T0, T1, T2, T3>::node_type);
-
-    auto tail = returnArg.write(nodeSize)
-                    .write(container.bucket_count())
-                    .write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)&container)
diff --git a/types/unordered_set_type.toml b/types/unordered_set_type.toml
index 8f085c4..62e0190 100644
--- a/types/unordered_set_type.toml
+++ b/types/unordered_set_type.toml
@@ -37,37 +37,6 @@ void getSizeType(const %1%<Key, Hasher, KeyEqual, Alloc> &container, size_t& ret
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
-  using type = types::st::Pair<
-      DB, types::st::VarInt<DB>,
-      types::st::Pair<
-          DB, types::st::VarInt<DB>,
-          types::st::List<DB, typename TypeHandler<DB, T0>::type>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3>>::type returnArg) {
-    constexpr size_t nodeSize = sizeof(typename %1%<T0, T1, T2, T3>::node_type);
-
-    auto tail = returnArg.write(nodeSize)
-                    .write(container.bucket_count())
-                    .write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)&container)
diff --git a/types/weak_ptr_type.toml b/types/weak_ptr_type.toml
index 83eaf56..33cb74e 100644
--- a/types/weak_ptr_type.toml
+++ b/types/weak_ptr_type.toml
@@ -24,18 +24,5 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0>
-struct TypeHandler<DB, %1%<T0>> {
-  using type = types::st::Unit<DB>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0>& container,
-      typename TypeHandler<DB, %1%<T0>>::type returnArg) {
-    return returnArg;
-  }
-};
-"""
-
 traversal_func = "return returnArg;"
 

From 4ee6fcce6ecf7e9ede811712f90094befb5c45a4 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 14 Nov 2023 10:15:01 -0800
Subject: [PATCH 021/188] tbv2: replace DB template param with Ctx

TreeBuilder v2 adds a DB template parameter to every function. This is used as
part of the static type to decide what type of DataBuffer is being used:
currently `BackInserterDataBuffer<std::vector<uint8_t>>` for OIL and it would
be `DataSegmentDataBuffer` for OID.

This change replaces the `DB` template parameter with a more general `Ctx`. Due
to issues with dependent naming it also adds a `using DB` to each `TypeHandler`
which has the same function as before. This allows us to add more "static
context" (typedefs and constants) to functions without changing this signature
again, because changing the signature of everything is a massive pain.

Currently this change achieves nothing because Ctx contains only DB in a static
wrapper. In the next change I'm going to pass a reference of type Ctx around to
add a "dynamic context" to invocations which will contain the pointer array. In
future we'll then be able to add either static or dynamic context without any
signature adjustments.

Test plan:
- CI
---
 oi/CodeGen.cpp                         | 59 ++++++++++++++------------
 oi/FuncGen.cpp                         | 45 +++++++++++---------
 types/array_type.toml                  |  6 +--
 types/cxx11_list_type.toml             |  8 ++--
 types/cxx11_string_type.toml           |  5 ++-
 types/f14_fast_map.toml                | 14 +++---
 types/f14_fast_set.toml                |  6 +--
 types/f14_node_map.toml                | 14 +++---
 types/f14_node_set.toml                |  6 +--
 types/f14_value_map.toml               | 14 +++---
 types/f14_value_set.toml               |  6 +--
 types/f14_vector_map.toml              | 14 +++---
 types/f14_vector_set.toml              |  6 +--
 types/list_type.toml                   |  8 ++--
 types/map_seq_type.toml                | 24 +++++------
 types/multi_map_type.toml              | 14 +++---
 types/multi_set_type.toml              |  6 +--
 types/optional_type.toml               |  8 ++--
 types/pair_type.toml                   | 10 ++---
 types/seq_type.toml                    |  6 +--
 types/set_type.toml                    |  6 +--
 types/shrd_ptr_type.toml               |  8 ++--
 types/small_vec_type.toml              |  8 ++--
 types/sorted_vec_set_type.toml         |  6 +--
 types/std_map_type.toml                | 24 +++++------
 types/std_unordered_map_type.toml      | 24 +++++------
 types/std_unordered_multimap_type.toml | 10 ++---
 types/uniq_ptr_type.toml               |  8 ++--
 types/unordered_multiset_type.toml     |  6 +--
 types/unordered_set_type.toml          |  6 +--
 30 files changed, 198 insertions(+), 187 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 73725e4..cfa87e7 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -651,7 +651,7 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
   code += funcName;
   code += "(\n      const ";
   code += c.name();
-  code += "& t,\n      typename TypeHandler<DB, ";
+  code += "& t,\n      typename TypeHandler<Ctx, ";
   code += c.name();
   code += ">::type returnArg) {\n";
 
@@ -683,7 +683,7 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
     } else {
       code += "delegate";
     }
-    code += "([&t](auto ret) { return OIInternal::getSizeType<DB>(t.";
+    code += "([&t](auto ret) { return OIInternal::getSizeType<Ctx>(t.";
     code += member.name;
     code += ", ret); })";
   }
@@ -695,7 +695,7 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
 }
 
 // Generate the static type for the class's representation in the data buffer.
-// For `class { int a,b,c; }` we generate (DB omitted for clarity):
+// For `class { int a,b,c; }` we generate (Ctx/DB omitted for clarity):
 // Pair<TypeHandler<int>::type,
 //   Pair<TypeHandler<int>::type,
 //     TypeHandler<int>::type
@@ -734,7 +734,7 @@ void CodeGen::genClassStaticType(const Class& c, std::string& code) {
     }
 
     code +=
-        (boost::format("typename TypeHandler<DB, decltype(%1%::%2%)>::type") %
+        (boost::format("typename TypeHandler<Ctx, decltype(%1%::%2%)>::type") %
          c.name() % member.name)
             .str();
 
@@ -787,8 +787,8 @@ void CodeGen::genClassTreeBuilderInstructions(const Class& c,
     code += "      inst::Field{sizeof(" + fullName + "), " +
             std::to_string(calculateExclusiveSize(m.type())) + ",\"" +
             m.inputName + "\", member_" + std::to_string(index) +
-            "_type_names, TypeHandler<DB, decltype(" + fullName +
-            ")>::fields, TypeHandler<DB, decltype(" + fullName +
+            "_type_names, TypeHandler<Ctx, decltype(" + fullName +
+            ")>::fields, TypeHandler<Ctx, decltype(" + fullName +
             ")>::processors},\n";
   }
   code += "  };\n";
@@ -820,10 +820,11 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) {
                    .str();
   }
 
-  code += "template <typename DB>\n";
-  code += "class TypeHandler<DB, ";
+  code += "template <typename Ctx>\n";
+  code += "class TypeHandler<Ctx, ";
   code += c.name();
   code += "> {\n";
+  code += "  using DB = typename Ctx::DataBuffer;\n";
   code += helpers;
   code += " public:\n";
   code += "  using type = ";
@@ -877,7 +878,7 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     containerWithTypes = "OICaptureKeys<" + containerWithTypes + ">";
   }
 
-  code += "template <typename DB";
+  code += "template <typename Ctx";
   types = 0, values = 0;
   for (const auto& p : templateParams) {
     if (p.value) {
@@ -889,9 +890,10 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     }
   }
   code += ">\n";
-  code += "struct TypeHandler<DB, ";
+  code += "struct TypeHandler<Ctx, ";
   code += containerWithTypes;
   code += "> {\n";
+  code += "  using DB = typename Ctx::DataBuffer;\n";
 
   if (c.captureKeys) {
     code += "  static constexpr bool captureKeys = true;\n";
@@ -922,7 +924,7 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   code += "      const ";
   code += containerWithTypes;
   code += "& container,\n";
-  code += "      typename TypeHandler<DB, ";
+  code += "      typename TypeHandler<Ctx, ";
   code += containerWithTypes;
   code += ">::type returnArg) {\n";
   code += func;  // has rubbish indentation
@@ -961,8 +963,9 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
 
 void addCaptureKeySupport(std::string& code) {
   code += R"(
-    template <typename DB, typename T>
+    template <typename Ctx, typename T>
     class CaptureKeyHandler {
+      using DB = typename Ctx::DataBuffer;
      public:
       using type = types::st::Sum<DB, types::st::VarInt<DB>, types::st::VarInt<DB>>;
 
@@ -975,24 +978,24 @@ void addCaptureKeySupport(std::string& code) {
       }
     };
 
-    template <bool CaptureKeys, typename DB, typename T>
+    template <bool CaptureKeys, typename Ctx, typename T>
     auto maybeCaptureKey(const T& key, auto returnArg) {
       if constexpr (CaptureKeys) {
         return returnArg.delegate([&key](auto ret) {
-          return CaptureKeyHandler<DB, T>::captureKey(key, ret);
+          return CaptureKeyHandler<Ctx, T>::captureKey(key, ret);
         });
       } else {
         return returnArg;
       }
     }
 
-    template <typename DB, typename T>
+    template <typename Ctx, typename T>
     static constexpr inst::ProcessorInst CaptureKeysProcessor{
-      CaptureKeyHandler<DB, T>::type::describe,
+      CaptureKeyHandler<Ctx, T>::type::describe,
       [](result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
         if constexpr (std::is_same_v<
-            typename CaptureKeyHandler<DB, T>::type,
-            types::st::List<DB, types::st::VarInt<DB>>>) {
+            typename CaptureKeyHandler<Ctx, T>::type,
+            types::st::List<typename Ctx::DataBuffer, types::st::VarInt<typename Ctx::DataBuffer>>>) {
           // String
           auto& str = el.data.emplace<std::string>();
           auto list = std::get<ParsedData::List>(d.val);
@@ -1013,11 +1016,11 @@ void addCaptureKeySupport(std::string& code) {
       }
     };
 
-    template <bool CaptureKeys, typename DB, typename T>
+    template <bool CaptureKeys, typename Ctx, typename T>
     static constexpr auto maybeCaptureKeysProcessor() {
       if constexpr (CaptureKeys) {
         return std::array<inst::ProcessorInst, 1>{
-          CaptureKeysProcessor<DB, T>,
+          CaptureKeysProcessor<Ctx, T>,
         };
       }
       else {
@@ -1034,28 +1037,28 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
     addCaptureKeySupport(code);
 
   // Provide a wrapper function, getSizeType, to infer T instead of having to
-  // explicitly specify it with TypeHandler<DB, T>::getSizeType every time.
+  // explicitly specify it with TypeHandler<Ctx, T>::getSizeType every time.
   code += R"(
-    template <typename DB, typename T>
-    types::st::Unit<DB>
-    getSizeType(const T &t, typename TypeHandler<DB, T>::type returnArg) {
+    template <typename Ctx, typename T>
+    types::st::Unit<typename Ctx::DataBuffer>
+    getSizeType(const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
       JLOG("obj @");
       JLOGPTR(&t);
-      return TypeHandler<DB, T>::getSizeType(t, returnArg);
+      return TypeHandler<Ctx, T>::getSizeType(t, returnArg);
     }
 )";
 
   if (features[Feature::TreeBuilderV2]) {
     code += R"(
-template <typename DB, typename T>
+template <typename Ctx, typename T>
 constexpr inst::Field make_field(std::string_view name) {
   return inst::Field{
     sizeof(T),
     ExclusiveSizeProvider<T>::size,
     name,
     NameProvider<T>::names,
-    TypeHandler<DB, T>::fields,
-    TypeHandler<DB, T>::processors,
+    TypeHandler<Ctx, T>::fields,
+    TypeHandler<Ctx, T>::processors,
   };
 }
 )";
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 7a1dd09..5b2ff35 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -268,11 +268,13 @@ void __attribute__((used, retain)) introspect_%2$016x(
   v.clear();
   v.reserve(4096);
 
-  using DataBufferType = DataBuffer::BackInserter<std::vector<uint8_t>>;
-  using ContentType = OIInternal::TypeHandler<DataBufferType, OIInternal::__ROOT_TYPE__>::type;
+  struct Context {
+    using DataBuffer = DataBuffer::BackInserter<std::vector<uint8_t>>;
+  };
+  using ContentType = OIInternal::TypeHandler<Context, OIInternal::__ROOT_TYPE__>::type;
 
-  ContentType ret{DataBufferType{v}};
-  OIInternal::getSizeType<DataBufferType>(t, ret);
+  ContentType ret{Context::DataBuffer{v}};
+  OIInternal::getSizeType<Context>(t, ret);
 }
 )";
 
@@ -364,6 +366,9 @@ void FuncGen::DefineTreeBuilderInstructions(
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunknown-attributes"
 namespace {
+struct FakeContext {
+  using DataBuffer = int;
+};
 const std::array<std::string_view, )";
   code += std::to_string(typeNames.size());
   code += "> typeNames";
@@ -382,8 +387,8 @@ const std::array<std::string_view, )";
   code += ", \"a0\", typeNames";
   code += typeHash;
   code +=
-      ", OIInternal::TypeHandler<int, OIInternal::__ROOT_TYPE__>::fields, "
-      "OIInternal::TypeHandler<int, OIInternal::__ROOT_TYPE__>::processors};\n";
+      ", OIInternal::TypeHandler<FakeContext, OIInternal::__ROOT_TYPE__>::fields, "
+      "OIInternal::TypeHandler<FakeContext, OIInternal::__ROOT_TYPE__>::processors};\n";
   code += "} // namespace\n";
   code +=
       "extern const exporters::inst::Inst __attribute__((used, retain)) "
@@ -595,14 +600,15 @@ class BackInserter {
  */
 void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   code += R"(
-    template <typename DB, typename T>
+    template <typename Ctx, typename T>
     struct TypeHandler {
+        using DB = typename Ctx::DataBuffer;
       private:
         static auto choose_type() {
             if constexpr(std::is_pointer_v<T>) {
                 return std::type_identity<types::st::Pair<DB,
                   types::st::VarInt<DB>,
-                  types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<DB, std::remove_pointer_t<T>>::type>
+                  types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>
                 >>();
             } else {
                 return std::type_identity<types::st::Unit<DB>>();
@@ -623,8 +629,8 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
             sizeof(T),
             "*",
             names,
-            TypeHandler<DB, T>::fields,
-            TypeHandler<DB, T>::processors,
+            TypeHandler<Ctx, T>::fields,
+            TypeHandler<Ctx, T>::processors,
           };
 
           const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
@@ -649,7 +655,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
           if constexpr(std::is_pointer_v<T>) {
             return std::array<inst::ProcessorInst, 2>{
               {types::st::VarInt<DB>::describe, &process_pointer},
-              {types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<DB, std::remove_pointer_t<T>>::type>::describe, &process_pointer_content},
+              {types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>::describe, &process_pointer_content},
             };
           } else {
             return std::array<inst::ProcessorInst, 0>{};
@@ -663,7 +669,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   code += R"(
         static types::st::Unit<DB> getSizeType(
           const T& t,
-          typename TypeHandler<DB, T>::type returnArg) {
+          typename TypeHandler<Ctx, T>::type returnArg) {
               if constexpr(std::is_pointer_v<T>) {
                 JLOG("ptr val @");
                 JLOGPTR(t);
@@ -671,7 +677,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
                 if (t && pointers.add((uintptr_t)t)) {
                   return r0.template delegate<1>([&t](auto ret) {
                     if constexpr (!std::is_void<std::remove_pointer_t<T>>::value) {
-                      return TypeHandler<DB, std::remove_pointer_t<T>>::getSizeType(*t, ret);
+                      return TypeHandler<Ctx, std::remove_pointer_t<T>>::getSizeType(*t, ret);
                     } else {
                       return ret;
                     }
@@ -687,8 +693,9 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   )";
 
   code += R"(
-    template <typename DB>
-    class TypeHandler<DB, void> {
+    template <typename Ctx>
+    class TypeHandler<Ctx, void> {
+        using DB = typename Ctx::DataBuffer;
       public:
         using type = types::st::Unit<DB>;
 )";
@@ -711,21 +718,21 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
   tail = tail.delegate([&container, i](auto ret) {
-      return TypeHandler<DB, T0>::getSizeType(container.vals[i], ret);
+      return TypeHandler<Ctx, T0>::getSizeType(container.vals[i], ret);
   });
 }
 return tail.finish();
 )";
   oiArray.codegen.processors.emplace_back(ContainerInfo::Processor{
-      .type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>",
+      .type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>",
       .func = R"(
 static constexpr std::array<std::string_view, 1> names{"TODO"};
 static constexpr auto childField = inst::Field{
   sizeof(T0),
   "[]",
   names,
-  TypeHandler<DB, T0>::fields,
-  TypeHandler<DB, T0>::processors,
+  TypeHandler<Ctx, T0>::fields,
+  TypeHandler<Ctx, T0>::processors,
 };
 
 el.exclusive_size = 0;
diff --git a/types/array_type.toml b/types/array_type.toml
index 6d73322..126af6a 100644
--- a/types/array_type.toml
+++ b/types/array_type.toml
@@ -34,7 +34,7 @@ traversal_func = """
 
     for (auto & it: container) {
         tail = tail.delegate([&it](auto ret) {
-            return TypeHandler<DB, T0>::getSizeType(it, ret);
+            return TypeHandler<Ctx, T0>::getSizeType(it, ret);
         });
     }
 
@@ -42,9 +42,9 @@ traversal_func = """
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 size_t size = std::get<ParsedData::List>(d.val).length;
 el.exclusive_size = N0 == 0 ? 1 : 0;
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index fb54d22..f838eee 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -39,7 +39,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -53,7 +53,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 static constexpr size_t element_size = sizeof(std::_List_node<T0>);
@@ -62,7 +62,7 @@ static_assert(false && "No known element_size for list. See types/cxx11_list_typ
 #endif
 
 static constexpr std::array<inst::Field, 1> child_field{
-  make_field<DB, T0>("*"),
+  make_field<Ctx, T0>("*"),
 };
 static constexpr inst::Field element{
   element_size,
@@ -72,7 +72,7 @@ static constexpr inst::Field element{
   child_field,
   std::array<inst::ProcessorInst, 0>{},
 };
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats{
diff --git a/types/cxx11_string_type.toml b/types/cxx11_string_type.toml
index 94e7183..c88debd 100644
--- a/types/cxx11_string_type.toml
+++ b/types/cxx11_string_type.toml
@@ -37,8 +37,9 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
 """
 
 extra = """
-template <typename DB, typename CharT, typename Traits, typename Allocator>
-class CaptureKeyHandler<DB, std::__cxx11::basic_string<CharT, Traits, Allocator>> {
+template <typename Ctx, typename CharT, typename Traits, typename Allocator>
+class CaptureKeyHandler<Ctx, std::__cxx11::basic_string<CharT, Traits, Allocator>> {
+  using DB = typename Ctx::DataBuffer;
  public:
   // List of characters
   using type = types::st::List<DB, types::st::VarInt<DB>>;
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index c8c84f2..faf2e9d 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -44,10 +44,10 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -69,8 +69,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 static constexpr size_t element_size = sizeof(typename container_type::value_type);
@@ -84,8 +84,8 @@ el.container_stats->length = list.length;
 el.exclusive_size += allocationSize - list.length * element_size;
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr inst::Field element{
diff --git a/types/f14_fast_set.toml b/types/f14_fast_set.toml
index 8516f8c..b66284b 100644
--- a/types/f14_fast_set.toml
+++ b/types/f14_fast_set.toml
@@ -43,7 +43,7 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<DB>(entry, ret);
+    return OIInternal::getSizeType<Ctx>(entry, ret);
   });
 }
 
@@ -64,7 +64,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 
 [[codegen.processor]]
 type = """
-types::st::List<DB, typename TypeHandler<DB, T0>::type>
+types::st::List<DB, typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto allocationSize = el.pointer.value();
@@ -75,7 +75,7 @@ el.container_stats->length = list.length;
 
 el.exclusive_size += allocationSize - list.length * sizeof(T0);
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index 93b53e9..a53c3e6 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -44,10 +44,10 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -69,8 +69,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 static constexpr size_t element_size = sizeof(typename container_type::value_type);
@@ -84,8 +84,8 @@ el.container_stats->length = list.length;
 el.exclusive_size += allocationSize - list.length * element_size;
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr inst::Field element{
diff --git a/types/f14_node_set.toml b/types/f14_node_set.toml
index 511c2f8..3ed0b5d 100644
--- a/types/f14_node_set.toml
+++ b/types/f14_node_set.toml
@@ -43,7 +43,7 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<DB>(entry, ret);
+    return OIInternal::getSizeType<Ctx>(entry, ret);
   });
 }
 
@@ -64,7 +64,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 
 [[codegen.processor]]
 type = """
-types::st::List<DB, typename TypeHandler<DB, T0>::type>
+types::st::List<DB, typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto allocationSize = el.pointer.value();
@@ -75,7 +75,7 @@ el.container_stats->length = list.length;
 
 el.exclusive_size += allocationSize - list.length * sizeof(T0);
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index ec4f187..8072abf 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -44,10 +44,10 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -69,8 +69,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 static constexpr size_t element_size = sizeof(typename container_type::value_type);
@@ -84,8 +84,8 @@ el.container_stats->length = list.length;
 el.exclusive_size += allocationSize - list.length * element_size;
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr inst::Field element{
diff --git a/types/f14_value_set.toml b/types/f14_value_set.toml
index 20aae30..dd4070d 100644
--- a/types/f14_value_set.toml
+++ b/types/f14_value_set.toml
@@ -43,7 +43,7 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<DB>(entry, ret);
+    return OIInternal::getSizeType<Ctx>(entry, ret);
   });
 }
 
@@ -64,7 +64,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 
 [[codegen.processor]]
 type = """
-types::st::List<DB, typename TypeHandler<DB, T0>::type>
+types::st::List<DB, typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto allocationSize = el.pointer.value();
@@ -75,7 +75,7 @@ el.container_stats->length = list.length;
 
 el.exclusive_size += allocationSize - list.length * sizeof(T0);
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index 5e29d82..ec7ac93 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -44,10 +44,10 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -69,8 +69,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 static constexpr size_t element_size = sizeof(typename container_type::value_type);
@@ -84,8 +84,8 @@ el.container_stats->length = list.length;
 el.exclusive_size += allocationSize - list.length * element_size;
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr inst::Field element{
diff --git a/types/f14_vector_set.toml b/types/f14_vector_set.toml
index 1aed210..e98b6d8 100644
--- a/types/f14_vector_set.toml
+++ b/types/f14_vector_set.toml
@@ -43,7 +43,7 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<DB>(entry, ret);
+    return OIInternal::getSizeType<Ctx>(entry, ret);
   });
 }
 
@@ -64,7 +64,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 
 [[codegen.processor]]
 type = """
-types::st::List<DB, typename TypeHandler<DB, T0>::type>
+types::st::List<DB, typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto allocationSize = el.pointer.value();
@@ -75,7 +75,7 @@ el.container_stats->length = list.length;
 
 el.exclusive_size += allocationSize - list.length * sizeof(T0);
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/list_type.toml b/types/list_type.toml
index 9ea8ef7..86d457e 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -39,7 +39,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -53,7 +53,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 static constexpr size_t element_size = sizeof(std::_List_node<T0>);
@@ -62,7 +62,7 @@ static_assert(false && "No known element_size for list. See types/cxx11_list_typ
 #endif
 
 static constexpr std::array<inst::Field, 1> child_field{
-  make_field<DB, T0>("*"),
+  make_field<Ctx, T0>("*"),
 };
 static constexpr inst::Field element{
   element_size,
@@ -72,7 +72,7 @@ static constexpr inst::Field element{
   child_field,
   std::array<inst::ProcessorInst, 0>{},
 };
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats{
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index 90d424d..887df6a 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -43,11 +43,11 @@ traversal_func = '''
 
     for (const auto& kv : container) {
       tail = tail.delegate([&kv](auto ret) {
-        auto start = maybeCaptureKey<captureKeys, DB, T0>(kv.first, ret);
-        auto next = start.delegate([&kv](typename TypeHandler<DB, T0>::type ret) {
-          return OIInternal::getSizeType<DB>(kv.first, ret);
+        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
+        auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
+          return OIInternal::getSizeType<Ctx>(kv.first, ret);
         });
-        return OIInternal::getSizeType<DB>(kv.second, next);
+        return OIInternal::getSizeType<Ctx>(kv.second, next);
       });
     }
 
@@ -68,22 +68,22 @@ el.container_stats.emplace(result::Element::ContainerStats{ .capacity = std::get
 type = '''
 std::conditional_t<captureKeys,
   types::st::List<DB, types::st::Pair<DB,
-    typename CaptureKeyHandler<DB, T0>::type,
+    typename CaptureKeyHandler<Ctx, T0>::type,
     types::st::Pair<DB,
-      typename TypeHandler<DB, T0>::type,
-      typename TypeHandler<DB, T1>::type>>>,
+      typename TypeHandler<Ctx, T0>::type,
+      typename TypeHandler<Ctx, T1>::type>>>,
   types::st::List<DB, types::st::Pair<DB,
-    typename TypeHandler<DB, T0>::type,
-    typename TypeHandler<DB, T1>::type>>>
+    typename TypeHandler<Ctx, T0>::type,
+    typename TypeHandler<Ctx, T1>::type>>>
 '''
 func = '''
 using element_type = std::pair<T0, T1>;
 
 static constexpr std::array<inst::Field, 2> entryFields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
-static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, DB, T0>();
+static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, Ctx, T0>();
 static constexpr auto entry = inst::Field {
   sizeof(element_type),
   sizeof(element_type) - sizeof(T0) - sizeof(T1),
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index 7b1e622..e05a1e3 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -40,10 +40,10 @@ auto tail = returnArg
 
 for (const auto &entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -57,8 +57,8 @@ func = "el.pointer = std::get<ParsedData::VarInt>(d.val).value;"
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 #ifdef __GLIBCXX__
@@ -108,8 +108,8 @@ static_assert(false && "No known element_size for sets. See types/multi_map_type
 #endif
 
 static constexpr std::array<inst::Field, 2> elementFields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 static constexpr auto element = inst::Field {
   element_size,
diff --git a/types/multi_set_type.toml b/types/multi_set_type.toml
index 34163e8..00f8239 100644
--- a/types/multi_set_type.toml
+++ b/types/multi_set_type.toml
@@ -43,7 +43,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 // vector<bool>
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -57,7 +57,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 /* We don't have access to the _Rb_tree_node struct, so we manually re-create it
@@ -103,7 +103,7 @@ static constexpr size_t element_size = sizeof(OI__tree_node);
 static_assert(false && "No known element_size for multisets. See types/multi_set_type.toml");
 #endif
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats {
diff --git a/types/optional_type.toml b/types/optional_type.toml
index 2178743..3bd9cff 100644
--- a/types/optional_type.toml
+++ b/types/optional_type.toml
@@ -31,7 +31,7 @@ void getSizeType(const %1%<T>& container, size_t& returnArg) {
 traversal_func = """
 if (container.has_value()) {
   return returnArg.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<DB>(*container, ret);
+    return OIInternal::getSizeType<Ctx>(*container, ret);
   });
 } else {
   return returnArg.template delegate<0>(std::identity());
@@ -39,15 +39,15 @@ if (container.has_value()) {
 """
 
 [[codegen.processor]]
-type = "types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<DB, T0>::type>"
+type = "types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, T0>::type>"
 func = """
 static constexpr std::array<std::string_view, 1> names{"TODO"};
 static constexpr auto elementField = inst::Field{
   sizeof(T0),
   "el",
   names,
-  TypeHandler<DB, T0>::fields,
-  TypeHandler<DB, T0>::processors,
+  TypeHandler<Ctx, T0>::fields,
+  TypeHandler<Ctx, T0>::processors,
 };
 
 auto sum = std::get<ParsedData::Sum>(d.val);
diff --git a/types/pair_type.toml b/types/pair_type.toml
index 570d289..ba5bc38 100644
--- a/types/pair_type.toml
+++ b/types/pair_type.toml
@@ -27,19 +27,19 @@ void getSizeType(const %1%<P,Q> &container, size_t& returnArg)
 """
 
 traversal_func = """
-    return OIInternal::getSizeType<DB>(
+    return OIInternal::getSizeType<Ctx>(
         container.second,
         returnArg.delegate([&container](auto ret) {
-            return OIInternal::getSizeType<DB>(container.first, ret);
+            return OIInternal::getSizeType<Ctx>(container.first, ret);
         })
     );
 """
 
 [[codegen.processor]]
-type = "types::st::Pair<DB, typename TypeHandler<DB, T0>::type, typename TypeHandler<DB, T1>::type>"
+type = "types::st::Pair<DB, typename TypeHandler<Ctx, T0>::type, typename TypeHandler<Ctx, T1>::type>"
 func = """
-static constexpr auto firstField = make_field<DB, T0>("first");
-static constexpr auto secondField = make_field<DB, T1>("second");
+static constexpr auto firstField = make_field<Ctx, T0>("first");
+static constexpr auto secondField = make_field<Ctx, T1>("second");
 
 el.exclusive_size = sizeof(std::pair<T0, T1>) - sizeof(T0) - sizeof(T1);
 stack_ins(secondField);
diff --git a/types/seq_type.toml b/types/seq_type.toml
index 0ccfee8..88ddd1d 100644
--- a/types/seq_type.toml
+++ b/types/seq_type.toml
@@ -44,7 +44,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 // vector<bool>
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -64,9 +64,9 @@ el.container_stats.emplace(result::Element::ContainerStats{ .capacity = std::get
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats->length = list.length;
diff --git a/types/set_type.toml b/types/set_type.toml
index 43206f6..e29611e 100644
--- a/types/set_type.toml
+++ b/types/set_type.toml
@@ -44,7 +44,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 // vector<bool>
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -58,7 +58,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 /* We don't have access to the _Rb_tree_node struct, so we manually re-create it
@@ -104,7 +104,7 @@ static constexpr size_t element_size = sizeof(OI__tree_node);
 static_assert(false && "No known element_size for sets. See types/set_type.toml");
 #endif
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats {
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 693f871..125fd71 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -45,7 +45,7 @@ if constexpr (std::is_void<T0>::value) {
     return tail.template delegate<0>(std::identity());
 
   return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<DB>(*container, ret);
+    return OIInternal::getSizeType<Ctx>(*container, ret);
   });
 }
 """
@@ -60,7 +60,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<DB, T0>::type>
+  typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 #ifdef __GLIBCXX__
@@ -84,10 +84,10 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = sum.index, // 0 for empty containers/void, 1 otherwise
 });
 
-// Must be in a `if constexpr` or the compiler will complain about make_field<DB, void>
+// Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
 if constexpr (!std::is_void<T0>::value) {
   if (sum.index == 1) {
-    static constexpr auto element = make_field<DB, T0>("ptr_val");
+    static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);
   }
 }
diff --git a/types/small_vec_type.toml b/types/small_vec_type.toml
index aca3e79..d129285 100644
--- a/types/small_vec_type.toml
+++ b/types/small_vec_type.toml
@@ -56,8 +56,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&it: container) {
-  tail = tail.delegate([&it](typename TypeHandler<DB, T0>::type ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+  tail = tail.delegate([&it](typename TypeHandler<Ctx, T0>::type ret) {
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -81,7 +81,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 // Reading the `uses_intern_storage` boolean that was stored in `pointer` by the processor above.
 bool uses_intern_storage = std::exchange(el.pointer.value(), (uintptr_t)0);
@@ -97,7 +97,7 @@ if (uses_intern_storage) {
   el.exclusive_size += (el.container_stats->capacity - el.container_stats->length) * sizeof(T0);
 }
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/sorted_vec_set_type.toml b/types/sorted_vec_set_type.toml
index f17a0f7..e8b9483 100644
--- a/types/sorted_vec_set_type.toml
+++ b/types/sorted_vec_set_type.toml
@@ -38,7 +38,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 
 for (const auto& el : container) {
   tail = tail.delegate([&el](auto ret) {
-    return OIInternal::getSizeType<DB>(el, ret);
+    return OIInternal::getSizeType<Ctx>(el, ret);
   });
 }
 
@@ -56,9 +56,9 @@ el.container_stats.emplace(result::Element::ContainerStats{ .capacity = std::get
 '''
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats->length = list.length;
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index 61be6be..df06cc7 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -44,11 +44,11 @@ auto tail = returnArg
 
 for (const auto &entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, DB, T0>(key, ret);
-    auto next =  start.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(key, ret);
+    auto next =  start.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -64,14 +64,14 @@ type = """
 std::conditional_t<captureKeys,
   types::st::List<DB,
     types::st::Pair<DB,
-      typename CaptureKeyHandler<DB, T0>::type,
+      typename CaptureKeyHandler<Ctx, T0>::type,
       types::st::Pair<DB,
-        typename TypeHandler<DB, T0>::type,
-        typename TypeHandler<DB, T1>::type>>>,
+        typename TypeHandler<Ctx, T0>::type,
+        typename TypeHandler<Ctx, T1>::type>>>,
   types::st::List<DB,
     types::st::Pair<DB,
-      typename TypeHandler<DB, T0>::type,
-      typename TypeHandler<DB, T1>::type>>
+      typename TypeHandler<Ctx, T0>::type,
+      typename TypeHandler<Ctx, T1>::type>>
 >
 """
 func = """
@@ -122,11 +122,11 @@ static_assert(false && "No known element_size for sets. See types/std_map_type.t
 #endif
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
-static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, DB, T0>();
+static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, Ctx, T0>();
 
 static constexpr inst::Field element{
   element_size,
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index 541576d..27142a1 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -47,11 +47,11 @@ auto tail = returnArg
 
 for (const auto& kv : container) {
   tail = tail.delegate([&kv](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, DB, T0>(kv.first, ret);
-    auto next = start.delegate([&kv](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(kv.first, ret);
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
+    auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(kv.first, ret);
     });
-    return OIInternal::getSizeType<DB>(kv.second, next);
+    return OIInternal::getSizeType<Ctx>(kv.second, next);
   });
 }
 
@@ -79,13 +79,13 @@ type = """
 std::conditional_t<captureKeys,
  types::st::List<DB,
    types::st::Pair<DB,
-     typename CaptureKeyHandler<DB, T0>::type,
+     typename CaptureKeyHandler<Ctx, T0>::type,
      types::st::Pair<DB,
-       typename TypeHandler<DB, T0>::type,
-       typename TypeHandler<DB, T1>::type>>>,
+       typename TypeHandler<Ctx, T0>::type,
+       typename TypeHandler<Ctx, T1>::type>>>,
  types::st::List<DB, types::st::Pair<DB,
-   typename TypeHandler<DB, T0>::type,
-   typename TypeHandler<DB, T1>::type>>
+   typename TypeHandler<Ctx, T0>::type,
+   typename TypeHandler<Ctx, T1>::type>>
 >
 """
 func = """
@@ -122,11 +122,11 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
-static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, DB, T0>();
+static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, Ctx, T0>();
 
 static constexpr auto element = inst::Field{
   element_size,
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index a548623..261a0be 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -47,7 +47,7 @@ auto tail = returnArg
 
 for (const auto &it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -73,8 +73,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 #ifdef __GLIBCXX__
@@ -110,8 +110,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr auto element = inst::Field{
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index 35155f9..58364cf 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -46,7 +46,7 @@ if constexpr (std::is_void<T0>::value) {
     return tail.template delegate<0>(std::identity());
 
   return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<DB>(*container, ret);
+    return OIInternal::getSizeType<Ctx>(*container, ret);
   });
 }
 """
@@ -61,7 +61,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<DB, T0>::type>
+  typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto sum = std::get<ParsedData::Sum>(d.val);
@@ -70,10 +70,10 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = sum.index, // 0 for empty containers/void, 1 otherwise
 });
 
-// Must be in a `if constexpr` or the compiler will complain about make_field<DB, void>
+// Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
 if constexpr (!std::is_void<T0>::value) {
   if (sum.index == 1) {
-    static constexpr auto element = make_field<DB, T0>("ptr_val");
+    static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);
   }
 }
diff --git a/types/unordered_multiset_type.toml b/types/unordered_multiset_type.toml
index 9d1449a..5c2e96b 100644
--- a/types/unordered_multiset_type.toml
+++ b/types/unordered_multiset_type.toml
@@ -45,7 +45,7 @@ auto tail = returnArg
 
 for (const auto &it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -69,7 +69,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 /* Use libstdc++ implementation __details to compute the size of Nodes and Buckets.
@@ -100,7 +100,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = list.length,
 });
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/unordered_set_type.toml b/types/unordered_set_type.toml
index 62e0190..f8d26c9 100644
--- a/types/unordered_set_type.toml
+++ b/types/unordered_set_type.toml
@@ -45,7 +45,7 @@ auto tail = returnArg
 
 for (const auto &it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -69,7 +69,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 /* Use libstdc++ implementation __details to compute the size of Nodes and Buckets.
@@ -100,7 +100,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = list.length,
 });
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """

From a60e9270be1d44794c58cfb677049a3cbf304702 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 15 Nov 2023 05:15:25 -0800
Subject: [PATCH 022/188] oil: use a unique pointer set per invocation

---
 oi/CodeGen.cpp                         | 24 +++++++++++++++++-------
 oi/FuncGen.cpp                         | 26 ++++++++++++++++----------
 oi/OICodeGen.cpp                       |  8 +++++++-
 oi/OITraceCode.cpp                     |  7 ++++---
 types/array_type.toml                  |  4 ++--
 types/cxx11_list_type.toml             |  4 ++--
 types/f14_fast_map.toml                |  8 ++++----
 types/f14_fast_set.toml                |  4 ++--
 types/f14_node_map.toml                |  8 ++++----
 types/f14_node_set.toml                |  4 ++--
 types/f14_value_map.toml               |  8 ++++----
 types/f14_value_set.toml               |  4 ++--
 types/f14_vector_map.toml              |  8 ++++----
 types/f14_vector_set.toml              |  4 ++--
 types/fb_string_type.toml              |  4 ++--
 types/folly_iobuf_queue_type.toml      |  2 +-
 types/list_type.toml                   |  4 ++--
 types/map_seq_type.toml                | 10 +++++-----
 types/multi_map_type.toml              |  8 ++++----
 types/multi_set_type.toml              |  4 ++--
 types/optional_type.toml               |  4 ++--
 types/pair_type.toml                   |  6 +++---
 types/ref_wrapper_type.toml            |  2 +-
 types/seq_type.toml                    |  4 ++--
 types/set_type.toml                    |  4 ++--
 types/shrd_ptr_type.toml               |  8 ++++----
 types/small_vec_type.toml              |  4 ++--
 types/sorted_vec_set_type.toml         |  4 ++--
 types/std_map_type.toml                | 10 +++++-----
 types/std_unordered_map_type.toml      | 10 +++++-----
 types/std_unordered_multimap_type.toml |  4 ++--
 types/uniq_ptr_type.toml               |  8 ++++----
 types/unordered_multiset_type.toml     |  4 ++--
 types/unordered_set_type.toml          |  4 ++--
 34 files changed, 126 insertions(+), 103 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index cfa87e7..c8737a1 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -128,8 +128,9 @@ void addIncludes(const TypeGraph& typeGraph,
     includes.emplace("oi/types/st.h");
   }
   if (features[Feature::Library]) {
-    includes.emplace("vector");
+    includes.emplace("memory");
     includes.emplace("oi/IntrospectionResult.h");
+    includes.emplace("vector");
   }
   if (features[Feature::JitTiming]) {
     includes.emplace("chrono");
@@ -428,7 +429,7 @@ void addStandardGetSizeFuncDefs(std::string& code) {
       JLOG("ptr val @");
       JLOGPTR(s_ptr);
       StoreData((uintptr_t)(s_ptr), returnArg);
-      if (s_ptr && pointers.add((uintptr_t)s_ptr)) {
+      if (s_ptr && ctx.pointers.add((uintptr_t)s_ptr)) {
         StoreData(1, returnArg);
         getSizeType(*(s_ptr), returnArg);
       } else {
@@ -649,7 +650,8 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
 
   code += "  static types::st::Unit<DB> ";
   code += funcName;
-  code += "(\n      const ";
+  code += "(\n      Ctx& ctx,\n";
+  code += "    const ";
   code += c.name();
   code += "& t,\n      typename TypeHandler<Ctx, ";
   code += c.name();
@@ -683,7 +685,7 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
     } else {
       code += "delegate";
     }
-    code += "([&t](auto ret) { return OIInternal::getSizeType<Ctx>(t.";
+    code += "([&ctx, &t](auto ret) { return OIInternal::getSizeType<Ctx>(ctx, t.";
     code += member.name;
     code += ", ret); })";
   }
@@ -921,6 +923,7 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   code += ";\n";
 
   code += "  static types::st::Unit<DB> getSizeType(\n";
+  code += "      Ctx& ctx,\n";
   code += "      const ";
   code += containerWithTypes;
   code += "& container,\n";
@@ -979,7 +982,7 @@ void addCaptureKeySupport(std::string& code) {
     };
 
     template <bool CaptureKeys, typename Ctx, typename T>
-    auto maybeCaptureKey(const T& key, auto returnArg) {
+    auto maybeCaptureKey(Ctx& ctx, const T& key, auto returnArg) {
       if constexpr (CaptureKeys) {
         return returnArg.delegate([&key](auto ret) {
           return CaptureKeyHandler<Ctx, T>::captureKey(key, ret);
@@ -1041,10 +1044,10 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
   code += R"(
     template <typename Ctx, typename T>
     types::st::Unit<typename Ctx::DataBuffer>
-    getSizeType(const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
+    getSizeType(Ctx& ctx, const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
       JLOG("obj @");
       JLOGPTR(&t);
-      return TypeHandler<Ctx, T>::getSizeType(t, returnArg);
+      return TypeHandler<Ctx, T>::getSizeType(ctx, t, returnArg);
     }
 )";
 
@@ -1233,6 +1236,13 @@ void CodeGen::generate(
   if (config_.features[Feature::CaptureThriftIsset]) {
     genDefsThrift(typeGraph, code);
   }
+  if (!config_.features[Feature::TreeBuilderV2]) {
+    code += "namespace {\n";
+    code += "static struct Context {\n";
+    code += "  PointerHashSet<(1 << 20) / sizeof(uintptr_t)> pointers;\n";
+    code += "} ctx;\n";
+    code += "} // namespace\n";
+  }
 
   /*
    * The purpose of the anonymous namespace within `OIInternal` is that
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 5b2ff35..4106aae 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -262,19 +262,24 @@ void __attribute__((used, retain)) introspect_%2$016x(
     std::vector<uint8_t>& v)
 #pragma GCC diagnostic pop
 {
-  pointers.initialize();
-  pointers.add((uintptr_t)&t);
-
   v.clear();
   v.reserve(4096);
 
+  auto pointers = std::make_unique<PointerHashSet<(1 << 20) / sizeof(uintptr_t)>>();
+  pointers->initialize();
+
   struct Context {
     using DataBuffer = DataBuffer::BackInserter<std::vector<uint8_t>>;
+
+    PointerHashSet<(1 << 20) / sizeof(uintptr_t)>& pointers;
   };
+  Context ctx{ .pointers = *pointers };
+  ctx.pointers.add((uintptr_t)&t);
+
   using ContentType = OIInternal::TypeHandler<Context, OIInternal::__ROOT_TYPE__>::type;
 
   ContentType ret{Context::DataBuffer{v}};
-  OIInternal::getSizeType<Context>(t, ret);
+  OIInternal::getSizeType<Context>(ctx, t, ret);
 }
 )";
 
@@ -319,8 +324,8 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
     func += "      const auto startTime = std::chrono::steady_clock::now();\n";
   }
   func += R"(
-      pointers.initialize();
-      pointers.add((uintptr_t)&t);
+      ctx.pointers.initialize();
+      ctx.pointers.add((uintptr_t)&t);
       auto data = reinterpret_cast<uintptr_t*>(dataBase);
 
       size_t dataSegOffset = 0;
@@ -415,7 +420,7 @@ void FuncGen::DefineTopLevelGetSizeSmartPtr(std::string& testCode,
     func += "      const auto startTime = std::chrono::steady_clock::now();\n";
   }
   func += R"(
-      pointers.initialize();
+      ctx.pointers.initialize();
       auto data = reinterpret_cast<uintptr_t*>(dataBase);
 
       size_t dataSegOffset = 0;
@@ -668,13 +673,14 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   }
   code += R"(
         static types::st::Unit<DB> getSizeType(
+          Ctx& ctx,
           const T& t,
           typename TypeHandler<Ctx, T>::type returnArg) {
               if constexpr(std::is_pointer_v<T>) {
                 JLOG("ptr val @");
                 JLOGPTR(t);
                 auto r0 = returnArg.write((uintptr_t)t);
-                if (t && pointers.add((uintptr_t)t)) {
+                if (t && ctx.pointers.add((uintptr_t)t)) {
                   return r0.template delegate<1>([&t](auto ret) {
                     if constexpr (!std::is_void<std::remove_pointer_t<T>>::value) {
                       return TypeHandler<Ctx, std::remove_pointer_t<T>>::getSizeType(*t, ret);
@@ -717,8 +723,8 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
   oiArray.codegen.traversalFunc = R"(
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
-  tail = tail.delegate([&container, i](auto ret) {
-      return TypeHandler<Ctx, T0>::getSizeType(container.vals[i], ret);
+  tail = tail.delegate([&ctx, &container, i](auto ret) {
+      return TypeHandler<Ctx, T0>::getSizeType(ctx, container.vals[i], ret);
   });
 }
 return tail.finish();
diff --git a/oi/OICodeGen.cpp b/oi/OICodeGen.cpp
index ee18429..b9cfcbb 100644
--- a/oi/OICodeGen.cpp
+++ b/oi/OICodeGen.cpp
@@ -3049,6 +3049,12 @@ bool OICodeGen::generateJitCode(std::string& code) {
     #define SAVE_DATA(val)    StoreData(val, returnArg)
   )");
 
+  code.append("namespace {\n");
+  code.append("static struct Context {\n");
+  code.append("  PointerHashSet<(1 << 20) / sizeof(uintptr_t)> pointers;\n");
+  code.append("} ctx;\n");
+  code.append("} // namespace\n");
+
   FuncGen::DefineJitLog(code, config.features);
 
   // The purpose of the anonymous namespace within `OIInternal` is that
@@ -3267,7 +3273,7 @@ bool OICodeGen::generateJitCode(std::string& code) {
       JLOG("ptr val @");
       JLOGPTR(s_ptr);
       StoreData((uintptr_t)(s_ptr), returnArg);
-      if (s_ptr && pointers.add((uintptr_t)s_ptr)) {
+      if (s_ptr && ctx.pointers.add((uintptr_t)s_ptr)) {
         StoreData(1, returnArg);
         getSizeType(*(s_ptr), returnArg);
       } else {
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index fa0beda..495c781 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -40,10 +40,11 @@ constexpr int oidMagicId = 0x01DE8;
 
 namespace {
 
-class {
+template <size_t Size>
+class PointerHashSet {
  private:
   // 1 MiB of pointers
-  std::array<uintptr_t, (1 << 20) / sizeof(uintptr_t)> data;
+  std::array<uintptr_t, Size> data;
 
   // twang_mix64 hash function, taken from Folly where it is used
   // as the default hash function for 64-bit integers
@@ -84,7 +85,7 @@ class {
   bool add(const auto* p) {
     return add((uintptr_t)p);
   }
-} static pointers;
+};
 
 }  // namespace
 
diff --git a/types/array_type.toml b/types/array_type.toml
index 126af6a..054d0c6 100644
--- a/types/array_type.toml
+++ b/types/array_type.toml
@@ -33,8 +33,8 @@ traversal_func = """
     auto tail = returnArg.write(container.size());
 
     for (auto & it: container) {
-        tail = tail.delegate([&it](auto ret) {
-            return TypeHandler<Ctx, T0>::getSizeType(it, ret);
+        tail = tail.delegate([&ctx, &it](auto ret) {
+            return TypeHandler<Ctx, T0>::getSizeType(ctx, it, ret);
         });
     }
 
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index f838eee..6788750 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -38,8 +38,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index faf2e9d..06c6f72 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_fast_set.toml b/types/f14_fast_set.toml
index b66284b..fb0017c 100644
--- a/types/f14_fast_set.toml
+++ b/types/f14_fast_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index a53c3e6..de1dcda 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_node_set.toml b/types/f14_node_set.toml
index 3ed0b5d..224789e 100644
--- a/types/f14_node_set.toml
+++ b/types/f14_node_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index 8072abf..c09bbf0 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_value_set.toml b/types/f14_value_set.toml
index dd4070d..f37f7d0 100644
--- a/types/f14_value_set.toml
+++ b/types/f14_value_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index ec7ac93..a673bbd 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_vector_set.toml b/types/f14_vector_set.toml
index e98b6d8..d85f36d 100644
--- a/types/f14_vector_set.toml
+++ b/types/f14_vector_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/fb_string_type.toml b/types/fb_string_type.toml
index e0679d3..169be87 100644
--- a/types/fb_string_type.toml
+++ b/types/fb_string_type.toml
@@ -29,7 +29,7 @@ void getSizeType(const %1%<E, T, A, Storage> &container, size_t& returnArg)
         &&
       ((uintptr_t)container.data() >= (uintptr_t)&container);
 
-    if (!inlined && pointers.add((uintptr_t)container.data())) {
+    if (!inlined && ctx.pointers.add((uintptr_t)container.data())) {
       SAVE_SIZE(container.capacity() * sizeof(T));
       SAVE_DATA(1);
     } else {
@@ -60,7 +60,7 @@ if (isStorageInline(container)) {
   category = Category::InlinedStorage;
 } else if (capacity < minLargeSize) {
   category = Category::OwnedHeapStorage;
-} else if (pointers.add(container.data())) {
+} else if (ctx.pointers.add(container.data())) {
   category = Category::ReferenceCountedStorage;
 } else {
   category = Category::AlreadyAttributed;
diff --git a/types/folly_iobuf_queue_type.toml b/types/folly_iobuf_queue_type.toml
index 0af134d..b9ef72c 100644
--- a/types/folly_iobuf_queue_type.toml
+++ b/types/folly_iobuf_queue_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1% &container, size_t& returnArg)
 
     const folly::IOBuf *head = container.front();
     SAVE_DATA((uintptr_t)head);
-    if (head && pointers.add((uintptr_t)head)) {
+    if (head && ctx.pointers.add((uintptr_t)head)) {
         SAVE_DATA(1);
         getSizeType(*head, returnArg);
     } else {
diff --git a/types/list_type.toml b/types/list_type.toml
index 86d457e..3e0da3c 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -38,8 +38,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index 887df6a..a09efff 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -42,12 +42,12 @@ traversal_func = '''
                     .write(container.size());
 
     for (const auto& kv : container) {
-      tail = tail.delegate([&kv](auto ret) {
-        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
-        auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
-          return OIInternal::getSizeType<Ctx>(kv.first, ret);
+      tail = tail.delegate([&ctx, &kv](auto ret) {
+        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, kv.first, ret);
+        auto next = start.delegate([&ctx, &kv](typename TypeHandler<Ctx, T0>::type ret) {
+          return OIInternal::getSizeType<Ctx>(ctx, kv.first, ret);
         });
-        return OIInternal::getSizeType<Ctx>(kv.second, next);
+        return OIInternal::getSizeType<Ctx>(ctx, kv.second, next);
       });
     }
 
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index e05a1e3..a5d628f 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -39,11 +39,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/multi_set_type.toml b/types/multi_set_type.toml
index 00f8239..f238356 100644
--- a/types/multi_set_type.toml
+++ b/types/multi_set_type.toml
@@ -42,8 +42,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/optional_type.toml b/types/optional_type.toml
index 3bd9cff..eeba64e 100644
--- a/types/optional_type.toml
+++ b/types/optional_type.toml
@@ -30,8 +30,8 @@ void getSizeType(const %1%<T>& container, size_t& returnArg) {
 
 traversal_func = """
 if (container.has_value()) {
-  return returnArg.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return returnArg.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 } else {
   return returnArg.template delegate<0>(std::identity());
diff --git a/types/pair_type.toml b/types/pair_type.toml
index ba5bc38..f157144 100644
--- a/types/pair_type.toml
+++ b/types/pair_type.toml
@@ -27,10 +27,10 @@ void getSizeType(const %1%<P,Q> &container, size_t& returnArg)
 """
 
 traversal_func = """
-    return OIInternal::getSizeType<Ctx>(
+    return OIInternal::getSizeType<Ctx>(ctx, 
         container.second,
-        returnArg.delegate([&container](auto ret) {
-            return OIInternal::getSizeType<Ctx>(container.first, ret);
+        returnArg.delegate([&ctx, &container](auto ret) {
+            return OIInternal::getSizeType<Ctx>(ctx, container.first, ret);
         })
     );
 """
diff --git a/types/ref_wrapper_type.toml b/types/ref_wrapper_type.toml
index 949b385..9ed5094 100644
--- a/types/ref_wrapper_type.toml
+++ b/types/ref_wrapper_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &ref, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
     SAVE_DATA((uintptr_t)&(ref.get()));
-    if (pointers.add((uintptr_t)&ref.get())) {
+    if (ctx.pointers.add((uintptr_t)&ref.get())) {
         SAVE_DATA(1);
         getSizeType(ref.get(), returnArg);
     } else {
diff --git a/types/seq_type.toml b/types/seq_type.toml
index 88ddd1d..f561d0b 100644
--- a/types/seq_type.toml
+++ b/types/seq_type.toml
@@ -43,8 +43,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/set_type.toml b/types/set_type.toml
index e29611e..9d3bc3c 100644
--- a/types/set_type.toml
+++ b/types/set_type.toml
@@ -43,8 +43,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 125fd71..bb3a622 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -24,7 +24,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
     if constexpr (!std::is_void<T>::value) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
-        if (s_ptr && pointers.add((uintptr_t)(s_ptr.get()))) {
+        if (s_ptr && ctx.pointers.add((uintptr_t)(s_ptr.get()))) {
             SAVE_DATA(1);
             getSizeType(*(s_ptr.get()), returnArg);
         } else {
@@ -40,12 +40,12 @@ auto tail = returnArg.write((uintptr_t)container.get());
 if constexpr (std::is_void<T0>::value) {
   return tail.template delegate<0>(std::identity());
 } else {
-  bool do_visit = container && pointers.add((uintptr_t)container.get());
+  bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
   if (!do_visit)
     return tail.template delegate<0>(std::identity());
 
-  return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return tail.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 }
 """
diff --git a/types/small_vec_type.toml b/types/small_vec_type.toml
index d129285..a5f1694 100644
--- a/types/small_vec_type.toml
+++ b/types/small_vec_type.toml
@@ -56,8 +56,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&it: container) {
-  tail = tail.delegate([&it](typename TypeHandler<Ctx, T0>::type ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](typename TypeHandler<Ctx, T0>::type ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/sorted_vec_set_type.toml b/types/sorted_vec_set_type.toml
index e8b9483..a803d61 100644
--- a/types/sorted_vec_set_type.toml
+++ b/types/sorted_vec_set_type.toml
@@ -37,8 +37,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (const auto& el : container) {
-  tail = tail.delegate([&el](auto ret) {
-    return OIInternal::getSizeType<Ctx>(el, ret);
+  tail = tail.delegate([&ctx, &el](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, el, ret);
   });
 }
 
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index df06cc7..ee652ec 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -43,12 +43,12 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(key, ret);
-    auto next =  start.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, key, ret);
+    auto next =  start.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index 27142a1..5b2402f 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -46,12 +46,12 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto& kv : container) {
-  tail = tail.delegate([&kv](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
-    auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(kv.first, ret);
+  tail = tail.delegate([&ctx, &kv](auto ret) {
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, kv.first, ret);
+    auto next = start.delegate([&ctx, &kv](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, kv.first, ret);
     });
-    return OIInternal::getSizeType<Ctx>(kv.second, next);
+    return OIInternal::getSizeType<Ctx>(ctx, kv.second, next);
   });
 }
 
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index 261a0be..a6f7671 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -46,8 +46,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index 58364cf..f723f80 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -25,7 +25,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
     if constexpr (!std::is_void<T>::value) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
-        if (u_ptr && pointers.add((uintptr_t)(u_ptr.get()))) {
+        if (u_ptr && ctx.pointers.add((uintptr_t)(u_ptr.get()))) {
             SAVE_DATA(1);
             getSizeType(*(u_ptr.get()), returnArg);
         } else {
@@ -41,12 +41,12 @@ auto tail = returnArg.write((uintptr_t)container.get());
 if constexpr (std::is_void<T0>::value) {
   return tail.template delegate<0>(std::identity());
 } else {
-  bool do_visit = container && pointers.add((uintptr_t)container.get());
+  bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
   if (!do_visit)
     return tail.template delegate<0>(std::identity());
 
-  return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return tail.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 }
 """
diff --git a/types/unordered_multiset_type.toml b/types/unordered_multiset_type.toml
index 5c2e96b..bfcf203 100644
--- a/types/unordered_multiset_type.toml
+++ b/types/unordered_multiset_type.toml
@@ -44,8 +44,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/unordered_set_type.toml b/types/unordered_set_type.toml
index f8d26c9..d9a827a 100644
--- a/types/unordered_set_type.toml
+++ b/types/unordered_set_type.toml
@@ -44,8 +44,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 

From ccc2aabad7c96b7db0dec62d22d686f41e0efd60 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 15 Nov 2023 05:24:28 -0800
Subject: [PATCH 023/188] tbv2: replace DB template param with Ctx

TreeBuilder v2 adds a DB template parameter to every function. This is used as
part of the static type to decide what type of DataBuffer is being used:
currently `BackInserterDataBuffer<std::vector<uint8_t>>` for OIL and it would
be `DataSegmentDataBuffer` for OID.

This change replaces the `DB` template parameter with a more general `Ctx`. Due
to issues with dependent naming it also adds a `using DB` to each `TypeHandler`
which has the same function as before. This allows us to add more "static
context" (typedefs and constants) to functions without changing this signature
again, because changing the signature of everything is a massive pain.

Currently this change achieves nothing because Ctx contains only DB in a static
wrapper. In the next change I'm going to pass a reference of type Ctx around to
add a "dynamic context" to invocations which will contain the pointer array. In
future we'll then be able to add either static or dynamic context without any
signature adjustments.

Test plan:
- CI
---
 oi/CodeGen.cpp                         | 59 ++++++++++++++------------
 oi/FuncGen.cpp                         | 47 +++++++++++---------
 types/array_type.toml                  |  6 +--
 types/cxx11_list_type.toml             |  8 ++--
 types/cxx11_string_type.toml           |  5 ++-
 types/f14_fast_map.toml                | 14 +++---
 types/f14_fast_set.toml                |  6 +--
 types/f14_node_map.toml                | 14 +++---
 types/f14_node_set.toml                |  6 +--
 types/f14_value_map.toml               | 14 +++---
 types/f14_value_set.toml               |  6 +--
 types/f14_vector_map.toml              | 14 +++---
 types/f14_vector_set.toml              |  6 +--
 types/list_type.toml                   |  8 ++--
 types/map_seq_type.toml                | 24 +++++------
 types/multi_map_type.toml              | 14 +++---
 types/multi_set_type.toml              |  6 +--
 types/optional_type.toml               |  8 ++--
 types/pair_type.toml                   | 10 ++---
 types/seq_type.toml                    |  6 +--
 types/set_type.toml                    |  6 +--
 types/shrd_ptr_type.toml               |  8 ++--
 types/small_vec_type.toml              |  8 ++--
 types/sorted_vec_set_type.toml         |  6 +--
 types/std_map_type.toml                | 24 +++++------
 types/std_unordered_map_type.toml      | 24 +++++------
 types/std_unordered_multimap_type.toml | 10 ++---
 types/uniq_ptr_type.toml               |  8 ++--
 types/unordered_multiset_type.toml     |  6 +--
 types/unordered_set_type.toml          |  6 +--
 30 files changed, 200 insertions(+), 187 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 73725e4..cfa87e7 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -651,7 +651,7 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
   code += funcName;
   code += "(\n      const ";
   code += c.name();
-  code += "& t,\n      typename TypeHandler<DB, ";
+  code += "& t,\n      typename TypeHandler<Ctx, ";
   code += c.name();
   code += ">::type returnArg) {\n";
 
@@ -683,7 +683,7 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
     } else {
       code += "delegate";
     }
-    code += "([&t](auto ret) { return OIInternal::getSizeType<DB>(t.";
+    code += "([&t](auto ret) { return OIInternal::getSizeType<Ctx>(t.";
     code += member.name;
     code += ", ret); })";
   }
@@ -695,7 +695,7 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
 }
 
 // Generate the static type for the class's representation in the data buffer.
-// For `class { int a,b,c; }` we generate (DB omitted for clarity):
+// For `class { int a,b,c; }` we generate (Ctx/DB omitted for clarity):
 // Pair<TypeHandler<int>::type,
 //   Pair<TypeHandler<int>::type,
 //     TypeHandler<int>::type
@@ -734,7 +734,7 @@ void CodeGen::genClassStaticType(const Class& c, std::string& code) {
     }
 
     code +=
-        (boost::format("typename TypeHandler<DB, decltype(%1%::%2%)>::type") %
+        (boost::format("typename TypeHandler<Ctx, decltype(%1%::%2%)>::type") %
          c.name() % member.name)
             .str();
 
@@ -787,8 +787,8 @@ void CodeGen::genClassTreeBuilderInstructions(const Class& c,
     code += "      inst::Field{sizeof(" + fullName + "), " +
             std::to_string(calculateExclusiveSize(m.type())) + ",\"" +
             m.inputName + "\", member_" + std::to_string(index) +
-            "_type_names, TypeHandler<DB, decltype(" + fullName +
-            ")>::fields, TypeHandler<DB, decltype(" + fullName +
+            "_type_names, TypeHandler<Ctx, decltype(" + fullName +
+            ")>::fields, TypeHandler<Ctx, decltype(" + fullName +
             ")>::processors},\n";
   }
   code += "  };\n";
@@ -820,10 +820,11 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) {
                    .str();
   }
 
-  code += "template <typename DB>\n";
-  code += "class TypeHandler<DB, ";
+  code += "template <typename Ctx>\n";
+  code += "class TypeHandler<Ctx, ";
   code += c.name();
   code += "> {\n";
+  code += "  using DB = typename Ctx::DataBuffer;\n";
   code += helpers;
   code += " public:\n";
   code += "  using type = ";
@@ -877,7 +878,7 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     containerWithTypes = "OICaptureKeys<" + containerWithTypes + ">";
   }
 
-  code += "template <typename DB";
+  code += "template <typename Ctx";
   types = 0, values = 0;
   for (const auto& p : templateParams) {
     if (p.value) {
@@ -889,9 +890,10 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     }
   }
   code += ">\n";
-  code += "struct TypeHandler<DB, ";
+  code += "struct TypeHandler<Ctx, ";
   code += containerWithTypes;
   code += "> {\n";
+  code += "  using DB = typename Ctx::DataBuffer;\n";
 
   if (c.captureKeys) {
     code += "  static constexpr bool captureKeys = true;\n";
@@ -922,7 +924,7 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   code += "      const ";
   code += containerWithTypes;
   code += "& container,\n";
-  code += "      typename TypeHandler<DB, ";
+  code += "      typename TypeHandler<Ctx, ";
   code += containerWithTypes;
   code += ">::type returnArg) {\n";
   code += func;  // has rubbish indentation
@@ -961,8 +963,9 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
 
 void addCaptureKeySupport(std::string& code) {
   code += R"(
-    template <typename DB, typename T>
+    template <typename Ctx, typename T>
     class CaptureKeyHandler {
+      using DB = typename Ctx::DataBuffer;
      public:
       using type = types::st::Sum<DB, types::st::VarInt<DB>, types::st::VarInt<DB>>;
 
@@ -975,24 +978,24 @@ void addCaptureKeySupport(std::string& code) {
       }
     };
 
-    template <bool CaptureKeys, typename DB, typename T>
+    template <bool CaptureKeys, typename Ctx, typename T>
     auto maybeCaptureKey(const T& key, auto returnArg) {
       if constexpr (CaptureKeys) {
         return returnArg.delegate([&key](auto ret) {
-          return CaptureKeyHandler<DB, T>::captureKey(key, ret);
+          return CaptureKeyHandler<Ctx, T>::captureKey(key, ret);
         });
       } else {
         return returnArg;
       }
     }
 
-    template <typename DB, typename T>
+    template <typename Ctx, typename T>
     static constexpr inst::ProcessorInst CaptureKeysProcessor{
-      CaptureKeyHandler<DB, T>::type::describe,
+      CaptureKeyHandler<Ctx, T>::type::describe,
       [](result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
         if constexpr (std::is_same_v<
-            typename CaptureKeyHandler<DB, T>::type,
-            types::st::List<DB, types::st::VarInt<DB>>>) {
+            typename CaptureKeyHandler<Ctx, T>::type,
+            types::st::List<typename Ctx::DataBuffer, types::st::VarInt<typename Ctx::DataBuffer>>>) {
           // String
           auto& str = el.data.emplace<std::string>();
           auto list = std::get<ParsedData::List>(d.val);
@@ -1013,11 +1016,11 @@ void addCaptureKeySupport(std::string& code) {
       }
     };
 
-    template <bool CaptureKeys, typename DB, typename T>
+    template <bool CaptureKeys, typename Ctx, typename T>
     static constexpr auto maybeCaptureKeysProcessor() {
       if constexpr (CaptureKeys) {
         return std::array<inst::ProcessorInst, 1>{
-          CaptureKeysProcessor<DB, T>,
+          CaptureKeysProcessor<Ctx, T>,
         };
       }
       else {
@@ -1034,28 +1037,28 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
     addCaptureKeySupport(code);
 
   // Provide a wrapper function, getSizeType, to infer T instead of having to
-  // explicitly specify it with TypeHandler<DB, T>::getSizeType every time.
+  // explicitly specify it with TypeHandler<Ctx, T>::getSizeType every time.
   code += R"(
-    template <typename DB, typename T>
-    types::st::Unit<DB>
-    getSizeType(const T &t, typename TypeHandler<DB, T>::type returnArg) {
+    template <typename Ctx, typename T>
+    types::st::Unit<typename Ctx::DataBuffer>
+    getSizeType(const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
       JLOG("obj @");
       JLOGPTR(&t);
-      return TypeHandler<DB, T>::getSizeType(t, returnArg);
+      return TypeHandler<Ctx, T>::getSizeType(t, returnArg);
     }
 )";
 
   if (features[Feature::TreeBuilderV2]) {
     code += R"(
-template <typename DB, typename T>
+template <typename Ctx, typename T>
 constexpr inst::Field make_field(std::string_view name) {
   return inst::Field{
     sizeof(T),
     ExclusiveSizeProvider<T>::size,
     name,
     NameProvider<T>::names,
-    TypeHandler<DB, T>::fields,
-    TypeHandler<DB, T>::processors,
+    TypeHandler<Ctx, T>::fields,
+    TypeHandler<Ctx, T>::processors,
   };
 }
 )";
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 7a1dd09..1a0315f 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -268,11 +268,13 @@ void __attribute__((used, retain)) introspect_%2$016x(
   v.clear();
   v.reserve(4096);
 
-  using DataBufferType = DataBuffer::BackInserter<std::vector<uint8_t>>;
-  using ContentType = OIInternal::TypeHandler<DataBufferType, OIInternal::__ROOT_TYPE__>::type;
+  struct Context {
+    using DataBuffer = DataBuffer::BackInserter<std::vector<uint8_t>>;
+  };
+  using ContentType = OIInternal::TypeHandler<Context, OIInternal::__ROOT_TYPE__>::type;
 
-  ContentType ret{DataBufferType{v}};
-  OIInternal::getSizeType<DataBufferType>(t, ret);
+  ContentType ret{Context::DataBuffer{v}};
+  OIInternal::getSizeType<Context>(t, ret);
 }
 )";
 
@@ -364,6 +366,9 @@ void FuncGen::DefineTreeBuilderInstructions(
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunknown-attributes"
 namespace {
+struct FakeContext {
+  using DataBuffer = int;
+};
 const std::array<std::string_view, )";
   code += std::to_string(typeNames.size());
   code += "> typeNames";
@@ -382,8 +387,10 @@ const std::array<std::string_view, )";
   code += ", \"a0\", typeNames";
   code += typeHash;
   code +=
-      ", OIInternal::TypeHandler<int, OIInternal::__ROOT_TYPE__>::fields, "
-      "OIInternal::TypeHandler<int, OIInternal::__ROOT_TYPE__>::processors};\n";
+      ", OIInternal::TypeHandler<FakeContext, "
+      "OIInternal::__ROOT_TYPE__>::fields, "
+      "OIInternal::TypeHandler<FakeContext, "
+      "OIInternal::__ROOT_TYPE__>::processors};\n";
   code += "} // namespace\n";
   code +=
       "extern const exporters::inst::Inst __attribute__((used, retain)) "
@@ -595,14 +602,15 @@ class BackInserter {
  */
 void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   code += R"(
-    template <typename DB, typename T>
+    template <typename Ctx, typename T>
     struct TypeHandler {
+        using DB = typename Ctx::DataBuffer;
       private:
         static auto choose_type() {
             if constexpr(std::is_pointer_v<T>) {
                 return std::type_identity<types::st::Pair<DB,
                   types::st::VarInt<DB>,
-                  types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<DB, std::remove_pointer_t<T>>::type>
+                  types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>
                 >>();
             } else {
                 return std::type_identity<types::st::Unit<DB>>();
@@ -623,8 +631,8 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
             sizeof(T),
             "*",
             names,
-            TypeHandler<DB, T>::fields,
-            TypeHandler<DB, T>::processors,
+            TypeHandler<Ctx, T>::fields,
+            TypeHandler<Ctx, T>::processors,
           };
 
           const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
@@ -649,7 +657,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
           if constexpr(std::is_pointer_v<T>) {
             return std::array<inst::ProcessorInst, 2>{
               {types::st::VarInt<DB>::describe, &process_pointer},
-              {types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<DB, std::remove_pointer_t<T>>::type>::describe, &process_pointer_content},
+              {types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>::describe, &process_pointer_content},
             };
           } else {
             return std::array<inst::ProcessorInst, 0>{};
@@ -663,7 +671,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   code += R"(
         static types::st::Unit<DB> getSizeType(
           const T& t,
-          typename TypeHandler<DB, T>::type returnArg) {
+          typename TypeHandler<Ctx, T>::type returnArg) {
               if constexpr(std::is_pointer_v<T>) {
                 JLOG("ptr val @");
                 JLOGPTR(t);
@@ -671,7 +679,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
                 if (t && pointers.add((uintptr_t)t)) {
                   return r0.template delegate<1>([&t](auto ret) {
                     if constexpr (!std::is_void<std::remove_pointer_t<T>>::value) {
-                      return TypeHandler<DB, std::remove_pointer_t<T>>::getSizeType(*t, ret);
+                      return TypeHandler<Ctx, std::remove_pointer_t<T>>::getSizeType(*t, ret);
                     } else {
                       return ret;
                     }
@@ -687,8 +695,9 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   )";
 
   code += R"(
-    template <typename DB>
-    class TypeHandler<DB, void> {
+    template <typename Ctx>
+    class TypeHandler<Ctx, void> {
+        using DB = typename Ctx::DataBuffer;
       public:
         using type = types::st::Unit<DB>;
 )";
@@ -711,21 +720,21 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
   tail = tail.delegate([&container, i](auto ret) {
-      return TypeHandler<DB, T0>::getSizeType(container.vals[i], ret);
+      return TypeHandler<Ctx, T0>::getSizeType(container.vals[i], ret);
   });
 }
 return tail.finish();
 )";
   oiArray.codegen.processors.emplace_back(ContainerInfo::Processor{
-      .type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>",
+      .type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>",
       .func = R"(
 static constexpr std::array<std::string_view, 1> names{"TODO"};
 static constexpr auto childField = inst::Field{
   sizeof(T0),
   "[]",
   names,
-  TypeHandler<DB, T0>::fields,
-  TypeHandler<DB, T0>::processors,
+  TypeHandler<Ctx, T0>::fields,
+  TypeHandler<Ctx, T0>::processors,
 };
 
 el.exclusive_size = 0;
diff --git a/types/array_type.toml b/types/array_type.toml
index 6d73322..126af6a 100644
--- a/types/array_type.toml
+++ b/types/array_type.toml
@@ -34,7 +34,7 @@ traversal_func = """
 
     for (auto & it: container) {
         tail = tail.delegate([&it](auto ret) {
-            return TypeHandler<DB, T0>::getSizeType(it, ret);
+            return TypeHandler<Ctx, T0>::getSizeType(it, ret);
         });
     }
 
@@ -42,9 +42,9 @@ traversal_func = """
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 size_t size = std::get<ParsedData::List>(d.val).length;
 el.exclusive_size = N0 == 0 ? 1 : 0;
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index fb54d22..f838eee 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -39,7 +39,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -53,7 +53,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 static constexpr size_t element_size = sizeof(std::_List_node<T0>);
@@ -62,7 +62,7 @@ static_assert(false && "No known element_size for list. See types/cxx11_list_typ
 #endif
 
 static constexpr std::array<inst::Field, 1> child_field{
-  make_field<DB, T0>("*"),
+  make_field<Ctx, T0>("*"),
 };
 static constexpr inst::Field element{
   element_size,
@@ -72,7 +72,7 @@ static constexpr inst::Field element{
   child_field,
   std::array<inst::ProcessorInst, 0>{},
 };
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats{
diff --git a/types/cxx11_string_type.toml b/types/cxx11_string_type.toml
index 94e7183..c88debd 100644
--- a/types/cxx11_string_type.toml
+++ b/types/cxx11_string_type.toml
@@ -37,8 +37,9 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
 """
 
 extra = """
-template <typename DB, typename CharT, typename Traits, typename Allocator>
-class CaptureKeyHandler<DB, std::__cxx11::basic_string<CharT, Traits, Allocator>> {
+template <typename Ctx, typename CharT, typename Traits, typename Allocator>
+class CaptureKeyHandler<Ctx, std::__cxx11::basic_string<CharT, Traits, Allocator>> {
+  using DB = typename Ctx::DataBuffer;
  public:
   // List of characters
   using type = types::st::List<DB, types::st::VarInt<DB>>;
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index c8c84f2..faf2e9d 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -44,10 +44,10 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -69,8 +69,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 static constexpr size_t element_size = sizeof(typename container_type::value_type);
@@ -84,8 +84,8 @@ el.container_stats->length = list.length;
 el.exclusive_size += allocationSize - list.length * element_size;
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr inst::Field element{
diff --git a/types/f14_fast_set.toml b/types/f14_fast_set.toml
index 8516f8c..b66284b 100644
--- a/types/f14_fast_set.toml
+++ b/types/f14_fast_set.toml
@@ -43,7 +43,7 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<DB>(entry, ret);
+    return OIInternal::getSizeType<Ctx>(entry, ret);
   });
 }
 
@@ -64,7 +64,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 
 [[codegen.processor]]
 type = """
-types::st::List<DB, typename TypeHandler<DB, T0>::type>
+types::st::List<DB, typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto allocationSize = el.pointer.value();
@@ -75,7 +75,7 @@ el.container_stats->length = list.length;
 
 el.exclusive_size += allocationSize - list.length * sizeof(T0);
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index 93b53e9..a53c3e6 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -44,10 +44,10 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -69,8 +69,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 static constexpr size_t element_size = sizeof(typename container_type::value_type);
@@ -84,8 +84,8 @@ el.container_stats->length = list.length;
 el.exclusive_size += allocationSize - list.length * element_size;
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr inst::Field element{
diff --git a/types/f14_node_set.toml b/types/f14_node_set.toml
index 511c2f8..3ed0b5d 100644
--- a/types/f14_node_set.toml
+++ b/types/f14_node_set.toml
@@ -43,7 +43,7 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<DB>(entry, ret);
+    return OIInternal::getSizeType<Ctx>(entry, ret);
   });
 }
 
@@ -64,7 +64,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 
 [[codegen.processor]]
 type = """
-types::st::List<DB, typename TypeHandler<DB, T0>::type>
+types::st::List<DB, typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto allocationSize = el.pointer.value();
@@ -75,7 +75,7 @@ el.container_stats->length = list.length;
 
 el.exclusive_size += allocationSize - list.length * sizeof(T0);
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index ec4f187..8072abf 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -44,10 +44,10 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -69,8 +69,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 static constexpr size_t element_size = sizeof(typename container_type::value_type);
@@ -84,8 +84,8 @@ el.container_stats->length = list.length;
 el.exclusive_size += allocationSize - list.length * element_size;
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr inst::Field element{
diff --git a/types/f14_value_set.toml b/types/f14_value_set.toml
index 20aae30..dd4070d 100644
--- a/types/f14_value_set.toml
+++ b/types/f14_value_set.toml
@@ -43,7 +43,7 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<DB>(entry, ret);
+    return OIInternal::getSizeType<Ctx>(entry, ret);
   });
 }
 
@@ -64,7 +64,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 
 [[codegen.processor]]
 type = """
-types::st::List<DB, typename TypeHandler<DB, T0>::type>
+types::st::List<DB, typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto allocationSize = el.pointer.value();
@@ -75,7 +75,7 @@ el.container_stats->length = list.length;
 
 el.exclusive_size += allocationSize - list.length * sizeof(T0);
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index 5e29d82..ec7ac93 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -44,10 +44,10 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -69,8 +69,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 static constexpr size_t element_size = sizeof(typename container_type::value_type);
@@ -84,8 +84,8 @@ el.container_stats->length = list.length;
 el.exclusive_size += allocationSize - list.length * element_size;
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr inst::Field element{
diff --git a/types/f14_vector_set.toml b/types/f14_vector_set.toml
index 1aed210..e98b6d8 100644
--- a/types/f14_vector_set.toml
+++ b/types/f14_vector_set.toml
@@ -43,7 +43,7 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<DB>(entry, ret);
+    return OIInternal::getSizeType<Ctx>(entry, ret);
   });
 }
 
@@ -64,7 +64,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 
 [[codegen.processor]]
 type = """
-types::st::List<DB, typename TypeHandler<DB, T0>::type>
+types::st::List<DB, typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto allocationSize = el.pointer.value();
@@ -75,7 +75,7 @@ el.container_stats->length = list.length;
 
 el.exclusive_size += allocationSize - list.length * sizeof(T0);
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/list_type.toml b/types/list_type.toml
index 9ea8ef7..86d457e 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -39,7 +39,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -53,7 +53,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 static constexpr size_t element_size = sizeof(std::_List_node<T0>);
@@ -62,7 +62,7 @@ static_assert(false && "No known element_size for list. See types/cxx11_list_typ
 #endif
 
 static constexpr std::array<inst::Field, 1> child_field{
-  make_field<DB, T0>("*"),
+  make_field<Ctx, T0>("*"),
 };
 static constexpr inst::Field element{
   element_size,
@@ -72,7 +72,7 @@ static constexpr inst::Field element{
   child_field,
   std::array<inst::ProcessorInst, 0>{},
 };
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats{
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index 90d424d..887df6a 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -43,11 +43,11 @@ traversal_func = '''
 
     for (const auto& kv : container) {
       tail = tail.delegate([&kv](auto ret) {
-        auto start = maybeCaptureKey<captureKeys, DB, T0>(kv.first, ret);
-        auto next = start.delegate([&kv](typename TypeHandler<DB, T0>::type ret) {
-          return OIInternal::getSizeType<DB>(kv.first, ret);
+        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
+        auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
+          return OIInternal::getSizeType<Ctx>(kv.first, ret);
         });
-        return OIInternal::getSizeType<DB>(kv.second, next);
+        return OIInternal::getSizeType<Ctx>(kv.second, next);
       });
     }
 
@@ -68,22 +68,22 @@ el.container_stats.emplace(result::Element::ContainerStats{ .capacity = std::get
 type = '''
 std::conditional_t<captureKeys,
   types::st::List<DB, types::st::Pair<DB,
-    typename CaptureKeyHandler<DB, T0>::type,
+    typename CaptureKeyHandler<Ctx, T0>::type,
     types::st::Pair<DB,
-      typename TypeHandler<DB, T0>::type,
-      typename TypeHandler<DB, T1>::type>>>,
+      typename TypeHandler<Ctx, T0>::type,
+      typename TypeHandler<Ctx, T1>::type>>>,
   types::st::List<DB, types::st::Pair<DB,
-    typename TypeHandler<DB, T0>::type,
-    typename TypeHandler<DB, T1>::type>>>
+    typename TypeHandler<Ctx, T0>::type,
+    typename TypeHandler<Ctx, T1>::type>>>
 '''
 func = '''
 using element_type = std::pair<T0, T1>;
 
 static constexpr std::array<inst::Field, 2> entryFields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
-static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, DB, T0>();
+static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, Ctx, T0>();
 static constexpr auto entry = inst::Field {
   sizeof(element_type),
   sizeof(element_type) - sizeof(T0) - sizeof(T1),
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index 7b1e622..e05a1e3 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -40,10 +40,10 @@ auto tail = returnArg
 
 for (const auto &entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -57,8 +57,8 @@ func = "el.pointer = std::get<ParsedData::VarInt>(d.val).value;"
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 #ifdef __GLIBCXX__
@@ -108,8 +108,8 @@ static_assert(false && "No known element_size for sets. See types/multi_map_type
 #endif
 
 static constexpr std::array<inst::Field, 2> elementFields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 static constexpr auto element = inst::Field {
   element_size,
diff --git a/types/multi_set_type.toml b/types/multi_set_type.toml
index 34163e8..00f8239 100644
--- a/types/multi_set_type.toml
+++ b/types/multi_set_type.toml
@@ -43,7 +43,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 // vector<bool>
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -57,7 +57,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 /* We don't have access to the _Rb_tree_node struct, so we manually re-create it
@@ -103,7 +103,7 @@ static constexpr size_t element_size = sizeof(OI__tree_node);
 static_assert(false && "No known element_size for multisets. See types/multi_set_type.toml");
 #endif
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats {
diff --git a/types/optional_type.toml b/types/optional_type.toml
index 2178743..3bd9cff 100644
--- a/types/optional_type.toml
+++ b/types/optional_type.toml
@@ -31,7 +31,7 @@ void getSizeType(const %1%<T>& container, size_t& returnArg) {
 traversal_func = """
 if (container.has_value()) {
   return returnArg.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<DB>(*container, ret);
+    return OIInternal::getSizeType<Ctx>(*container, ret);
   });
 } else {
   return returnArg.template delegate<0>(std::identity());
@@ -39,15 +39,15 @@ if (container.has_value()) {
 """
 
 [[codegen.processor]]
-type = "types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<DB, T0>::type>"
+type = "types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, T0>::type>"
 func = """
 static constexpr std::array<std::string_view, 1> names{"TODO"};
 static constexpr auto elementField = inst::Field{
   sizeof(T0),
   "el",
   names,
-  TypeHandler<DB, T0>::fields,
-  TypeHandler<DB, T0>::processors,
+  TypeHandler<Ctx, T0>::fields,
+  TypeHandler<Ctx, T0>::processors,
 };
 
 auto sum = std::get<ParsedData::Sum>(d.val);
diff --git a/types/pair_type.toml b/types/pair_type.toml
index 570d289..ba5bc38 100644
--- a/types/pair_type.toml
+++ b/types/pair_type.toml
@@ -27,19 +27,19 @@ void getSizeType(const %1%<P,Q> &container, size_t& returnArg)
 """
 
 traversal_func = """
-    return OIInternal::getSizeType<DB>(
+    return OIInternal::getSizeType<Ctx>(
         container.second,
         returnArg.delegate([&container](auto ret) {
-            return OIInternal::getSizeType<DB>(container.first, ret);
+            return OIInternal::getSizeType<Ctx>(container.first, ret);
         })
     );
 """
 
 [[codegen.processor]]
-type = "types::st::Pair<DB, typename TypeHandler<DB, T0>::type, typename TypeHandler<DB, T1>::type>"
+type = "types::st::Pair<DB, typename TypeHandler<Ctx, T0>::type, typename TypeHandler<Ctx, T1>::type>"
 func = """
-static constexpr auto firstField = make_field<DB, T0>("first");
-static constexpr auto secondField = make_field<DB, T1>("second");
+static constexpr auto firstField = make_field<Ctx, T0>("first");
+static constexpr auto secondField = make_field<Ctx, T1>("second");
 
 el.exclusive_size = sizeof(std::pair<T0, T1>) - sizeof(T0) - sizeof(T1);
 stack_ins(secondField);
diff --git a/types/seq_type.toml b/types/seq_type.toml
index 0ccfee8..88ddd1d 100644
--- a/types/seq_type.toml
+++ b/types/seq_type.toml
@@ -44,7 +44,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 // vector<bool>
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -64,9 +64,9 @@ el.container_stats.emplace(result::Element::ContainerStats{ .capacity = std::get
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats->length = list.length;
diff --git a/types/set_type.toml b/types/set_type.toml
index 43206f6..e29611e 100644
--- a/types/set_type.toml
+++ b/types/set_type.toml
@@ -44,7 +44,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 // vector<bool>
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -58,7 +58,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 /* We don't have access to the _Rb_tree_node struct, so we manually re-create it
@@ -104,7 +104,7 @@ static constexpr size_t element_size = sizeof(OI__tree_node);
 static_assert(false && "No known element_size for sets. See types/set_type.toml");
 #endif
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats {
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 693f871..125fd71 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -45,7 +45,7 @@ if constexpr (std::is_void<T0>::value) {
     return tail.template delegate<0>(std::identity());
 
   return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<DB>(*container, ret);
+    return OIInternal::getSizeType<Ctx>(*container, ret);
   });
 }
 """
@@ -60,7 +60,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<DB, T0>::type>
+  typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 #ifdef __GLIBCXX__
@@ -84,10 +84,10 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = sum.index, // 0 for empty containers/void, 1 otherwise
 });
 
-// Must be in a `if constexpr` or the compiler will complain about make_field<DB, void>
+// Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
 if constexpr (!std::is_void<T0>::value) {
   if (sum.index == 1) {
-    static constexpr auto element = make_field<DB, T0>("ptr_val");
+    static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);
   }
 }
diff --git a/types/small_vec_type.toml b/types/small_vec_type.toml
index aca3e79..d129285 100644
--- a/types/small_vec_type.toml
+++ b/types/small_vec_type.toml
@@ -56,8 +56,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&it: container) {
-  tail = tail.delegate([&it](typename TypeHandler<DB, T0>::type ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+  tail = tail.delegate([&it](typename TypeHandler<Ctx, T0>::type ret) {
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -81,7 +81,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 // Reading the `uses_intern_storage` boolean that was stored in `pointer` by the processor above.
 bool uses_intern_storage = std::exchange(el.pointer.value(), (uintptr_t)0);
@@ -97,7 +97,7 @@ if (uses_intern_storage) {
   el.exclusive_size += (el.container_stats->capacity - el.container_stats->length) * sizeof(T0);
 }
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/sorted_vec_set_type.toml b/types/sorted_vec_set_type.toml
index f17a0f7..e8b9483 100644
--- a/types/sorted_vec_set_type.toml
+++ b/types/sorted_vec_set_type.toml
@@ -38,7 +38,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 
 for (const auto& el : container) {
   tail = tail.delegate([&el](auto ret) {
-    return OIInternal::getSizeType<DB>(el, ret);
+    return OIInternal::getSizeType<Ctx>(el, ret);
   });
 }
 
@@ -56,9 +56,9 @@ el.container_stats.emplace(result::Element::ContainerStats{ .capacity = std::get
 '''
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats->length = list.length;
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index 61be6be..df06cc7 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -44,11 +44,11 @@ auto tail = returnArg
 
 for (const auto &entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, DB, T0>(key, ret);
-    auto next =  start.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(key, ret);
+    auto next =  start.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -64,14 +64,14 @@ type = """
 std::conditional_t<captureKeys,
   types::st::List<DB,
     types::st::Pair<DB,
-      typename CaptureKeyHandler<DB, T0>::type,
+      typename CaptureKeyHandler<Ctx, T0>::type,
       types::st::Pair<DB,
-        typename TypeHandler<DB, T0>::type,
-        typename TypeHandler<DB, T1>::type>>>,
+        typename TypeHandler<Ctx, T0>::type,
+        typename TypeHandler<Ctx, T1>::type>>>,
   types::st::List<DB,
     types::st::Pair<DB,
-      typename TypeHandler<DB, T0>::type,
-      typename TypeHandler<DB, T1>::type>>
+      typename TypeHandler<Ctx, T0>::type,
+      typename TypeHandler<Ctx, T1>::type>>
 >
 """
 func = """
@@ -122,11 +122,11 @@ static_assert(false && "No known element_size for sets. See types/std_map_type.t
 #endif
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
-static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, DB, T0>();
+static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, Ctx, T0>();
 
 static constexpr inst::Field element{
   element_size,
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index 541576d..27142a1 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -47,11 +47,11 @@ auto tail = returnArg
 
 for (const auto& kv : container) {
   tail = tail.delegate([&kv](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, DB, T0>(kv.first, ret);
-    auto next = start.delegate([&kv](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(kv.first, ret);
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
+    auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(kv.first, ret);
     });
-    return OIInternal::getSizeType<DB>(kv.second, next);
+    return OIInternal::getSizeType<Ctx>(kv.second, next);
   });
 }
 
@@ -79,13 +79,13 @@ type = """
 std::conditional_t<captureKeys,
  types::st::List<DB,
    types::st::Pair<DB,
-     typename CaptureKeyHandler<DB, T0>::type,
+     typename CaptureKeyHandler<Ctx, T0>::type,
      types::st::Pair<DB,
-       typename TypeHandler<DB, T0>::type,
-       typename TypeHandler<DB, T1>::type>>>,
+       typename TypeHandler<Ctx, T0>::type,
+       typename TypeHandler<Ctx, T1>::type>>>,
  types::st::List<DB, types::st::Pair<DB,
-   typename TypeHandler<DB, T0>::type,
-   typename TypeHandler<DB, T1>::type>>
+   typename TypeHandler<Ctx, T0>::type,
+   typename TypeHandler<Ctx, T1>::type>>
 >
 """
 func = """
@@ -122,11 +122,11 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
-static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, DB, T0>();
+static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, Ctx, T0>();
 
 static constexpr auto element = inst::Field{
   element_size,
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index a548623..261a0be 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -47,7 +47,7 @@ auto tail = returnArg
 
 for (const auto &it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -73,8 +73,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 #ifdef __GLIBCXX__
@@ -110,8 +110,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr auto element = inst::Field{
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index 35155f9..58364cf 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -46,7 +46,7 @@ if constexpr (std::is_void<T0>::value) {
     return tail.template delegate<0>(std::identity());
 
   return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<DB>(*container, ret);
+    return OIInternal::getSizeType<Ctx>(*container, ret);
   });
 }
 """
@@ -61,7 +61,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<DB, T0>::type>
+  typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto sum = std::get<ParsedData::Sum>(d.val);
@@ -70,10 +70,10 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = sum.index, // 0 for empty containers/void, 1 otherwise
 });
 
-// Must be in a `if constexpr` or the compiler will complain about make_field<DB, void>
+// Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
 if constexpr (!std::is_void<T0>::value) {
   if (sum.index == 1) {
-    static constexpr auto element = make_field<DB, T0>("ptr_val");
+    static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);
   }
 }
diff --git a/types/unordered_multiset_type.toml b/types/unordered_multiset_type.toml
index 9d1449a..5c2e96b 100644
--- a/types/unordered_multiset_type.toml
+++ b/types/unordered_multiset_type.toml
@@ -45,7 +45,7 @@ auto tail = returnArg
 
 for (const auto &it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -69,7 +69,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 /* Use libstdc++ implementation __details to compute the size of Nodes and Buckets.
@@ -100,7 +100,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = list.length,
 });
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/unordered_set_type.toml b/types/unordered_set_type.toml
index 62e0190..f8d26c9 100644
--- a/types/unordered_set_type.toml
+++ b/types/unordered_set_type.toml
@@ -45,7 +45,7 @@ auto tail = returnArg
 
 for (const auto &it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -69,7 +69,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 /* Use libstdc++ implementation __details to compute the size of Nodes and Buckets.
@@ -100,7 +100,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = list.length,
 });
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """

From 1021d12320060503cb562780f4491354f2ec8a61 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 15 Nov 2023 05:15:26 -0800
Subject: [PATCH 024/188] oil: use a unique pointer set per invocation

---
 oi/CodeGen.cpp                         | 25 ++++++++++++++++++-------
 oi/FuncGen.cpp                         | 26 ++++++++++++++++----------
 oi/OICodeGen.cpp                       |  8 +++++++-
 oi/OITraceCode.cpp                     |  7 ++++---
 types/array_type.toml                  |  4 ++--
 types/cxx11_list_type.toml             |  4 ++--
 types/f14_fast_map.toml                |  8 ++++----
 types/f14_fast_set.toml                |  4 ++--
 types/f14_node_map.toml                |  8 ++++----
 types/f14_node_set.toml                |  4 ++--
 types/f14_value_map.toml               |  8 ++++----
 types/f14_value_set.toml               |  4 ++--
 types/f14_vector_map.toml              |  8 ++++----
 types/f14_vector_set.toml              |  4 ++--
 types/fb_string_type.toml              |  4 ++--
 types/folly_iobuf_queue_type.toml      |  2 +-
 types/list_type.toml                   |  4 ++--
 types/map_seq_type.toml                | 10 +++++-----
 types/multi_map_type.toml              |  8 ++++----
 types/multi_set_type.toml              |  4 ++--
 types/optional_type.toml               |  4 ++--
 types/pair_type.toml                   |  6 +++---
 types/ref_wrapper_type.toml            |  2 +-
 types/seq_type.toml                    |  4 ++--
 types/set_type.toml                    |  4 ++--
 types/shrd_ptr_type.toml               |  8 ++++----
 types/small_vec_type.toml              |  4 ++--
 types/sorted_vec_set_type.toml         |  4 ++--
 types/std_map_type.toml                | 10 +++++-----
 types/std_unordered_map_type.toml      | 10 +++++-----
 types/std_unordered_multimap_type.toml |  4 ++--
 types/uniq_ptr_type.toml               |  8 ++++----
 types/unordered_multiset_type.toml     |  4 ++--
 types/unordered_set_type.toml          |  4 ++--
 34 files changed, 127 insertions(+), 103 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index cfa87e7..78b370e 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -128,8 +128,9 @@ void addIncludes(const TypeGraph& typeGraph,
     includes.emplace("oi/types/st.h");
   }
   if (features[Feature::Library]) {
-    includes.emplace("vector");
+    includes.emplace("memory");
     includes.emplace("oi/IntrospectionResult.h");
+    includes.emplace("vector");
   }
   if (features[Feature::JitTiming]) {
     includes.emplace("chrono");
@@ -428,7 +429,7 @@ void addStandardGetSizeFuncDefs(std::string& code) {
       JLOG("ptr val @");
       JLOGPTR(s_ptr);
       StoreData((uintptr_t)(s_ptr), returnArg);
-      if (s_ptr && pointers.add((uintptr_t)s_ptr)) {
+      if (s_ptr && ctx.pointers.add((uintptr_t)s_ptr)) {
         StoreData(1, returnArg);
         getSizeType(*(s_ptr), returnArg);
       } else {
@@ -649,7 +650,8 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
 
   code += "  static types::st::Unit<DB> ";
   code += funcName;
-  code += "(\n      const ";
+  code += "(\n      Ctx& ctx,\n";
+  code += "    const ";
   code += c.name();
   code += "& t,\n      typename TypeHandler<Ctx, ";
   code += c.name();
@@ -683,7 +685,8 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
     } else {
       code += "delegate";
     }
-    code += "([&t](auto ret) { return OIInternal::getSizeType<Ctx>(t.";
+    code +=
+        "([&ctx, &t](auto ret) { return OIInternal::getSizeType<Ctx>(ctx, t.";
     code += member.name;
     code += ", ret); })";
   }
@@ -921,6 +924,7 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   code += ";\n";
 
   code += "  static types::st::Unit<DB> getSizeType(\n";
+  code += "      Ctx& ctx,\n";
   code += "      const ";
   code += containerWithTypes;
   code += "& container,\n";
@@ -979,7 +983,7 @@ void addCaptureKeySupport(std::string& code) {
     };
 
     template <bool CaptureKeys, typename Ctx, typename T>
-    auto maybeCaptureKey(const T& key, auto returnArg) {
+    auto maybeCaptureKey(Ctx& ctx, const T& key, auto returnArg) {
       if constexpr (CaptureKeys) {
         return returnArg.delegate([&key](auto ret) {
           return CaptureKeyHandler<Ctx, T>::captureKey(key, ret);
@@ -1041,10 +1045,10 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
   code += R"(
     template <typename Ctx, typename T>
     types::st::Unit<typename Ctx::DataBuffer>
-    getSizeType(const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
+    getSizeType(Ctx& ctx, const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
       JLOG("obj @");
       JLOGPTR(&t);
-      return TypeHandler<Ctx, T>::getSizeType(t, returnArg);
+      return TypeHandler<Ctx, T>::getSizeType(ctx, t, returnArg);
     }
 )";
 
@@ -1233,6 +1237,13 @@ void CodeGen::generate(
   if (config_.features[Feature::CaptureThriftIsset]) {
     genDefsThrift(typeGraph, code);
   }
+  if (!config_.features[Feature::TreeBuilderV2]) {
+    code += "namespace {\n";
+    code += "static struct Context {\n";
+    code += "  PointerHashSet<(1 << 20) / sizeof(uintptr_t)> pointers;\n";
+    code += "} ctx;\n";
+    code += "} // namespace\n";
+  }
 
   /*
    * The purpose of the anonymous namespace within `OIInternal` is that
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 1a0315f..8c5dc37 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -262,19 +262,24 @@ void __attribute__((used, retain)) introspect_%2$016x(
     std::vector<uint8_t>& v)
 #pragma GCC diagnostic pop
 {
-  pointers.initialize();
-  pointers.add((uintptr_t)&t);
-
   v.clear();
   v.reserve(4096);
 
+  auto pointers = std::make_unique<PointerHashSet<(1 << 20) / sizeof(uintptr_t)>>();
+  pointers->initialize();
+
   struct Context {
     using DataBuffer = DataBuffer::BackInserter<std::vector<uint8_t>>;
+
+    PointerHashSet<(1 << 20) / sizeof(uintptr_t)>& pointers;
   };
+  Context ctx{ .pointers = *pointers };
+  ctx.pointers.add((uintptr_t)&t);
+
   using ContentType = OIInternal::TypeHandler<Context, OIInternal::__ROOT_TYPE__>::type;
 
   ContentType ret{Context::DataBuffer{v}};
-  OIInternal::getSizeType<Context>(t, ret);
+  OIInternal::getSizeType<Context>(ctx, t, ret);
 }
 )";
 
@@ -319,8 +324,8 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
     func += "      const auto startTime = std::chrono::steady_clock::now();\n";
   }
   func += R"(
-      pointers.initialize();
-      pointers.add((uintptr_t)&t);
+      ctx.pointers.initialize();
+      ctx.pointers.add((uintptr_t)&t);
       auto data = reinterpret_cast<uintptr_t*>(dataBase);
 
       size_t dataSegOffset = 0;
@@ -417,7 +422,7 @@ void FuncGen::DefineTopLevelGetSizeSmartPtr(std::string& testCode,
     func += "      const auto startTime = std::chrono::steady_clock::now();\n";
   }
   func += R"(
-      pointers.initialize();
+      ctx.pointers.initialize();
       auto data = reinterpret_cast<uintptr_t*>(dataBase);
 
       size_t dataSegOffset = 0;
@@ -670,13 +675,14 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   }
   code += R"(
         static types::st::Unit<DB> getSizeType(
+          Ctx& ctx,
           const T& t,
           typename TypeHandler<Ctx, T>::type returnArg) {
               if constexpr(std::is_pointer_v<T>) {
                 JLOG("ptr val @");
                 JLOGPTR(t);
                 auto r0 = returnArg.write((uintptr_t)t);
-                if (t && pointers.add((uintptr_t)t)) {
+                if (t && ctx.pointers.add((uintptr_t)t)) {
                   return r0.template delegate<1>([&t](auto ret) {
                     if constexpr (!std::is_void<std::remove_pointer_t<T>>::value) {
                       return TypeHandler<Ctx, std::remove_pointer_t<T>>::getSizeType(*t, ret);
@@ -719,8 +725,8 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
   oiArray.codegen.traversalFunc = R"(
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
-  tail = tail.delegate([&container, i](auto ret) {
-      return TypeHandler<Ctx, T0>::getSizeType(container.vals[i], ret);
+  tail = tail.delegate([&ctx, &container, i](auto ret) {
+      return TypeHandler<Ctx, T0>::getSizeType(ctx, container.vals[i], ret);
   });
 }
 return tail.finish();
diff --git a/oi/OICodeGen.cpp b/oi/OICodeGen.cpp
index ee18429..b9cfcbb 100644
--- a/oi/OICodeGen.cpp
+++ b/oi/OICodeGen.cpp
@@ -3049,6 +3049,12 @@ bool OICodeGen::generateJitCode(std::string& code) {
     #define SAVE_DATA(val)    StoreData(val, returnArg)
   )");
 
+  code.append("namespace {\n");
+  code.append("static struct Context {\n");
+  code.append("  PointerHashSet<(1 << 20) / sizeof(uintptr_t)> pointers;\n");
+  code.append("} ctx;\n");
+  code.append("} // namespace\n");
+
   FuncGen::DefineJitLog(code, config.features);
 
   // The purpose of the anonymous namespace within `OIInternal` is that
@@ -3267,7 +3273,7 @@ bool OICodeGen::generateJitCode(std::string& code) {
       JLOG("ptr val @");
       JLOGPTR(s_ptr);
       StoreData((uintptr_t)(s_ptr), returnArg);
-      if (s_ptr && pointers.add((uintptr_t)s_ptr)) {
+      if (s_ptr && ctx.pointers.add((uintptr_t)s_ptr)) {
         StoreData(1, returnArg);
         getSizeType(*(s_ptr), returnArg);
       } else {
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index fa0beda..495c781 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -40,10 +40,11 @@ constexpr int oidMagicId = 0x01DE8;
 
 namespace {
 
-class {
+template <size_t Size>
+class PointerHashSet {
  private:
   // 1 MiB of pointers
-  std::array<uintptr_t, (1 << 20) / sizeof(uintptr_t)> data;
+  std::array<uintptr_t, Size> data;
 
   // twang_mix64 hash function, taken from Folly where it is used
   // as the default hash function for 64-bit integers
@@ -84,7 +85,7 @@ class {
   bool add(const auto* p) {
     return add((uintptr_t)p);
   }
-} static pointers;
+};
 
 }  // namespace
 
diff --git a/types/array_type.toml b/types/array_type.toml
index 126af6a..054d0c6 100644
--- a/types/array_type.toml
+++ b/types/array_type.toml
@@ -33,8 +33,8 @@ traversal_func = """
     auto tail = returnArg.write(container.size());
 
     for (auto & it: container) {
-        tail = tail.delegate([&it](auto ret) {
-            return TypeHandler<Ctx, T0>::getSizeType(it, ret);
+        tail = tail.delegate([&ctx, &it](auto ret) {
+            return TypeHandler<Ctx, T0>::getSizeType(ctx, it, ret);
         });
     }
 
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index f838eee..6788750 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -38,8 +38,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index faf2e9d..06c6f72 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_fast_set.toml b/types/f14_fast_set.toml
index b66284b..fb0017c 100644
--- a/types/f14_fast_set.toml
+++ b/types/f14_fast_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index a53c3e6..de1dcda 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_node_set.toml b/types/f14_node_set.toml
index 3ed0b5d..224789e 100644
--- a/types/f14_node_set.toml
+++ b/types/f14_node_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index 8072abf..c09bbf0 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_value_set.toml b/types/f14_value_set.toml
index dd4070d..f37f7d0 100644
--- a/types/f14_value_set.toml
+++ b/types/f14_value_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index ec7ac93..a673bbd 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_vector_set.toml b/types/f14_vector_set.toml
index e98b6d8..d85f36d 100644
--- a/types/f14_vector_set.toml
+++ b/types/f14_vector_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/fb_string_type.toml b/types/fb_string_type.toml
index e0679d3..169be87 100644
--- a/types/fb_string_type.toml
+++ b/types/fb_string_type.toml
@@ -29,7 +29,7 @@ void getSizeType(const %1%<E, T, A, Storage> &container, size_t& returnArg)
         &&
       ((uintptr_t)container.data() >= (uintptr_t)&container);
 
-    if (!inlined && pointers.add((uintptr_t)container.data())) {
+    if (!inlined && ctx.pointers.add((uintptr_t)container.data())) {
       SAVE_SIZE(container.capacity() * sizeof(T));
       SAVE_DATA(1);
     } else {
@@ -60,7 +60,7 @@ if (isStorageInline(container)) {
   category = Category::InlinedStorage;
 } else if (capacity < minLargeSize) {
   category = Category::OwnedHeapStorage;
-} else if (pointers.add(container.data())) {
+} else if (ctx.pointers.add(container.data())) {
   category = Category::ReferenceCountedStorage;
 } else {
   category = Category::AlreadyAttributed;
diff --git a/types/folly_iobuf_queue_type.toml b/types/folly_iobuf_queue_type.toml
index 0af134d..b9ef72c 100644
--- a/types/folly_iobuf_queue_type.toml
+++ b/types/folly_iobuf_queue_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1% &container, size_t& returnArg)
 
     const folly::IOBuf *head = container.front();
     SAVE_DATA((uintptr_t)head);
-    if (head && pointers.add((uintptr_t)head)) {
+    if (head && ctx.pointers.add((uintptr_t)head)) {
         SAVE_DATA(1);
         getSizeType(*head, returnArg);
     } else {
diff --git a/types/list_type.toml b/types/list_type.toml
index 86d457e..3e0da3c 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -38,8 +38,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index 887df6a..a09efff 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -42,12 +42,12 @@ traversal_func = '''
                     .write(container.size());
 
     for (const auto& kv : container) {
-      tail = tail.delegate([&kv](auto ret) {
-        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
-        auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
-          return OIInternal::getSizeType<Ctx>(kv.first, ret);
+      tail = tail.delegate([&ctx, &kv](auto ret) {
+        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, kv.first, ret);
+        auto next = start.delegate([&ctx, &kv](typename TypeHandler<Ctx, T0>::type ret) {
+          return OIInternal::getSizeType<Ctx>(ctx, kv.first, ret);
         });
-        return OIInternal::getSizeType<Ctx>(kv.second, next);
+        return OIInternal::getSizeType<Ctx>(ctx, kv.second, next);
       });
     }
 
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index e05a1e3..a5d628f 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -39,11 +39,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/multi_set_type.toml b/types/multi_set_type.toml
index 00f8239..f238356 100644
--- a/types/multi_set_type.toml
+++ b/types/multi_set_type.toml
@@ -42,8 +42,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/optional_type.toml b/types/optional_type.toml
index 3bd9cff..eeba64e 100644
--- a/types/optional_type.toml
+++ b/types/optional_type.toml
@@ -30,8 +30,8 @@ void getSizeType(const %1%<T>& container, size_t& returnArg) {
 
 traversal_func = """
 if (container.has_value()) {
-  return returnArg.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return returnArg.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 } else {
   return returnArg.template delegate<0>(std::identity());
diff --git a/types/pair_type.toml b/types/pair_type.toml
index ba5bc38..f157144 100644
--- a/types/pair_type.toml
+++ b/types/pair_type.toml
@@ -27,10 +27,10 @@ void getSizeType(const %1%<P,Q> &container, size_t& returnArg)
 """
 
 traversal_func = """
-    return OIInternal::getSizeType<Ctx>(
+    return OIInternal::getSizeType<Ctx>(ctx, 
         container.second,
-        returnArg.delegate([&container](auto ret) {
-            return OIInternal::getSizeType<Ctx>(container.first, ret);
+        returnArg.delegate([&ctx, &container](auto ret) {
+            return OIInternal::getSizeType<Ctx>(ctx, container.first, ret);
         })
     );
 """
diff --git a/types/ref_wrapper_type.toml b/types/ref_wrapper_type.toml
index 949b385..9ed5094 100644
--- a/types/ref_wrapper_type.toml
+++ b/types/ref_wrapper_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &ref, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
     SAVE_DATA((uintptr_t)&(ref.get()));
-    if (pointers.add((uintptr_t)&ref.get())) {
+    if (ctx.pointers.add((uintptr_t)&ref.get())) {
         SAVE_DATA(1);
         getSizeType(ref.get(), returnArg);
     } else {
diff --git a/types/seq_type.toml b/types/seq_type.toml
index 88ddd1d..f561d0b 100644
--- a/types/seq_type.toml
+++ b/types/seq_type.toml
@@ -43,8 +43,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/set_type.toml b/types/set_type.toml
index e29611e..9d3bc3c 100644
--- a/types/set_type.toml
+++ b/types/set_type.toml
@@ -43,8 +43,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 125fd71..bb3a622 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -24,7 +24,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
     if constexpr (!std::is_void<T>::value) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
-        if (s_ptr && pointers.add((uintptr_t)(s_ptr.get()))) {
+        if (s_ptr && ctx.pointers.add((uintptr_t)(s_ptr.get()))) {
             SAVE_DATA(1);
             getSizeType(*(s_ptr.get()), returnArg);
         } else {
@@ -40,12 +40,12 @@ auto tail = returnArg.write((uintptr_t)container.get());
 if constexpr (std::is_void<T0>::value) {
   return tail.template delegate<0>(std::identity());
 } else {
-  bool do_visit = container && pointers.add((uintptr_t)container.get());
+  bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
   if (!do_visit)
     return tail.template delegate<0>(std::identity());
 
-  return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return tail.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 }
 """
diff --git a/types/small_vec_type.toml b/types/small_vec_type.toml
index d129285..a5f1694 100644
--- a/types/small_vec_type.toml
+++ b/types/small_vec_type.toml
@@ -56,8 +56,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&it: container) {
-  tail = tail.delegate([&it](typename TypeHandler<Ctx, T0>::type ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](typename TypeHandler<Ctx, T0>::type ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/sorted_vec_set_type.toml b/types/sorted_vec_set_type.toml
index e8b9483..a803d61 100644
--- a/types/sorted_vec_set_type.toml
+++ b/types/sorted_vec_set_type.toml
@@ -37,8 +37,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (const auto& el : container) {
-  tail = tail.delegate([&el](auto ret) {
-    return OIInternal::getSizeType<Ctx>(el, ret);
+  tail = tail.delegate([&ctx, &el](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, el, ret);
   });
 }
 
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index df06cc7..ee652ec 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -43,12 +43,12 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(key, ret);
-    auto next =  start.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, key, ret);
+    auto next =  start.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index 27142a1..5b2402f 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -46,12 +46,12 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto& kv : container) {
-  tail = tail.delegate([&kv](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
-    auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(kv.first, ret);
+  tail = tail.delegate([&ctx, &kv](auto ret) {
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, kv.first, ret);
+    auto next = start.delegate([&ctx, &kv](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, kv.first, ret);
     });
-    return OIInternal::getSizeType<Ctx>(kv.second, next);
+    return OIInternal::getSizeType<Ctx>(ctx, kv.second, next);
   });
 }
 
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index 261a0be..a6f7671 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -46,8 +46,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index 58364cf..f723f80 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -25,7 +25,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
     if constexpr (!std::is_void<T>::value) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
-        if (u_ptr && pointers.add((uintptr_t)(u_ptr.get()))) {
+        if (u_ptr && ctx.pointers.add((uintptr_t)(u_ptr.get()))) {
             SAVE_DATA(1);
             getSizeType(*(u_ptr.get()), returnArg);
         } else {
@@ -41,12 +41,12 @@ auto tail = returnArg.write((uintptr_t)container.get());
 if constexpr (std::is_void<T0>::value) {
   return tail.template delegate<0>(std::identity());
 } else {
-  bool do_visit = container && pointers.add((uintptr_t)container.get());
+  bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
   if (!do_visit)
     return tail.template delegate<0>(std::identity());
 
-  return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return tail.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 }
 """
diff --git a/types/unordered_multiset_type.toml b/types/unordered_multiset_type.toml
index 5c2e96b..bfcf203 100644
--- a/types/unordered_multiset_type.toml
+++ b/types/unordered_multiset_type.toml
@@ -44,8 +44,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/unordered_set_type.toml b/types/unordered_set_type.toml
index f8d26c9..d9a827a 100644
--- a/types/unordered_set_type.toml
+++ b/types/unordered_set_type.toml
@@ -44,8 +44,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 

From aa9395ef9195101bc118ff122acfbfff864ae581 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 15 Nov 2023 05:26:48 -0800
Subject: [PATCH 025/188] tbv2: add dynamic context passed through all
 functions

Previously for we had some shared state between all requests, noticeably the
pointers set. This change adds a by reference value to all requests which can
hold additional mutable state. The pointers set is moved into this mutable
state for OIL, which means each concurrent request will have its own pointer
set. Doing things this way allows more features to be added in the future
without such a big code modification.

Test plan:
- CI
---
 oi/CodeGen.cpp                         | 25 ++++++++++++++++++-------
 oi/FuncGen.cpp                         | 26 ++++++++++++++++----------
 oi/OICodeGen.cpp                       |  8 +++++++-
 oi/OITraceCode.cpp                     |  7 ++++---
 types/array_type.toml                  |  4 ++--
 types/cxx11_list_type.toml             |  4 ++--
 types/f14_fast_map.toml                |  8 ++++----
 types/f14_fast_set.toml                |  4 ++--
 types/f14_node_map.toml                |  8 ++++----
 types/f14_node_set.toml                |  4 ++--
 types/f14_value_map.toml               |  8 ++++----
 types/f14_value_set.toml               |  4 ++--
 types/f14_vector_map.toml              |  8 ++++----
 types/f14_vector_set.toml              |  4 ++--
 types/fb_string_type.toml              |  4 ++--
 types/folly_iobuf_queue_type.toml      |  2 +-
 types/list_type.toml                   |  4 ++--
 types/map_seq_type.toml                | 10 +++++-----
 types/multi_map_type.toml              |  8 ++++----
 types/multi_set_type.toml              |  4 ++--
 types/optional_type.toml               |  4 ++--
 types/pair_type.toml                   |  6 +++---
 types/ref_wrapper_type.toml            |  2 +-
 types/seq_type.toml                    |  4 ++--
 types/set_type.toml                    |  4 ++--
 types/shrd_ptr_type.toml               |  8 ++++----
 types/small_vec_type.toml              |  4 ++--
 types/sorted_vec_set_type.toml         |  4 ++--
 types/std_map_type.toml                | 10 +++++-----
 types/std_unordered_map_type.toml      | 10 +++++-----
 types/std_unordered_multimap_type.toml |  4 ++--
 types/uniq_ptr_type.toml               |  8 ++++----
 types/unordered_multiset_type.toml     |  4 ++--
 types/unordered_set_type.toml          |  4 ++--
 34 files changed, 127 insertions(+), 103 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index cfa87e7..78b370e 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -128,8 +128,9 @@ void addIncludes(const TypeGraph& typeGraph,
     includes.emplace("oi/types/st.h");
   }
   if (features[Feature::Library]) {
-    includes.emplace("vector");
+    includes.emplace("memory");
     includes.emplace("oi/IntrospectionResult.h");
+    includes.emplace("vector");
   }
   if (features[Feature::JitTiming]) {
     includes.emplace("chrono");
@@ -428,7 +429,7 @@ void addStandardGetSizeFuncDefs(std::string& code) {
       JLOG("ptr val @");
       JLOGPTR(s_ptr);
       StoreData((uintptr_t)(s_ptr), returnArg);
-      if (s_ptr && pointers.add((uintptr_t)s_ptr)) {
+      if (s_ptr && ctx.pointers.add((uintptr_t)s_ptr)) {
         StoreData(1, returnArg);
         getSizeType(*(s_ptr), returnArg);
       } else {
@@ -649,7 +650,8 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
 
   code += "  static types::st::Unit<DB> ";
   code += funcName;
-  code += "(\n      const ";
+  code += "(\n      Ctx& ctx,\n";
+  code += "    const ";
   code += c.name();
   code += "& t,\n      typename TypeHandler<Ctx, ";
   code += c.name();
@@ -683,7 +685,8 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
     } else {
       code += "delegate";
     }
-    code += "([&t](auto ret) { return OIInternal::getSizeType<Ctx>(t.";
+    code +=
+        "([&ctx, &t](auto ret) { return OIInternal::getSizeType<Ctx>(ctx, t.";
     code += member.name;
     code += ", ret); })";
   }
@@ -921,6 +924,7 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   code += ";\n";
 
   code += "  static types::st::Unit<DB> getSizeType(\n";
+  code += "      Ctx& ctx,\n";
   code += "      const ";
   code += containerWithTypes;
   code += "& container,\n";
@@ -979,7 +983,7 @@ void addCaptureKeySupport(std::string& code) {
     };
 
     template <bool CaptureKeys, typename Ctx, typename T>
-    auto maybeCaptureKey(const T& key, auto returnArg) {
+    auto maybeCaptureKey(Ctx& ctx, const T& key, auto returnArg) {
       if constexpr (CaptureKeys) {
         return returnArg.delegate([&key](auto ret) {
           return CaptureKeyHandler<Ctx, T>::captureKey(key, ret);
@@ -1041,10 +1045,10 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
   code += R"(
     template <typename Ctx, typename T>
     types::st::Unit<typename Ctx::DataBuffer>
-    getSizeType(const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
+    getSizeType(Ctx& ctx, const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
       JLOG("obj @");
       JLOGPTR(&t);
-      return TypeHandler<Ctx, T>::getSizeType(t, returnArg);
+      return TypeHandler<Ctx, T>::getSizeType(ctx, t, returnArg);
     }
 )";
 
@@ -1233,6 +1237,13 @@ void CodeGen::generate(
   if (config_.features[Feature::CaptureThriftIsset]) {
     genDefsThrift(typeGraph, code);
   }
+  if (!config_.features[Feature::TreeBuilderV2]) {
+    code += "namespace {\n";
+    code += "static struct Context {\n";
+    code += "  PointerHashSet<(1 << 20) / sizeof(uintptr_t)> pointers;\n";
+    code += "} ctx;\n";
+    code += "} // namespace\n";
+  }
 
   /*
    * The purpose of the anonymous namespace within `OIInternal` is that
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 1a0315f..8c5dc37 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -262,19 +262,24 @@ void __attribute__((used, retain)) introspect_%2$016x(
     std::vector<uint8_t>& v)
 #pragma GCC diagnostic pop
 {
-  pointers.initialize();
-  pointers.add((uintptr_t)&t);
-
   v.clear();
   v.reserve(4096);
 
+  auto pointers = std::make_unique<PointerHashSet<(1 << 20) / sizeof(uintptr_t)>>();
+  pointers->initialize();
+
   struct Context {
     using DataBuffer = DataBuffer::BackInserter<std::vector<uint8_t>>;
+
+    PointerHashSet<(1 << 20) / sizeof(uintptr_t)>& pointers;
   };
+  Context ctx{ .pointers = *pointers };
+  ctx.pointers.add((uintptr_t)&t);
+
   using ContentType = OIInternal::TypeHandler<Context, OIInternal::__ROOT_TYPE__>::type;
 
   ContentType ret{Context::DataBuffer{v}};
-  OIInternal::getSizeType<Context>(t, ret);
+  OIInternal::getSizeType<Context>(ctx, t, ret);
 }
 )";
 
@@ -319,8 +324,8 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
     func += "      const auto startTime = std::chrono::steady_clock::now();\n";
   }
   func += R"(
-      pointers.initialize();
-      pointers.add((uintptr_t)&t);
+      ctx.pointers.initialize();
+      ctx.pointers.add((uintptr_t)&t);
       auto data = reinterpret_cast<uintptr_t*>(dataBase);
 
       size_t dataSegOffset = 0;
@@ -417,7 +422,7 @@ void FuncGen::DefineTopLevelGetSizeSmartPtr(std::string& testCode,
     func += "      const auto startTime = std::chrono::steady_clock::now();\n";
   }
   func += R"(
-      pointers.initialize();
+      ctx.pointers.initialize();
       auto data = reinterpret_cast<uintptr_t*>(dataBase);
 
       size_t dataSegOffset = 0;
@@ -670,13 +675,14 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   }
   code += R"(
         static types::st::Unit<DB> getSizeType(
+          Ctx& ctx,
           const T& t,
           typename TypeHandler<Ctx, T>::type returnArg) {
               if constexpr(std::is_pointer_v<T>) {
                 JLOG("ptr val @");
                 JLOGPTR(t);
                 auto r0 = returnArg.write((uintptr_t)t);
-                if (t && pointers.add((uintptr_t)t)) {
+                if (t && ctx.pointers.add((uintptr_t)t)) {
                   return r0.template delegate<1>([&t](auto ret) {
                     if constexpr (!std::is_void<std::remove_pointer_t<T>>::value) {
                       return TypeHandler<Ctx, std::remove_pointer_t<T>>::getSizeType(*t, ret);
@@ -719,8 +725,8 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
   oiArray.codegen.traversalFunc = R"(
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
-  tail = tail.delegate([&container, i](auto ret) {
-      return TypeHandler<Ctx, T0>::getSizeType(container.vals[i], ret);
+  tail = tail.delegate([&ctx, &container, i](auto ret) {
+      return TypeHandler<Ctx, T0>::getSizeType(ctx, container.vals[i], ret);
   });
 }
 return tail.finish();
diff --git a/oi/OICodeGen.cpp b/oi/OICodeGen.cpp
index ee18429..b9cfcbb 100644
--- a/oi/OICodeGen.cpp
+++ b/oi/OICodeGen.cpp
@@ -3049,6 +3049,12 @@ bool OICodeGen::generateJitCode(std::string& code) {
     #define SAVE_DATA(val)    StoreData(val, returnArg)
   )");
 
+  code.append("namespace {\n");
+  code.append("static struct Context {\n");
+  code.append("  PointerHashSet<(1 << 20) / sizeof(uintptr_t)> pointers;\n");
+  code.append("} ctx;\n");
+  code.append("} // namespace\n");
+
   FuncGen::DefineJitLog(code, config.features);
 
   // The purpose of the anonymous namespace within `OIInternal` is that
@@ -3267,7 +3273,7 @@ bool OICodeGen::generateJitCode(std::string& code) {
       JLOG("ptr val @");
       JLOGPTR(s_ptr);
       StoreData((uintptr_t)(s_ptr), returnArg);
-      if (s_ptr && pointers.add((uintptr_t)s_ptr)) {
+      if (s_ptr && ctx.pointers.add((uintptr_t)s_ptr)) {
         StoreData(1, returnArg);
         getSizeType(*(s_ptr), returnArg);
       } else {
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index fa0beda..495c781 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -40,10 +40,11 @@ constexpr int oidMagicId = 0x01DE8;
 
 namespace {
 
-class {
+template <size_t Size>
+class PointerHashSet {
  private:
   // 1 MiB of pointers
-  std::array<uintptr_t, (1 << 20) / sizeof(uintptr_t)> data;
+  std::array<uintptr_t, Size> data;
 
   // twang_mix64 hash function, taken from Folly where it is used
   // as the default hash function for 64-bit integers
@@ -84,7 +85,7 @@ class {
   bool add(const auto* p) {
     return add((uintptr_t)p);
   }
-} static pointers;
+};
 
 }  // namespace
 
diff --git a/types/array_type.toml b/types/array_type.toml
index 126af6a..054d0c6 100644
--- a/types/array_type.toml
+++ b/types/array_type.toml
@@ -33,8 +33,8 @@ traversal_func = """
     auto tail = returnArg.write(container.size());
 
     for (auto & it: container) {
-        tail = tail.delegate([&it](auto ret) {
-            return TypeHandler<Ctx, T0>::getSizeType(it, ret);
+        tail = tail.delegate([&ctx, &it](auto ret) {
+            return TypeHandler<Ctx, T0>::getSizeType(ctx, it, ret);
         });
     }
 
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index f838eee..6788750 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -38,8 +38,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index faf2e9d..06c6f72 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_fast_set.toml b/types/f14_fast_set.toml
index b66284b..fb0017c 100644
--- a/types/f14_fast_set.toml
+++ b/types/f14_fast_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index a53c3e6..de1dcda 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_node_set.toml b/types/f14_node_set.toml
index 3ed0b5d..224789e 100644
--- a/types/f14_node_set.toml
+++ b/types/f14_node_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index 8072abf..c09bbf0 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_value_set.toml b/types/f14_value_set.toml
index dd4070d..f37f7d0 100644
--- a/types/f14_value_set.toml
+++ b/types/f14_value_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index ec7ac93..a673bbd 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_vector_set.toml b/types/f14_vector_set.toml
index e98b6d8..d85f36d 100644
--- a/types/f14_vector_set.toml
+++ b/types/f14_vector_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/fb_string_type.toml b/types/fb_string_type.toml
index e0679d3..169be87 100644
--- a/types/fb_string_type.toml
+++ b/types/fb_string_type.toml
@@ -29,7 +29,7 @@ void getSizeType(const %1%<E, T, A, Storage> &container, size_t& returnArg)
         &&
       ((uintptr_t)container.data() >= (uintptr_t)&container);
 
-    if (!inlined && pointers.add((uintptr_t)container.data())) {
+    if (!inlined && ctx.pointers.add((uintptr_t)container.data())) {
       SAVE_SIZE(container.capacity() * sizeof(T));
       SAVE_DATA(1);
     } else {
@@ -60,7 +60,7 @@ if (isStorageInline(container)) {
   category = Category::InlinedStorage;
 } else if (capacity < minLargeSize) {
   category = Category::OwnedHeapStorage;
-} else if (pointers.add(container.data())) {
+} else if (ctx.pointers.add(container.data())) {
   category = Category::ReferenceCountedStorage;
 } else {
   category = Category::AlreadyAttributed;
diff --git a/types/folly_iobuf_queue_type.toml b/types/folly_iobuf_queue_type.toml
index 0af134d..b9ef72c 100644
--- a/types/folly_iobuf_queue_type.toml
+++ b/types/folly_iobuf_queue_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1% &container, size_t& returnArg)
 
     const folly::IOBuf *head = container.front();
     SAVE_DATA((uintptr_t)head);
-    if (head && pointers.add((uintptr_t)head)) {
+    if (head && ctx.pointers.add((uintptr_t)head)) {
         SAVE_DATA(1);
         getSizeType(*head, returnArg);
     } else {
diff --git a/types/list_type.toml b/types/list_type.toml
index 86d457e..3e0da3c 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -38,8 +38,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index 887df6a..a09efff 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -42,12 +42,12 @@ traversal_func = '''
                     .write(container.size());
 
     for (const auto& kv : container) {
-      tail = tail.delegate([&kv](auto ret) {
-        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
-        auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
-          return OIInternal::getSizeType<Ctx>(kv.first, ret);
+      tail = tail.delegate([&ctx, &kv](auto ret) {
+        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, kv.first, ret);
+        auto next = start.delegate([&ctx, &kv](typename TypeHandler<Ctx, T0>::type ret) {
+          return OIInternal::getSizeType<Ctx>(ctx, kv.first, ret);
         });
-        return OIInternal::getSizeType<Ctx>(kv.second, next);
+        return OIInternal::getSizeType<Ctx>(ctx, kv.second, next);
       });
     }
 
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index e05a1e3..a5d628f 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -39,11 +39,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/multi_set_type.toml b/types/multi_set_type.toml
index 00f8239..f238356 100644
--- a/types/multi_set_type.toml
+++ b/types/multi_set_type.toml
@@ -42,8 +42,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/optional_type.toml b/types/optional_type.toml
index 3bd9cff..eeba64e 100644
--- a/types/optional_type.toml
+++ b/types/optional_type.toml
@@ -30,8 +30,8 @@ void getSizeType(const %1%<T>& container, size_t& returnArg) {
 
 traversal_func = """
 if (container.has_value()) {
-  return returnArg.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return returnArg.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 } else {
   return returnArg.template delegate<0>(std::identity());
diff --git a/types/pair_type.toml b/types/pair_type.toml
index ba5bc38..f157144 100644
--- a/types/pair_type.toml
+++ b/types/pair_type.toml
@@ -27,10 +27,10 @@ void getSizeType(const %1%<P,Q> &container, size_t& returnArg)
 """
 
 traversal_func = """
-    return OIInternal::getSizeType<Ctx>(
+    return OIInternal::getSizeType<Ctx>(ctx, 
         container.second,
-        returnArg.delegate([&container](auto ret) {
-            return OIInternal::getSizeType<Ctx>(container.first, ret);
+        returnArg.delegate([&ctx, &container](auto ret) {
+            return OIInternal::getSizeType<Ctx>(ctx, container.first, ret);
         })
     );
 """
diff --git a/types/ref_wrapper_type.toml b/types/ref_wrapper_type.toml
index 949b385..9ed5094 100644
--- a/types/ref_wrapper_type.toml
+++ b/types/ref_wrapper_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &ref, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
     SAVE_DATA((uintptr_t)&(ref.get()));
-    if (pointers.add((uintptr_t)&ref.get())) {
+    if (ctx.pointers.add((uintptr_t)&ref.get())) {
         SAVE_DATA(1);
         getSizeType(ref.get(), returnArg);
     } else {
diff --git a/types/seq_type.toml b/types/seq_type.toml
index 88ddd1d..f561d0b 100644
--- a/types/seq_type.toml
+++ b/types/seq_type.toml
@@ -43,8 +43,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/set_type.toml b/types/set_type.toml
index e29611e..9d3bc3c 100644
--- a/types/set_type.toml
+++ b/types/set_type.toml
@@ -43,8 +43,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 125fd71..bb3a622 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -24,7 +24,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
     if constexpr (!std::is_void<T>::value) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
-        if (s_ptr && pointers.add((uintptr_t)(s_ptr.get()))) {
+        if (s_ptr && ctx.pointers.add((uintptr_t)(s_ptr.get()))) {
             SAVE_DATA(1);
             getSizeType(*(s_ptr.get()), returnArg);
         } else {
@@ -40,12 +40,12 @@ auto tail = returnArg.write((uintptr_t)container.get());
 if constexpr (std::is_void<T0>::value) {
   return tail.template delegate<0>(std::identity());
 } else {
-  bool do_visit = container && pointers.add((uintptr_t)container.get());
+  bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
   if (!do_visit)
     return tail.template delegate<0>(std::identity());
 
-  return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return tail.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 }
 """
diff --git a/types/small_vec_type.toml b/types/small_vec_type.toml
index d129285..a5f1694 100644
--- a/types/small_vec_type.toml
+++ b/types/small_vec_type.toml
@@ -56,8 +56,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&it: container) {
-  tail = tail.delegate([&it](typename TypeHandler<Ctx, T0>::type ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](typename TypeHandler<Ctx, T0>::type ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/sorted_vec_set_type.toml b/types/sorted_vec_set_type.toml
index e8b9483..a803d61 100644
--- a/types/sorted_vec_set_type.toml
+++ b/types/sorted_vec_set_type.toml
@@ -37,8 +37,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (const auto& el : container) {
-  tail = tail.delegate([&el](auto ret) {
-    return OIInternal::getSizeType<Ctx>(el, ret);
+  tail = tail.delegate([&ctx, &el](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, el, ret);
   });
 }
 
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index df06cc7..ee652ec 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -43,12 +43,12 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(key, ret);
-    auto next =  start.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, key, ret);
+    auto next =  start.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index 27142a1..5b2402f 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -46,12 +46,12 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto& kv : container) {
-  tail = tail.delegate([&kv](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
-    auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(kv.first, ret);
+  tail = tail.delegate([&ctx, &kv](auto ret) {
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, kv.first, ret);
+    auto next = start.delegate([&ctx, &kv](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, kv.first, ret);
     });
-    return OIInternal::getSizeType<Ctx>(kv.second, next);
+    return OIInternal::getSizeType<Ctx>(ctx, kv.second, next);
   });
 }
 
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index 261a0be..a6f7671 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -46,8 +46,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index 58364cf..f723f80 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -25,7 +25,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
     if constexpr (!std::is_void<T>::value) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
-        if (u_ptr && pointers.add((uintptr_t)(u_ptr.get()))) {
+        if (u_ptr && ctx.pointers.add((uintptr_t)(u_ptr.get()))) {
             SAVE_DATA(1);
             getSizeType(*(u_ptr.get()), returnArg);
         } else {
@@ -41,12 +41,12 @@ auto tail = returnArg.write((uintptr_t)container.get());
 if constexpr (std::is_void<T0>::value) {
   return tail.template delegate<0>(std::identity());
 } else {
-  bool do_visit = container && pointers.add((uintptr_t)container.get());
+  bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
   if (!do_visit)
     return tail.template delegate<0>(std::identity());
 
-  return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return tail.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 }
 """
diff --git a/types/unordered_multiset_type.toml b/types/unordered_multiset_type.toml
index 5c2e96b..bfcf203 100644
--- a/types/unordered_multiset_type.toml
+++ b/types/unordered_multiset_type.toml
@@ -44,8 +44,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/unordered_set_type.toml b/types/unordered_set_type.toml
index f8d26c9..d9a827a 100644
--- a/types/unordered_set_type.toml
+++ b/types/unordered_set_type.toml
@@ -44,8 +44,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 

From 7072751f5675993ed3c2fe0169c4c224b56c357d Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 15 Nov 2023 05:38:09 -0800
Subject: [PATCH 026/188] tbv2: add dynamic context passed through all
 functions

Previously for we had some shared state between all requests, noticeably the
pointers set. This change adds a by reference value to all requests which can
hold additional mutable state. The pointers set is moved into this mutable
state for OIL, which means each concurrent request will have its own pointer
set. Doing things this way allows more features to be added in the future
without such a big code modification.

Closes #404

Test plan:
- CI
---
 oi/CodeGen.cpp                         | 25 ++++++++++++++++++-------
 oi/FuncGen.cpp                         | 26 ++++++++++++++++----------
 oi/OICodeGen.cpp                       |  8 +++++++-
 oi/OITraceCode.cpp                     |  7 ++++---
 types/array_type.toml                  |  4 ++--
 types/cxx11_list_type.toml             |  4 ++--
 types/f14_fast_map.toml                |  8 ++++----
 types/f14_fast_set.toml                |  4 ++--
 types/f14_node_map.toml                |  8 ++++----
 types/f14_node_set.toml                |  4 ++--
 types/f14_value_map.toml               |  8 ++++----
 types/f14_value_set.toml               |  4 ++--
 types/f14_vector_map.toml              |  8 ++++----
 types/f14_vector_set.toml              |  4 ++--
 types/fb_string_type.toml              |  4 ++--
 types/folly_iobuf_queue_type.toml      |  2 +-
 types/list_type.toml                   |  4 ++--
 types/map_seq_type.toml                | 10 +++++-----
 types/multi_map_type.toml              |  8 ++++----
 types/multi_set_type.toml              |  4 ++--
 types/optional_type.toml               |  4 ++--
 types/pair_type.toml                   |  6 +++---
 types/ref_wrapper_type.toml            |  2 +-
 types/seq_type.toml                    |  4 ++--
 types/set_type.toml                    |  4 ++--
 types/shrd_ptr_type.toml               |  8 ++++----
 types/small_vec_type.toml              |  4 ++--
 types/sorted_vec_set_type.toml         |  4 ++--
 types/std_map_type.toml                | 10 +++++-----
 types/std_unordered_map_type.toml      | 10 +++++-----
 types/std_unordered_multimap_type.toml |  4 ++--
 types/uniq_ptr_type.toml               |  8 ++++----
 types/unordered_multiset_type.toml     |  4 ++--
 types/unordered_set_type.toml          |  4 ++--
 34 files changed, 127 insertions(+), 103 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index cfa87e7..78b370e 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -128,8 +128,9 @@ void addIncludes(const TypeGraph& typeGraph,
     includes.emplace("oi/types/st.h");
   }
   if (features[Feature::Library]) {
-    includes.emplace("vector");
+    includes.emplace("memory");
     includes.emplace("oi/IntrospectionResult.h");
+    includes.emplace("vector");
   }
   if (features[Feature::JitTiming]) {
     includes.emplace("chrono");
@@ -428,7 +429,7 @@ void addStandardGetSizeFuncDefs(std::string& code) {
       JLOG("ptr val @");
       JLOGPTR(s_ptr);
       StoreData((uintptr_t)(s_ptr), returnArg);
-      if (s_ptr && pointers.add((uintptr_t)s_ptr)) {
+      if (s_ptr && ctx.pointers.add((uintptr_t)s_ptr)) {
         StoreData(1, returnArg);
         getSizeType(*(s_ptr), returnArg);
       } else {
@@ -649,7 +650,8 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
 
   code += "  static types::st::Unit<DB> ";
   code += funcName;
-  code += "(\n      const ";
+  code += "(\n      Ctx& ctx,\n";
+  code += "    const ";
   code += c.name();
   code += "& t,\n      typename TypeHandler<Ctx, ";
   code += c.name();
@@ -683,7 +685,8 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
     } else {
       code += "delegate";
     }
-    code += "([&t](auto ret) { return OIInternal::getSizeType<Ctx>(t.";
+    code +=
+        "([&ctx, &t](auto ret) { return OIInternal::getSizeType<Ctx>(ctx, t.";
     code += member.name;
     code += ", ret); })";
   }
@@ -921,6 +924,7 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   code += ";\n";
 
   code += "  static types::st::Unit<DB> getSizeType(\n";
+  code += "      Ctx& ctx,\n";
   code += "      const ";
   code += containerWithTypes;
   code += "& container,\n";
@@ -979,7 +983,7 @@ void addCaptureKeySupport(std::string& code) {
     };
 
     template <bool CaptureKeys, typename Ctx, typename T>
-    auto maybeCaptureKey(const T& key, auto returnArg) {
+    auto maybeCaptureKey(Ctx& ctx, const T& key, auto returnArg) {
       if constexpr (CaptureKeys) {
         return returnArg.delegate([&key](auto ret) {
           return CaptureKeyHandler<Ctx, T>::captureKey(key, ret);
@@ -1041,10 +1045,10 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
   code += R"(
     template <typename Ctx, typename T>
     types::st::Unit<typename Ctx::DataBuffer>
-    getSizeType(const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
+    getSizeType(Ctx& ctx, const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
       JLOG("obj @");
       JLOGPTR(&t);
-      return TypeHandler<Ctx, T>::getSizeType(t, returnArg);
+      return TypeHandler<Ctx, T>::getSizeType(ctx, t, returnArg);
     }
 )";
 
@@ -1233,6 +1237,13 @@ void CodeGen::generate(
   if (config_.features[Feature::CaptureThriftIsset]) {
     genDefsThrift(typeGraph, code);
   }
+  if (!config_.features[Feature::TreeBuilderV2]) {
+    code += "namespace {\n";
+    code += "static struct Context {\n";
+    code += "  PointerHashSet<(1 << 20) / sizeof(uintptr_t)> pointers;\n";
+    code += "} ctx;\n";
+    code += "} // namespace\n";
+  }
 
   /*
    * The purpose of the anonymous namespace within `OIInternal` is that
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 1a0315f..8c5dc37 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -262,19 +262,24 @@ void __attribute__((used, retain)) introspect_%2$016x(
     std::vector<uint8_t>& v)
 #pragma GCC diagnostic pop
 {
-  pointers.initialize();
-  pointers.add((uintptr_t)&t);
-
   v.clear();
   v.reserve(4096);
 
+  auto pointers = std::make_unique<PointerHashSet<(1 << 20) / sizeof(uintptr_t)>>();
+  pointers->initialize();
+
   struct Context {
     using DataBuffer = DataBuffer::BackInserter<std::vector<uint8_t>>;
+
+    PointerHashSet<(1 << 20) / sizeof(uintptr_t)>& pointers;
   };
+  Context ctx{ .pointers = *pointers };
+  ctx.pointers.add((uintptr_t)&t);
+
   using ContentType = OIInternal::TypeHandler<Context, OIInternal::__ROOT_TYPE__>::type;
 
   ContentType ret{Context::DataBuffer{v}};
-  OIInternal::getSizeType<Context>(t, ret);
+  OIInternal::getSizeType<Context>(ctx, t, ret);
 }
 )";
 
@@ -319,8 +324,8 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
     func += "      const auto startTime = std::chrono::steady_clock::now();\n";
   }
   func += R"(
-      pointers.initialize();
-      pointers.add((uintptr_t)&t);
+      ctx.pointers.initialize();
+      ctx.pointers.add((uintptr_t)&t);
       auto data = reinterpret_cast<uintptr_t*>(dataBase);
 
       size_t dataSegOffset = 0;
@@ -417,7 +422,7 @@ void FuncGen::DefineTopLevelGetSizeSmartPtr(std::string& testCode,
     func += "      const auto startTime = std::chrono::steady_clock::now();\n";
   }
   func += R"(
-      pointers.initialize();
+      ctx.pointers.initialize();
       auto data = reinterpret_cast<uintptr_t*>(dataBase);
 
       size_t dataSegOffset = 0;
@@ -670,13 +675,14 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   }
   code += R"(
         static types::st::Unit<DB> getSizeType(
+          Ctx& ctx,
           const T& t,
           typename TypeHandler<Ctx, T>::type returnArg) {
               if constexpr(std::is_pointer_v<T>) {
                 JLOG("ptr val @");
                 JLOGPTR(t);
                 auto r0 = returnArg.write((uintptr_t)t);
-                if (t && pointers.add((uintptr_t)t)) {
+                if (t && ctx.pointers.add((uintptr_t)t)) {
                   return r0.template delegate<1>([&t](auto ret) {
                     if constexpr (!std::is_void<std::remove_pointer_t<T>>::value) {
                       return TypeHandler<Ctx, std::remove_pointer_t<T>>::getSizeType(*t, ret);
@@ -719,8 +725,8 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
   oiArray.codegen.traversalFunc = R"(
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
-  tail = tail.delegate([&container, i](auto ret) {
-      return TypeHandler<Ctx, T0>::getSizeType(container.vals[i], ret);
+  tail = tail.delegate([&ctx, &container, i](auto ret) {
+      return TypeHandler<Ctx, T0>::getSizeType(ctx, container.vals[i], ret);
   });
 }
 return tail.finish();
diff --git a/oi/OICodeGen.cpp b/oi/OICodeGen.cpp
index ee18429..b9cfcbb 100644
--- a/oi/OICodeGen.cpp
+++ b/oi/OICodeGen.cpp
@@ -3049,6 +3049,12 @@ bool OICodeGen::generateJitCode(std::string& code) {
     #define SAVE_DATA(val)    StoreData(val, returnArg)
   )");
 
+  code.append("namespace {\n");
+  code.append("static struct Context {\n");
+  code.append("  PointerHashSet<(1 << 20) / sizeof(uintptr_t)> pointers;\n");
+  code.append("} ctx;\n");
+  code.append("} // namespace\n");
+
   FuncGen::DefineJitLog(code, config.features);
 
   // The purpose of the anonymous namespace within `OIInternal` is that
@@ -3267,7 +3273,7 @@ bool OICodeGen::generateJitCode(std::string& code) {
       JLOG("ptr val @");
       JLOGPTR(s_ptr);
       StoreData((uintptr_t)(s_ptr), returnArg);
-      if (s_ptr && pointers.add((uintptr_t)s_ptr)) {
+      if (s_ptr && ctx.pointers.add((uintptr_t)s_ptr)) {
         StoreData(1, returnArg);
         getSizeType(*(s_ptr), returnArg);
       } else {
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index fa0beda..495c781 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -40,10 +40,11 @@ constexpr int oidMagicId = 0x01DE8;
 
 namespace {
 
-class {
+template <size_t Size>
+class PointerHashSet {
  private:
   // 1 MiB of pointers
-  std::array<uintptr_t, (1 << 20) / sizeof(uintptr_t)> data;
+  std::array<uintptr_t, Size> data;
 
   // twang_mix64 hash function, taken from Folly where it is used
   // as the default hash function for 64-bit integers
@@ -84,7 +85,7 @@ class {
   bool add(const auto* p) {
     return add((uintptr_t)p);
   }
-} static pointers;
+};
 
 }  // namespace
 
diff --git a/types/array_type.toml b/types/array_type.toml
index 126af6a..054d0c6 100644
--- a/types/array_type.toml
+++ b/types/array_type.toml
@@ -33,8 +33,8 @@ traversal_func = """
     auto tail = returnArg.write(container.size());
 
     for (auto & it: container) {
-        tail = tail.delegate([&it](auto ret) {
-            return TypeHandler<Ctx, T0>::getSizeType(it, ret);
+        tail = tail.delegate([&ctx, &it](auto ret) {
+            return TypeHandler<Ctx, T0>::getSizeType(ctx, it, ret);
         });
     }
 
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index f838eee..6788750 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -38,8 +38,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index faf2e9d..06c6f72 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_fast_set.toml b/types/f14_fast_set.toml
index b66284b..fb0017c 100644
--- a/types/f14_fast_set.toml
+++ b/types/f14_fast_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index a53c3e6..de1dcda 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_node_set.toml b/types/f14_node_set.toml
index 3ed0b5d..224789e 100644
--- a/types/f14_node_set.toml
+++ b/types/f14_node_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index 8072abf..c09bbf0 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_value_set.toml b/types/f14_value_set.toml
index dd4070d..f37f7d0 100644
--- a/types/f14_value_set.toml
+++ b/types/f14_value_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index ec7ac93..a673bbd 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_vector_set.toml b/types/f14_vector_set.toml
index e98b6d8..d85f36d 100644
--- a/types/f14_vector_set.toml
+++ b/types/f14_vector_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/fb_string_type.toml b/types/fb_string_type.toml
index e0679d3..169be87 100644
--- a/types/fb_string_type.toml
+++ b/types/fb_string_type.toml
@@ -29,7 +29,7 @@ void getSizeType(const %1%<E, T, A, Storage> &container, size_t& returnArg)
         &&
       ((uintptr_t)container.data() >= (uintptr_t)&container);
 
-    if (!inlined && pointers.add((uintptr_t)container.data())) {
+    if (!inlined && ctx.pointers.add((uintptr_t)container.data())) {
       SAVE_SIZE(container.capacity() * sizeof(T));
       SAVE_DATA(1);
     } else {
@@ -60,7 +60,7 @@ if (isStorageInline(container)) {
   category = Category::InlinedStorage;
 } else if (capacity < minLargeSize) {
   category = Category::OwnedHeapStorage;
-} else if (pointers.add(container.data())) {
+} else if (ctx.pointers.add(container.data())) {
   category = Category::ReferenceCountedStorage;
 } else {
   category = Category::AlreadyAttributed;
diff --git a/types/folly_iobuf_queue_type.toml b/types/folly_iobuf_queue_type.toml
index 0af134d..b9ef72c 100644
--- a/types/folly_iobuf_queue_type.toml
+++ b/types/folly_iobuf_queue_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1% &container, size_t& returnArg)
 
     const folly::IOBuf *head = container.front();
     SAVE_DATA((uintptr_t)head);
-    if (head && pointers.add((uintptr_t)head)) {
+    if (head && ctx.pointers.add((uintptr_t)head)) {
         SAVE_DATA(1);
         getSizeType(*head, returnArg);
     } else {
diff --git a/types/list_type.toml b/types/list_type.toml
index 86d457e..3e0da3c 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -38,8 +38,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index 887df6a..a09efff 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -42,12 +42,12 @@ traversal_func = '''
                     .write(container.size());
 
     for (const auto& kv : container) {
-      tail = tail.delegate([&kv](auto ret) {
-        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
-        auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
-          return OIInternal::getSizeType<Ctx>(kv.first, ret);
+      tail = tail.delegate([&ctx, &kv](auto ret) {
+        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, kv.first, ret);
+        auto next = start.delegate([&ctx, &kv](typename TypeHandler<Ctx, T0>::type ret) {
+          return OIInternal::getSizeType<Ctx>(ctx, kv.first, ret);
         });
-        return OIInternal::getSizeType<Ctx>(kv.second, next);
+        return OIInternal::getSizeType<Ctx>(ctx, kv.second, next);
       });
     }
 
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index e05a1e3..a5d628f 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -39,11 +39,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/multi_set_type.toml b/types/multi_set_type.toml
index 00f8239..f238356 100644
--- a/types/multi_set_type.toml
+++ b/types/multi_set_type.toml
@@ -42,8 +42,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/optional_type.toml b/types/optional_type.toml
index 3bd9cff..eeba64e 100644
--- a/types/optional_type.toml
+++ b/types/optional_type.toml
@@ -30,8 +30,8 @@ void getSizeType(const %1%<T>& container, size_t& returnArg) {
 
 traversal_func = """
 if (container.has_value()) {
-  return returnArg.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return returnArg.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 } else {
   return returnArg.template delegate<0>(std::identity());
diff --git a/types/pair_type.toml b/types/pair_type.toml
index ba5bc38..f157144 100644
--- a/types/pair_type.toml
+++ b/types/pair_type.toml
@@ -27,10 +27,10 @@ void getSizeType(const %1%<P,Q> &container, size_t& returnArg)
 """
 
 traversal_func = """
-    return OIInternal::getSizeType<Ctx>(
+    return OIInternal::getSizeType<Ctx>(ctx, 
         container.second,
-        returnArg.delegate([&container](auto ret) {
-            return OIInternal::getSizeType<Ctx>(container.first, ret);
+        returnArg.delegate([&ctx, &container](auto ret) {
+            return OIInternal::getSizeType<Ctx>(ctx, container.first, ret);
         })
     );
 """
diff --git a/types/ref_wrapper_type.toml b/types/ref_wrapper_type.toml
index 949b385..9ed5094 100644
--- a/types/ref_wrapper_type.toml
+++ b/types/ref_wrapper_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &ref, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
     SAVE_DATA((uintptr_t)&(ref.get()));
-    if (pointers.add((uintptr_t)&ref.get())) {
+    if (ctx.pointers.add((uintptr_t)&ref.get())) {
         SAVE_DATA(1);
         getSizeType(ref.get(), returnArg);
     } else {
diff --git a/types/seq_type.toml b/types/seq_type.toml
index 88ddd1d..f561d0b 100644
--- a/types/seq_type.toml
+++ b/types/seq_type.toml
@@ -43,8 +43,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/set_type.toml b/types/set_type.toml
index e29611e..9d3bc3c 100644
--- a/types/set_type.toml
+++ b/types/set_type.toml
@@ -43,8 +43,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 125fd71..bb3a622 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -24,7 +24,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
     if constexpr (!std::is_void<T>::value) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
-        if (s_ptr && pointers.add((uintptr_t)(s_ptr.get()))) {
+        if (s_ptr && ctx.pointers.add((uintptr_t)(s_ptr.get()))) {
             SAVE_DATA(1);
             getSizeType(*(s_ptr.get()), returnArg);
         } else {
@@ -40,12 +40,12 @@ auto tail = returnArg.write((uintptr_t)container.get());
 if constexpr (std::is_void<T0>::value) {
   return tail.template delegate<0>(std::identity());
 } else {
-  bool do_visit = container && pointers.add((uintptr_t)container.get());
+  bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
   if (!do_visit)
     return tail.template delegate<0>(std::identity());
 
-  return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return tail.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 }
 """
diff --git a/types/small_vec_type.toml b/types/small_vec_type.toml
index d129285..a5f1694 100644
--- a/types/small_vec_type.toml
+++ b/types/small_vec_type.toml
@@ -56,8 +56,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&it: container) {
-  tail = tail.delegate([&it](typename TypeHandler<Ctx, T0>::type ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](typename TypeHandler<Ctx, T0>::type ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/sorted_vec_set_type.toml b/types/sorted_vec_set_type.toml
index e8b9483..a803d61 100644
--- a/types/sorted_vec_set_type.toml
+++ b/types/sorted_vec_set_type.toml
@@ -37,8 +37,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (const auto& el : container) {
-  tail = tail.delegate([&el](auto ret) {
-    return OIInternal::getSizeType<Ctx>(el, ret);
+  tail = tail.delegate([&ctx, &el](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, el, ret);
   });
 }
 
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index df06cc7..ee652ec 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -43,12 +43,12 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(key, ret);
-    auto next =  start.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, key, ret);
+    auto next =  start.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index 27142a1..5b2402f 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -46,12 +46,12 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto& kv : container) {
-  tail = tail.delegate([&kv](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
-    auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(kv.first, ret);
+  tail = tail.delegate([&ctx, &kv](auto ret) {
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, kv.first, ret);
+    auto next = start.delegate([&ctx, &kv](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, kv.first, ret);
     });
-    return OIInternal::getSizeType<Ctx>(kv.second, next);
+    return OIInternal::getSizeType<Ctx>(ctx, kv.second, next);
   });
 }
 
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index 261a0be..a6f7671 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -46,8 +46,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index 58364cf..f723f80 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -25,7 +25,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
     if constexpr (!std::is_void<T>::value) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
-        if (u_ptr && pointers.add((uintptr_t)(u_ptr.get()))) {
+        if (u_ptr && ctx.pointers.add((uintptr_t)(u_ptr.get()))) {
             SAVE_DATA(1);
             getSizeType(*(u_ptr.get()), returnArg);
         } else {
@@ -41,12 +41,12 @@ auto tail = returnArg.write((uintptr_t)container.get());
 if constexpr (std::is_void<T0>::value) {
   return tail.template delegate<0>(std::identity());
 } else {
-  bool do_visit = container && pointers.add((uintptr_t)container.get());
+  bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
   if (!do_visit)
     return tail.template delegate<0>(std::identity());
 
-  return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return tail.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 }
 """
diff --git a/types/unordered_multiset_type.toml b/types/unordered_multiset_type.toml
index 5c2e96b..bfcf203 100644
--- a/types/unordered_multiset_type.toml
+++ b/types/unordered_multiset_type.toml
@@ -44,8 +44,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/unordered_set_type.toml b/types/unordered_set_type.toml
index f8d26c9..d9a827a 100644
--- a/types/unordered_set_type.toml
+++ b/types/unordered_set_type.toml
@@ -44,8 +44,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 

From 8647b87c1b88a95bafe9813b61f4141aa74c8a9f Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 15 Nov 2023 06:14:52 -0800
Subject: [PATCH 027/188] types: remove now unused handlers

Summary:

Handlers were added in an intermediate form of tbv2 but those intermediate
forms have now been removed. Remove all the handlers to make grepping/find and
replaces easier across the types.

Test Plan:

- CI
---
 types/array_type.toml                         | 21 -----------
 types/cxx11_string_type.toml                  | 18 ----------
 types/deque_list_type.toml                    | 25 -------------
 types/f14_fast_map.toml                       | 34 ------------------
 types/f14_fast_set.toml                       | 30 ----------------
 types/f14_node_map.toml                       | 34 ------------------
 types/f14_node_set.toml                       | 30 ----------------
 types/f14_value_map.toml                      | 34 ------------------
 types/f14_value_set.toml                      | 30 ----------------
 types/f14_vector_map.toml                     | 34 ------------------
 types/f14_vector_set.toml                     | 30 ----------------
 types/fb_string_type.toml                     | 32 -----------------
 types/map_seq_type.toml                       | 31 ----------------
 types/multi_map_type.toml                     | 28 ---------------
 types/multi_set_type.toml                     | 25 -------------
 types/optional_type.toml                      | 22 ------------
 types/pair_type.toml                          | 20 -----------
 ...priority_queue_container_adapter_type.toml | 18 ----------
 types/queue_container_adapter_type.toml       | 17 ---------
 types/ref_wrapper_type.toml                   | 26 --------------
 types/seq_type.toml                           | 29 ---------------
 types/set_type.toml                           | 28 ---------------
 types/shrd_ptr_type.toml                      | 32 -----------------
 types/small_vec_type.toml                     | 27 --------------
 types/sorted_vec_set_type.toml                | 17 ---------
 types/stack_container_adapter_type.toml       | 18 ----------
 types/std_map_type.toml                       | 32 -----------------
 types/std_unordered_map_type.toml             | 36 -------------------
 types/std_unordered_multimap_type.toml        | 36 -------------------
 types/std_variant.toml                        | 32 ++---------------
 types/thrift_isset_type.toml                  |  3 --
 types/uniq_ptr_type.toml                      | 32 -----------------
 types/unordered_multiset_type.toml            | 31 ----------------
 types/unordered_set_type.toml                 | 31 ----------------
 types/weak_ptr_type.toml                      | 13 -------
 35 files changed, 2 insertions(+), 934 deletions(-)

diff --git a/types/array_type.toml b/types/array_type.toml
index 92e72ca..6d73322 100644
--- a/types/array_type.toml
+++ b/types/array_type.toml
@@ -29,27 +29,6 @@ void getSizeType(const %1%<T,N> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template<typename DB, typename T0, long unsigned int N>
-struct TypeHandler<DB, %1%<T0, N>> {
-  using type = types::st::List<DB, typename TypeHandler<DB, T0>::type>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, N> &container,
-      typename TypeHandler<DB, %1%<T0,N>>::type returnArg) {
-    auto tail = returnArg.write(container.size());
-
-    for (auto & it: container) {
-        tail = tail.delegate([&it](auto ret) {
-            return TypeHandler<DB, T0>::getSizeType(it, ret);
-        });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
     auto tail = returnArg.write(container.size());
 
diff --git a/types/cxx11_string_type.toml b/types/cxx11_string_type.toml
index 1ecc931..94e7183 100644
--- a/types/cxx11_string_type.toml
+++ b/types/cxx11_string_type.toml
@@ -36,24 +36,6 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0>
-struct TypeHandler<DB, %1% <T0>> {
-  using type =
-      types::st::Pair<DB, types::st::VarInt<DB>, types::st::VarInt<DB>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1% <T0> & container,
-      typename TypeHandler<DB, %1% <T0>>::type returnArg) {
-    bool sso = ((uintptr_t)container.data() <
-                (uintptr_t)(&container + sizeof(%1% <T0>))) &&
-               ((uintptr_t)container.data() >= (uintptr_t)&container);
-
-    return returnArg.write(container.capacity()).write(container.size());
-  }
-};
-"""
-
 extra = """
 template <typename DB, typename CharT, typename Traits, typename Allocator>
 class CaptureKeyHandler<DB, std::__cxx11::basic_string<CharT, Traits, Allocator>> {
diff --git a/types/deque_list_type.toml b/types/deque_list_type.toml
index eb0015d..a59742a 100644
--- a/types/deque_list_type.toml
+++ b/types/deque_list_type.toml
@@ -33,28 +33,3 @@ void getSizeType(const %1%<T, Allocator> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1>
-struct TypeHandler<DB, %1%<T0, T1>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB, typename TypeHandler<DB, T0>::type>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1>& container,
-      typename TypeHandler<DB, %1%<T0, T1>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container)
-                    .write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return TypeHandler<DB, T0>::getSizeType(it, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index 87d8d87..c8c84f2 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -36,40 +36,6 @@ void getSizeType(const %1%<Key, Mapped, Hasher, KeyEqual, Alloc> &container, siz
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3, typename T4>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3, T4>> {
-  using type = types::st::Pair<DB,
-    types::st::VarInt<DB>,
-    types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB, types::st::Pair<DB,
-        typename TypeHandler<DB, T0>::type,
-        typename TypeHandler<DB, T1>::type>>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3, T4>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3, T4>>::type returnArg) {
-    size_t memorySize = container.getAllocatedMemorySize();
-    auto tail = returnArg
-      .write(memorySize)
-      .write(container.bucket_count())
-      .write(container.size());
-
-    for (auto &&entry: container) {
-      tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-        auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-          return OIInternal::getSizeType<DB>(key, ret);
-        });
-        return OIInternal::getSizeType<DB>(value, next);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)container.getAllocatedMemorySize())
diff --git a/types/f14_fast_set.toml b/types/f14_fast_set.toml
index 63262c5..8516f8c 100644
--- a/types/f14_fast_set.toml
+++ b/types/f14_fast_set.toml
@@ -35,36 +35,6 @@ void getSizeType(const %1%<Key, Hasher, KeyEqual, Alloc> &container, size_t& ret
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
-  using type = types::st::Pair<DB,
-    types::st::VarInt<DB>,
-    types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB,
-        typename TypeHandler<DB, T0>::type>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3>>::type returnArg) {
-    size_t memorySize = container.getAllocatedMemorySize();
-    auto tail = returnArg
-      .write(memorySize)
-      .write(container.bucket_count())
-      .write(container.size());
-
-    for (auto &&entry: container) {
-      tail = tail.delegate([&entry](auto ret) {
-        return OIInternal::getSizeType<DB>(entry, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)container.getAllocatedMemorySize())
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index 4d483e6..93b53e9 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -36,40 +36,6 @@ void getSizeType(const %1%<Key, Mapped, Hasher, KeyEqual, Alloc> &container, siz
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3, typename T4>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3, T4>> {
-  using type = types::st::Pair<DB,
-    types::st::VarInt<DB>,
-    types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB, types::st::Pair<DB,
-        typename TypeHandler<DB, T0>::type,
-        typename TypeHandler<DB, T1>::type>>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3, T4>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3, T4>>::type returnArg) {
-    size_t memorySize = container.getAllocatedMemorySize();
-    auto tail = returnArg
-      .write(memorySize)
-      .write(container.bucket_count())
-      .write(container.size());
-
-    for (auto &&entry: container) {
-      tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-        auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-          return OIInternal::getSizeType<DB>(key, ret);
-        });
-        return OIInternal::getSizeType<DB>(value, next);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)container.getAllocatedMemorySize())
diff --git a/types/f14_node_set.toml b/types/f14_node_set.toml
index 8413cf0..511c2f8 100644
--- a/types/f14_node_set.toml
+++ b/types/f14_node_set.toml
@@ -35,36 +35,6 @@ void getSizeType(const %1%<Key, Hasher, KeyEqual, Alloc> &container, size_t& ret
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
-  using type = types::st::Pair<DB,
-    types::st::VarInt<DB>,
-    types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB,
-        typename TypeHandler<DB, T0>::type>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3>>::type returnArg) {
-    size_t memorySize = container.getAllocatedMemorySize();
-    auto tail = returnArg
-      .write(memorySize)
-      .write(container.bucket_count())
-      .write(container.size());
-
-    for (auto &&entry: container) {
-      tail = tail.delegate([&entry](auto ret) {
-        return OIInternal::getSizeType<DB>(entry, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)container.getAllocatedMemorySize())
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index ed569de..ec4f187 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -36,40 +36,6 @@ void getSizeType(const %1%<Key, Mapped, Hasher, KeyEqual, Alloc> &container, siz
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3, typename T4>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3, T4>> {
-  using type = types::st::Pair<DB,
-    types::st::VarInt<DB>,
-    types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB, types::st::Pair<DB,
-        typename TypeHandler<DB, T0>::type,
-        typename TypeHandler<DB, T1>::type>>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3, T4>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3, T4>>::type returnArg) {
-    size_t memorySize = container.getAllocatedMemorySize();
-    auto tail = returnArg
-      .write(memorySize)
-      .write(container.bucket_count())
-      .write(container.size());
-
-    for (auto &&entry: container) {
-      tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-        auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-          return OIInternal::getSizeType<DB>(key, ret);
-        });
-        return OIInternal::getSizeType<DB>(value, next);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)container.getAllocatedMemorySize())
diff --git a/types/f14_value_set.toml b/types/f14_value_set.toml
index 07887e7..20aae30 100644
--- a/types/f14_value_set.toml
+++ b/types/f14_value_set.toml
@@ -35,36 +35,6 @@ void getSizeType(const %1%<Key, Hasher, KeyEqual, Alloc> &container, size_t& ret
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
-  using type = types::st::Pair<DB,
-    types::st::VarInt<DB>,
-    types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB,
-        typename TypeHandler<DB, T0>::type>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3>>::type returnArg) {
-    size_t memorySize = container.getAllocatedMemorySize();
-    auto tail = returnArg
-      .write(memorySize)
-      .write(container.bucket_count())
-      .write(container.size());
-
-    for (auto &&entry: container) {
-      tail = tail.delegate([&entry](auto ret) {
-        return OIInternal::getSizeType<DB>(entry, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)container.getAllocatedMemorySize())
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index 9287d1b..5e29d82 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -36,40 +36,6 @@ void getSizeType(const %1%<Key, Mapped, Hasher, KeyEqual, Alloc> &container, siz
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3, typename T4>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3, T4>> {
-  using type = types::st::Pair<DB,
-    types::st::VarInt<DB>,
-    types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB, types::st::Pair<DB,
-        typename TypeHandler<DB, T0>::type,
-        typename TypeHandler<DB, T1>::type>>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3, T4>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3, T4>>::type returnArg) {
-    size_t memorySize = container.getAllocatedMemorySize();
-    auto tail = returnArg
-      .write(memorySize)
-      .write(container.bucket_count())
-      .write(container.size());
-
-    for (auto &&entry: container) {
-      tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-        auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-          return OIInternal::getSizeType<DB>(key, ret);
-        });
-        return OIInternal::getSizeType<DB>(value, next);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)container.getAllocatedMemorySize())
diff --git a/types/f14_vector_set.toml b/types/f14_vector_set.toml
index e94b8b9..1aed210 100644
--- a/types/f14_vector_set.toml
+++ b/types/f14_vector_set.toml
@@ -35,36 +35,6 @@ void getSizeType(const %1%<Key, Hasher, KeyEqual, Alloc> &container, size_t& ret
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
-  using type = types::st::Pair<DB,
-    types::st::VarInt<DB>,
-    types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB,
-        typename TypeHandler<DB, T0>::type>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3>>::type returnArg) {
-    size_t memorySize = container.getAllocatedMemorySize();
-    auto tail = returnArg
-      .write(memorySize)
-      .write(container.bucket_count())
-      .write(container.size());
-
-    for (auto &&entry: container) {
-      tail = tail.delegate([&entry](auto ret) {
-        return OIInternal::getSizeType<DB>(entry, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)container.getAllocatedMemorySize())
diff --git a/types/fb_string_type.toml b/types/fb_string_type.toml
index 19b9f34..e0679d3 100644
--- a/types/fb_string_type.toml
+++ b/types/fb_string_type.toml
@@ -38,38 +38,6 @@ void getSizeType(const %1%<E, T, A, Storage> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::Pair<DB,
-        types::st::VarInt<DB>,
-        types::st::Pair<DB,
-          types::st::VarInt<DB>,
-          types::st::VarInt<DB>
-      >>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3>>::type returnArg) {
-    auto last = returnArg.write((uintptr_t)container.data())
-                    .write(container.capacity())
-                    .write(container.size());
-
-    bool inlined = ((uintptr_t)container.data() < (uintptr_t)(&container + sizeof(%1%<T0, T1, T2, T3>)))
-        &&
-      ((uintptr_t)container.data() >= (uintptr_t)&container);
-
-    if (!inlined && pointers.add((uintptr_t)container.data())) {
-      return last.write(1);
-    } else {
-      return last.write(0);
-    }
-  }
-};
-"""
-
 traversal_func = """
 // fbstring has inlining (SSO) and allocates large strings as
 // reference counted strings. Reference counted strings have an
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index 16b9615..90d424d 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -35,37 +35,6 @@ void getSizeType(const %1%<Key, Value, Compare, Allocator, GrowthPolicy, Contain
     }
 }
 """
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3, typename T4, typename T5>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3, T4, T5>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::Pair<DB,
-        types::st::VarInt<DB>,
-        types::st::List<DB, types::st::Pair<DB,
-          typename TypeHandler<DB, T0>::type,
-          typename TypeHandler<DB, T1>::type
-    >>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3, T4, T5>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3, T4, T5>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container)
-                    .write(container.capacity())
-                    .write(container.size());
-
-    for (const auto& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it.second, ret.delegate([&it](auto ret) {
-            return OIInternal::getSizeType<DB>(it.first, ret);
-        }));
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
 
 traversal_func = '''
     auto tail = returnArg.write((uintptr_t)&container)
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index f29eb8d..7b1e622 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -33,34 +33,6 @@ void getSizeType(const %1%<Key,T,Compare,Allocator> &container, size_t& returnAr
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
-  using type = types::st::List<DB, types::st::Pair<DB,
-      typename TypeHandler<DB, T0>::type,
-      typename TypeHandler<DB, T1>::type
-    >>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3>>::type returnArg) {
-    auto tail = returnArg.write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it.second, ret.delegate([&it](auto ret) {
-            return OIInternal::getSizeType<DB>(it.first, ret);
-        }));
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)&container)
diff --git a/types/multi_set_type.toml b/types/multi_set_type.toml
index aab9fd7..34163e8 100644
--- a/types/multi_set_type.toml
+++ b/types/multi_set_type.toml
@@ -35,31 +35,6 @@ void getSizeType(const %1%<Key, Compare, Alloc> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2>
-struct TypeHandler<DB, %1% <T0, T1, T2>> {
-  using type = types::st::List<DB, typename TypeHandler<DB, T0>::type>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2>>::type returnArg) {
-    constexpr size_t nodeSize = sizeof(typename %1%<T0, T1, T2>::node_type);
-
-    auto tail = returnArg.write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
diff --git a/types/optional_type.toml b/types/optional_type.toml
index d153537..2178743 100644
--- a/types/optional_type.toml
+++ b/types/optional_type.toml
@@ -28,28 +28,6 @@ void getSizeType(const %1%<T>& container, size_t& returnArg) {
 }
 """
 
-handler = """
-template <typename DB, typename T0>
-struct TypeHandler<DB, %1%<T0>> {
-    using type = types::st::Sum<DB,
-      types::st::Unit<DB>,
-      typename TypeHandler<DB, T0>::type
-    >;
-
-    static types::st::Unit<DB> getSizeType(
-        const %1%<T0>& container,
-        typename TypeHandler<DB, %1%<T0>>::type returnArg) {
-      if (container) {
-        return returnArg.template delegate<1>([&container](auto ret) {
-          return OIInternal::getSizeType<DB>(*container, ret);
-        });
-      } else {
-        return returnArg.template delegate<0>(std::identity());
-      }
-    }
-};
-"""
-
 traversal_func = """
 if (container.has_value()) {
   return returnArg.template delegate<1>([&container](auto ret) {
diff --git a/types/pair_type.toml b/types/pair_type.toml
index e904db8..570d289 100644
--- a/types/pair_type.toml
+++ b/types/pair_type.toml
@@ -26,26 +26,6 @@ void getSizeType(const %1%<P,Q> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1>
-struct TypeHandler<DB, %1%<T0, T1>> {
-  using type = types::st::Pair<DB,
-      typename TypeHandler<DB, T0>::type,
-      typename TypeHandler<DB, T1>::type>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1> & container,
-      typename TypeHandler<DB, %1%<T0, T1>>::type returnArg) {
-    return OIInternal::getSizeType<DB>(
-        container.second,
-        returnArg.delegate([&container](auto ret) {
-            return OIInternal::getSizeType<DB>(container.first, ret);
-        })
-    );
-  }
-};
-"""
-
 traversal_func = """
     return OIInternal::getSizeType<DB>(
         container.second,
diff --git a/types/priority_queue_container_adapter_type.toml b/types/priority_queue_container_adapter_type.toml
index cc0b33c..7bdebb6 100644
--- a/types/priority_queue_container_adapter_type.toml
+++ b/types/priority_queue_container_adapter_type.toml
@@ -32,21 +32,3 @@ void getSizeType(const %1%<T, Container, Cmp> &containerAdapter, size_t& returnA
     getSizeType(container, returnArg);
 }
 """
-
-handler = """
-template <typename DB, typename T0, typename T1, typename T2>
-struct TypeHandler<DB, %1%<T0, T1, T2>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      typename TypeHandler<DB, T1>::type>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container);
-
-    const T1 &underlyingContainer = get_container(container);
-    return OIInternal::getSizeType<DB>(underlyingContainer, tail);
-  }
-};
-"""
diff --git a/types/queue_container_adapter_type.toml b/types/queue_container_adapter_type.toml
index 10891f8..7870901 100644
--- a/types/queue_container_adapter_type.toml
+++ b/types/queue_container_adapter_type.toml
@@ -32,20 +32,3 @@ void getSizeType(const %1%<T, Container> &containerAdapter, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1>
-struct TypeHandler<DB, %1%<T0, T1>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      typename TypeHandler<DB, T1>::type>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1% <T0, T1> & container,
-      typename TypeHandler<DB, %1% <T0, T1>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container);
-
-    const T1 &underlyingContainer = get_container(container);
-    return OIInternal::getSizeType<DB>(underlyingContainer, tail);
-  }
-};
-"""
diff --git a/types/ref_wrapper_type.toml b/types/ref_wrapper_type.toml
index 9c87b2a..949b385 100644
--- a/types/ref_wrapper_type.toml
+++ b/types/ref_wrapper_type.toml
@@ -29,29 +29,3 @@ void getSizeType(const %1%<T> &ref, size_t& returnArg)
     }
 }
 """
-
-handler = """
-template <typename DB, typename T0>
-struct TypeHandler<DB, %1%<T0>> {
-    using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::Sum<DB,
-        types::st::Unit<DB>,
-        typename TypeHandler<DB, T0>::type
-    >>;
-
-    static types::st::Unit<DB> getSizeType(
-        const %1%<T0>& container,
-        typename TypeHandler<DB, %1%<T0>>::type returnArg) {
-      auto r0 = returnArg.write((uintptr_t)&(container.get()));
-
-      if (pointers.add((uintptr_t)&container.get())) {
-        return r0.template delegate<1>([&container](auto ret) {
-          return OIInternal::getSizeType<DB>(container.get(), ret);
-        });
-      } else {
-        return r0.template delegate<0>(std::identity());
-      }
-    }
-};
-"""
diff --git a/types/seq_type.toml b/types/seq_type.toml
index 1da8fd6..0ccfee8 100644
--- a/types/seq_type.toml
+++ b/types/seq_type.toml
@@ -35,35 +35,6 @@ void getSizeType(const %1%<T, Allocator> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1>
-struct TypeHandler<DB, %1%<T0, T1>> {
-  using type = types::st::Pair<
-      DB, types::st::VarInt<DB>,
-      types::st::Pair<
-          DB, types::st::VarInt<DB>,
-          types::st::List<DB, typename TypeHandler<DB, T0>::type>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1> & container,
-      typename TypeHandler<DB, %1%<T0, T1>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container)
-                    .write(container.capacity())
-                    .write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.capacity())
diff --git a/types/set_type.toml b/types/set_type.toml
index bd22ad6..43206f6 100644
--- a/types/set_type.toml
+++ b/types/set_type.toml
@@ -36,34 +36,6 @@ void getSizeType(const %1%<Key, Compare, Alloc> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2>
-struct TypeHandler<DB, %1% <T0, T1, T2>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB, typename TypeHandler<DB, T0>::type>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2>>::type returnArg) {
-    constexpr size_t nodeSize = sizeof(typename %1%<T0, T1, T2>::node_type);
-
-    auto tail = returnArg.write(nodeSize)
-                    .write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 3154fe4..693f871 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -34,38 +34,6 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0>
-struct TypeHandler<DB, %1%<T0>> {
-    using type = typename std::conditional<
-      std::is_void<T0>::value,
-      types::st::Unit<DB>,
-      types::st::Pair<DB,
-        types::st::VarInt<DB>,
-        types::st::Sum<DB,
-          types::st::Unit<DB>,
-          typename TypeHandler<DB, T0>::type
-      >>>::type;
-
-    static types::st::Unit<DB> getSizeType(
-        const %1%<T0>& container,
-        typename TypeHandler<DB, %1%<T0>>::type returnArg) {
-      if constexpr (!std::is_void<T0>::value) {
-        auto r0 = returnArg.write((uintptr_t)(container.get()));
-        if (container && pointers.add((uintptr_t)(container.get()))) {
-          return r0.template delegate<1>([&container](auto ret) {
-            return OIInternal::getSizeType<DB>(*(container.get()), ret);
-          });
-        } else {
-          return r0.template delegate<0>(std::identity());
-        }
-      } else {
-        return returnArg;
-      }
-    }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
diff --git a/types/small_vec_type.toml b/types/small_vec_type.toml
index 0bdd1dd..aca3e79 100644
--- a/types/small_vec_type.toml
+++ b/types/small_vec_type.toml
@@ -42,33 +42,6 @@ void getSizeType(const %1%<V, N, P> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, std::size_t N0, typename T1>
-struct TypeHandler<DB, %1%<T0, N0, T1>> {
-  using type = types::st::Pair<
-      DB, types::st::VarInt<DB>,
-      types::st::Pair<
-          DB, types::st::VarInt<DB>,
-          types::st::List<DB, typename TypeHandler<DB, T0>::type>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, N0, T1> & container,
-      typename TypeHandler<DB, %1%<T0, N0, T1>>::type returnArg) {
-    auto tail = returnArg.write(N0)
-                    .write(container.capacity())
-                    .write(container.size());
-
-    for (auto& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 // If `container.data()` pointer is within the container struct,
 // then the container's storage is inlined and doesn't uses the heap.
diff --git a/types/sorted_vec_set_type.toml b/types/sorted_vec_set_type.toml
index bc44219..f17a0f7 100644
--- a/types/sorted_vec_set_type.toml
+++ b/types/sorted_vec_set_type.toml
@@ -31,23 +31,6 @@ void getSizeType(const %1%<T,Compare, Allocator, GrowthPolicy, Container> &conta
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3, typename T4>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3, T4>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      typename TypeHandler<DB, T4>::type>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3, T4>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3, T4>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container);
-    const T4 &underlyingContainer = container.get_container();
-    return OIInternal::getSizeType<DB>(underlyingContainer, tail);
-  }
-};
-"""
-
 traversal_func = '''
 auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.capacity())
diff --git a/types/stack_container_adapter_type.toml b/types/stack_container_adapter_type.toml
index 67187bb..db91880 100644
--- a/types/stack_container_adapter_type.toml
+++ b/types/stack_container_adapter_type.toml
@@ -31,21 +31,3 @@ void getSizeType(const %1%<T, Container> &containerAdapter, size_t& returnArg)
     getSizeType(container, returnArg);
 }
 """
-
-handler = """
-template <typename DB, typename T0, typename T1>
-struct TypeHandler<DB, %1%<T0, T1>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      typename TypeHandler<DB, T1>::type>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1% <T0, T1> & container,
-      typename TypeHandler<DB, %1% <T0, T1>>::type returnArg) {
-    auto tail = returnArg.write((uintptr_t)&container);
-
-    const T1 &underlyingContainer = get_container(container);
-    return OIInternal::getSizeType<DB>(underlyingContainer, tail);
-  }
-};
-"""
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index ecceb36..61be6be 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -37,38 +37,6 @@ void getSizeType(const %1%<K, T, C, A> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::List<DB, types::st::Pair<DB,
-        typename TypeHandler<DB, T0>::type,
-        typename TypeHandler<DB, T1>::type
-    >>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3>>::type returnArg) {
-    constexpr size_t nodeSize = sizeof(typename %1%<T0, T1, T2, T3>::node_type);
-
-    auto tail = returnArg.write(nodeSize).write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (const auto& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it.second, ret.delegate([&it](auto ret) {
-            return OIInternal::getSizeType<DB>(it.first, ret);
-        }));
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)&container)
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index b7f99d7..541576d 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -39,42 +39,6 @@ void getSizeType(const %1%<K, T, H, KE, A> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3, typename T4>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3, T4>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::Pair<DB,
-        types::st::VarInt<DB>,
-        types::st::List<DB, types::st::Pair<DB,
-          typename TypeHandler<DB, T0>::type,
-          typename TypeHandler<DB, T1>::type
-    >>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3, T4>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3, T4>>::type returnArg) {
-    constexpr size_t nodeSize = sizeof(typename %1%<T0, T1, T2, T3, T4>::node_type);
-
-    auto tail = returnArg.write(nodeSize)
-                    .write(container.bucket_count())
-                    .write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (const auto& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it.second, ret.delegate([&it](auto ret) {
-            return OIInternal::getSizeType<DB>(it.first, ret);
-        }));
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)&container)
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index 2782118..a548623 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -39,42 +39,6 @@ void getSizeType(const %1%<K, T, H, KE, A> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3, typename T4>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3, T4>> {
-  using type = types::st::Pair<DB,
-      types::st::VarInt<DB>,
-      types::st::Pair<DB,
-        types::st::VarInt<DB>,
-        types::st::List<DB, types::st::Pair<DB,
-          typename TypeHandler<DB, T0>::type,
-          typename TypeHandler<DB, T1>::type
-    >>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3, T4>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3, T4>>::type returnArg) {
-    constexpr size_t nodeSize = sizeof(typename %1%<T0, T1, T2, T3, T4>::node_type);
-
-    auto tail = returnArg.write(nodeSize)
-                    .write(container.bucket_count())
-                    .write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (const auto& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it.second, ret.delegate([&it](auto ret) {
-            return OIInternal::getSizeType<DB>(it.first, ret);
-        }));
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)&container)
diff --git a/types/std_variant.toml b/types/std_variant.toml
index d8d5f9b..c7b9dcc 100644
--- a/types/std_variant.toml
+++ b/types/std_variant.toml
@@ -36,33 +36,5 @@ void getSizeType(const %1%<Types...> &container, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename... Types>
-struct TypeHandler<DB, %1%<Types...>> {
-    using type = types::st::Sum<DB, typename TypeHandler<DB, Types>::type..., types::st::Unit<DB>>;
-
-    static types::st::Unit<DB> getSizeType(
-        const %1%<Types...>& container,
-        typename TypeHandler<DB, %1%<Types...>>::type returnArg) {
-      return getSizeTypeRecursive(container, returnArg);
-    }
-
-  private:
-    template <size_t I = 0>
-    static types::st::Unit<DB> getSizeTypeRecursive(
-        const %1%<Types...>& container,
-        typename TypeHandler<DB, %1%<Types...>>::type returnArg) {
-      if constexpr (I < sizeof...(Types)) {
-        if (I == container.index()) {
-          return returnArg.template delegate<I>([&container](auto ret) {
-            return OIInternal::getSizeType<DB>(std::get<I>(container), ret);
-          });
-        } else {
-          return getSizeTypeRecursive<I+1>(container, returnArg);
-        }
-      } else {
-        return returnArg.template delegate<sizeof...(Types)>(std::identity());
-      }
-    }
-};
-"""
+# TODO: Add tbv2 definitions. The removed intermediate handler is a good
+# template for this, find it in the git logs.
diff --git a/types/thrift_isset_type.toml b/types/thrift_isset_type.toml
index 213cb58..ce60791 100644
--- a/types/thrift_isset_type.toml
+++ b/types/thrift_isset_type.toml
@@ -17,9 +17,6 @@ decl = """
 func = """
 // DummyFunc %1%
 """
-handler = """
-// DummyHandler %1%
-"""
 
 traversal_func = """
 return returnArg;
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index eb3be12..35155f9 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -35,38 +35,6 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1>
-struct TypeHandler<DB, %1%<T0,T1>> {
-    using type = typename std::conditional<
-      std::is_void<T0>::value,
-      types::st::Unit<DB>,
-      types::st::Pair<DB,
-        types::st::VarInt<DB>,
-        types::st::Sum<DB,
-          types::st::Unit<DB>,
-          typename TypeHandler<DB, T0>::type
-      >>>::type;
-
-    static types::st::Unit<DB> getSizeType(
-        const %1%<T0,T1>& container,
-        typename TypeHandler<DB, %1%<T0,T1>>::type returnArg) {
-      if constexpr (!std::is_void<T0>::value) {
-        auto r0 = returnArg.write((uintptr_t)(container.get()));
-        if (container && pointers.add((uintptr_t)(container.get()))) {
-          return r0.template delegate<1>([&container](auto ret) {
-            return OIInternal::getSizeType<DB>(*(container.get()), ret);
-          });
-        } else {
-          return r0.template delegate<0>(std::identity());
-        }
-      } else {
-        return returnArg;
-      }
-    }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
diff --git a/types/unordered_multiset_type.toml b/types/unordered_multiset_type.toml
index bf6e86f..9d1449a 100644
--- a/types/unordered_multiset_type.toml
+++ b/types/unordered_multiset_type.toml
@@ -37,37 +37,6 @@ void getSizeType(const %1%<Key, Hasher, KeyEqual, Alloc> &container, size_t& ret
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
-  using type = types::st::Pair<
-      DB, types::st::VarInt<DB>,
-      types::st::Pair<
-          DB, types::st::VarInt<DB>,
-          types::st::List<DB, typename TypeHandler<DB, T0>::type>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3>>::type returnArg) {
-    constexpr size_t nodeSize = sizeof(typename %1%<T0, T1, T2, T3>::node_type);
-
-    auto tail = returnArg.write(nodeSize)
-                    .write(container.bucket_count())
-                    .write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)&container)
diff --git a/types/unordered_set_type.toml b/types/unordered_set_type.toml
index 8f085c4..62e0190 100644
--- a/types/unordered_set_type.toml
+++ b/types/unordered_set_type.toml
@@ -37,37 +37,6 @@ void getSizeType(const %1%<Key, Hasher, KeyEqual, Alloc> &container, size_t& ret
 }
 """
 
-handler = """
-template <typename DB, typename T0, typename T1, typename T2, typename T3>
-struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
-  using type = types::st::Pair<
-      DB, types::st::VarInt<DB>,
-      types::st::Pair<
-          DB, types::st::VarInt<DB>,
-          types::st::List<DB, typename TypeHandler<DB, T0>::type>>>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0, T1, T2, T3>& container,
-      typename TypeHandler<DB, %1%<T0, T1, T2, T3>>::type returnArg) {
-    constexpr size_t nodeSize = sizeof(typename %1%<T0, T1, T2, T3>::node_type);
-
-    auto tail = returnArg.write(nodeSize)
-                    .write(container.bucket_count())
-                    .write(container.size());
-
-    // The double ampersand is needed otherwise this loop doesn't work with
-    // vector<bool>
-    for (auto&& it : container) {
-      tail = tail.delegate([&it](auto ret) {
-        return OIInternal::getSizeType<DB>(it, ret);
-      });
-    }
-
-    return tail.finish();
-  }
-};
-"""
-
 traversal_func = """
 auto tail = returnArg
   .write((uintptr_t)&container)
diff --git a/types/weak_ptr_type.toml b/types/weak_ptr_type.toml
index 83eaf56..33cb74e 100644
--- a/types/weak_ptr_type.toml
+++ b/types/weak_ptr_type.toml
@@ -24,18 +24,5 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 }
 """
 
-handler = """
-template <typename DB, typename T0>
-struct TypeHandler<DB, %1%<T0>> {
-  using type = types::st::Unit<DB>;
-
-  static types::st::Unit<DB> getSizeType(
-      const %1%<T0>& container,
-      typename TypeHandler<DB, %1%<T0>>::type returnArg) {
-    return returnArg;
-  }
-};
-"""
-
 traversal_func = "return returnArg;"
 

From ba79e50fb40f5c6258a24c8806ddc6aa6238805d Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 15 Nov 2023 06:14:52 -0800
Subject: [PATCH 028/188] tbv2: replace DB template param with Ctx

TreeBuilder v2 adds a DB template parameter to every function. This is used as
part of the static type to decide what type of DataBuffer is being used:
currently `BackInserterDataBuffer<std::vector<uint8_t>>` for OIL and it would
be `DataSegmentDataBuffer` for OID.

This change replaces the `DB` template parameter with a more general `Ctx`. Due
to issues with dependent naming it also adds a `using DB` to each `TypeHandler`
which has the same function as before. This allows us to add more "static
context" (typedefs and constants) to functions without changing this signature
again, because changing the signature of everything is a massive pain.

Currently this change achieves nothing because Ctx contains only DB in a static
wrapper. In the next change I'm going to pass a reference of type Ctx around to
add a "dynamic context" to invocations which will contain the pointer array. In
future we'll then be able to add either static or dynamic context without any
signature adjustments.

Test plan:
- CI
---
 oi/CodeGen.cpp                         | 59 ++++++++++++++------------
 oi/FuncGen.cpp                         | 47 +++++++++++---------
 types/array_type.toml                  |  6 +--
 types/cxx11_list_type.toml             |  8 ++--
 types/cxx11_string_type.toml           |  5 ++-
 types/f14_fast_map.toml                | 14 +++---
 types/f14_fast_set.toml                |  6 +--
 types/f14_node_map.toml                | 14 +++---
 types/f14_node_set.toml                |  6 +--
 types/f14_value_map.toml               | 14 +++---
 types/f14_value_set.toml               |  6 +--
 types/f14_vector_map.toml              | 14 +++---
 types/f14_vector_set.toml              |  6 +--
 types/list_type.toml                   |  8 ++--
 types/map_seq_type.toml                | 24 +++++------
 types/multi_map_type.toml              | 14 +++---
 types/multi_set_type.toml              |  6 +--
 types/optional_type.toml               |  8 ++--
 types/pair_type.toml                   | 10 ++---
 types/seq_type.toml                    |  6 +--
 types/set_type.toml                    |  6 +--
 types/shrd_ptr_type.toml               |  8 ++--
 types/small_vec_type.toml              |  8 ++--
 types/sorted_vec_set_type.toml         |  6 +--
 types/std_map_type.toml                | 24 +++++------
 types/std_unordered_map_type.toml      | 24 +++++------
 types/std_unordered_multimap_type.toml | 10 ++---
 types/uniq_ptr_type.toml               |  8 ++--
 types/unordered_multiset_type.toml     |  6 +--
 types/unordered_set_type.toml          |  6 +--
 30 files changed, 200 insertions(+), 187 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 73725e4..cfa87e7 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -651,7 +651,7 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
   code += funcName;
   code += "(\n      const ";
   code += c.name();
-  code += "& t,\n      typename TypeHandler<DB, ";
+  code += "& t,\n      typename TypeHandler<Ctx, ";
   code += c.name();
   code += ">::type returnArg) {\n";
 
@@ -683,7 +683,7 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
     } else {
       code += "delegate";
     }
-    code += "([&t](auto ret) { return OIInternal::getSizeType<DB>(t.";
+    code += "([&t](auto ret) { return OIInternal::getSizeType<Ctx>(t.";
     code += member.name;
     code += ", ret); })";
   }
@@ -695,7 +695,7 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
 }
 
 // Generate the static type for the class's representation in the data buffer.
-// For `class { int a,b,c; }` we generate (DB omitted for clarity):
+// For `class { int a,b,c; }` we generate (Ctx/DB omitted for clarity):
 // Pair<TypeHandler<int>::type,
 //   Pair<TypeHandler<int>::type,
 //     TypeHandler<int>::type
@@ -734,7 +734,7 @@ void CodeGen::genClassStaticType(const Class& c, std::string& code) {
     }
 
     code +=
-        (boost::format("typename TypeHandler<DB, decltype(%1%::%2%)>::type") %
+        (boost::format("typename TypeHandler<Ctx, decltype(%1%::%2%)>::type") %
          c.name() % member.name)
             .str();
 
@@ -787,8 +787,8 @@ void CodeGen::genClassTreeBuilderInstructions(const Class& c,
     code += "      inst::Field{sizeof(" + fullName + "), " +
             std::to_string(calculateExclusiveSize(m.type())) + ",\"" +
             m.inputName + "\", member_" + std::to_string(index) +
-            "_type_names, TypeHandler<DB, decltype(" + fullName +
-            ")>::fields, TypeHandler<DB, decltype(" + fullName +
+            "_type_names, TypeHandler<Ctx, decltype(" + fullName +
+            ")>::fields, TypeHandler<Ctx, decltype(" + fullName +
             ")>::processors},\n";
   }
   code += "  };\n";
@@ -820,10 +820,11 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) {
                    .str();
   }
 
-  code += "template <typename DB>\n";
-  code += "class TypeHandler<DB, ";
+  code += "template <typename Ctx>\n";
+  code += "class TypeHandler<Ctx, ";
   code += c.name();
   code += "> {\n";
+  code += "  using DB = typename Ctx::DataBuffer;\n";
   code += helpers;
   code += " public:\n";
   code += "  using type = ";
@@ -877,7 +878,7 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     containerWithTypes = "OICaptureKeys<" + containerWithTypes + ">";
   }
 
-  code += "template <typename DB";
+  code += "template <typename Ctx";
   types = 0, values = 0;
   for (const auto& p : templateParams) {
     if (p.value) {
@@ -889,9 +890,10 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     }
   }
   code += ">\n";
-  code += "struct TypeHandler<DB, ";
+  code += "struct TypeHandler<Ctx, ";
   code += containerWithTypes;
   code += "> {\n";
+  code += "  using DB = typename Ctx::DataBuffer;\n";
 
   if (c.captureKeys) {
     code += "  static constexpr bool captureKeys = true;\n";
@@ -922,7 +924,7 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   code += "      const ";
   code += containerWithTypes;
   code += "& container,\n";
-  code += "      typename TypeHandler<DB, ";
+  code += "      typename TypeHandler<Ctx, ";
   code += containerWithTypes;
   code += ">::type returnArg) {\n";
   code += func;  // has rubbish indentation
@@ -961,8 +963,9 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
 
 void addCaptureKeySupport(std::string& code) {
   code += R"(
-    template <typename DB, typename T>
+    template <typename Ctx, typename T>
     class CaptureKeyHandler {
+      using DB = typename Ctx::DataBuffer;
      public:
       using type = types::st::Sum<DB, types::st::VarInt<DB>, types::st::VarInt<DB>>;
 
@@ -975,24 +978,24 @@ void addCaptureKeySupport(std::string& code) {
       }
     };
 
-    template <bool CaptureKeys, typename DB, typename T>
+    template <bool CaptureKeys, typename Ctx, typename T>
     auto maybeCaptureKey(const T& key, auto returnArg) {
       if constexpr (CaptureKeys) {
         return returnArg.delegate([&key](auto ret) {
-          return CaptureKeyHandler<DB, T>::captureKey(key, ret);
+          return CaptureKeyHandler<Ctx, T>::captureKey(key, ret);
         });
       } else {
         return returnArg;
       }
     }
 
-    template <typename DB, typename T>
+    template <typename Ctx, typename T>
     static constexpr inst::ProcessorInst CaptureKeysProcessor{
-      CaptureKeyHandler<DB, T>::type::describe,
+      CaptureKeyHandler<Ctx, T>::type::describe,
       [](result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
         if constexpr (std::is_same_v<
-            typename CaptureKeyHandler<DB, T>::type,
-            types::st::List<DB, types::st::VarInt<DB>>>) {
+            typename CaptureKeyHandler<Ctx, T>::type,
+            types::st::List<typename Ctx::DataBuffer, types::st::VarInt<typename Ctx::DataBuffer>>>) {
           // String
           auto& str = el.data.emplace<std::string>();
           auto list = std::get<ParsedData::List>(d.val);
@@ -1013,11 +1016,11 @@ void addCaptureKeySupport(std::string& code) {
       }
     };
 
-    template <bool CaptureKeys, typename DB, typename T>
+    template <bool CaptureKeys, typename Ctx, typename T>
     static constexpr auto maybeCaptureKeysProcessor() {
       if constexpr (CaptureKeys) {
         return std::array<inst::ProcessorInst, 1>{
-          CaptureKeysProcessor<DB, T>,
+          CaptureKeysProcessor<Ctx, T>,
         };
       }
       else {
@@ -1034,28 +1037,28 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
     addCaptureKeySupport(code);
 
   // Provide a wrapper function, getSizeType, to infer T instead of having to
-  // explicitly specify it with TypeHandler<DB, T>::getSizeType every time.
+  // explicitly specify it with TypeHandler<Ctx, T>::getSizeType every time.
   code += R"(
-    template <typename DB, typename T>
-    types::st::Unit<DB>
-    getSizeType(const T &t, typename TypeHandler<DB, T>::type returnArg) {
+    template <typename Ctx, typename T>
+    types::st::Unit<typename Ctx::DataBuffer>
+    getSizeType(const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
       JLOG("obj @");
       JLOGPTR(&t);
-      return TypeHandler<DB, T>::getSizeType(t, returnArg);
+      return TypeHandler<Ctx, T>::getSizeType(t, returnArg);
     }
 )";
 
   if (features[Feature::TreeBuilderV2]) {
     code += R"(
-template <typename DB, typename T>
+template <typename Ctx, typename T>
 constexpr inst::Field make_field(std::string_view name) {
   return inst::Field{
     sizeof(T),
     ExclusiveSizeProvider<T>::size,
     name,
     NameProvider<T>::names,
-    TypeHandler<DB, T>::fields,
-    TypeHandler<DB, T>::processors,
+    TypeHandler<Ctx, T>::fields,
+    TypeHandler<Ctx, T>::processors,
   };
 }
 )";
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 7a1dd09..1a0315f 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -268,11 +268,13 @@ void __attribute__((used, retain)) introspect_%2$016x(
   v.clear();
   v.reserve(4096);
 
-  using DataBufferType = DataBuffer::BackInserter<std::vector<uint8_t>>;
-  using ContentType = OIInternal::TypeHandler<DataBufferType, OIInternal::__ROOT_TYPE__>::type;
+  struct Context {
+    using DataBuffer = DataBuffer::BackInserter<std::vector<uint8_t>>;
+  };
+  using ContentType = OIInternal::TypeHandler<Context, OIInternal::__ROOT_TYPE__>::type;
 
-  ContentType ret{DataBufferType{v}};
-  OIInternal::getSizeType<DataBufferType>(t, ret);
+  ContentType ret{Context::DataBuffer{v}};
+  OIInternal::getSizeType<Context>(t, ret);
 }
 )";
 
@@ -364,6 +366,9 @@ void FuncGen::DefineTreeBuilderInstructions(
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunknown-attributes"
 namespace {
+struct FakeContext {
+  using DataBuffer = int;
+};
 const std::array<std::string_view, )";
   code += std::to_string(typeNames.size());
   code += "> typeNames";
@@ -382,8 +387,10 @@ const std::array<std::string_view, )";
   code += ", \"a0\", typeNames";
   code += typeHash;
   code +=
-      ", OIInternal::TypeHandler<int, OIInternal::__ROOT_TYPE__>::fields, "
-      "OIInternal::TypeHandler<int, OIInternal::__ROOT_TYPE__>::processors};\n";
+      ", OIInternal::TypeHandler<FakeContext, "
+      "OIInternal::__ROOT_TYPE__>::fields, "
+      "OIInternal::TypeHandler<FakeContext, "
+      "OIInternal::__ROOT_TYPE__>::processors};\n";
   code += "} // namespace\n";
   code +=
       "extern const exporters::inst::Inst __attribute__((used, retain)) "
@@ -595,14 +602,15 @@ class BackInserter {
  */
 void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   code += R"(
-    template <typename DB, typename T>
+    template <typename Ctx, typename T>
     struct TypeHandler {
+        using DB = typename Ctx::DataBuffer;
       private:
         static auto choose_type() {
             if constexpr(std::is_pointer_v<T>) {
                 return std::type_identity<types::st::Pair<DB,
                   types::st::VarInt<DB>,
-                  types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<DB, std::remove_pointer_t<T>>::type>
+                  types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>
                 >>();
             } else {
                 return std::type_identity<types::st::Unit<DB>>();
@@ -623,8 +631,8 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
             sizeof(T),
             "*",
             names,
-            TypeHandler<DB, T>::fields,
-            TypeHandler<DB, T>::processors,
+            TypeHandler<Ctx, T>::fields,
+            TypeHandler<Ctx, T>::processors,
           };
 
           const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
@@ -649,7 +657,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
           if constexpr(std::is_pointer_v<T>) {
             return std::array<inst::ProcessorInst, 2>{
               {types::st::VarInt<DB>::describe, &process_pointer},
-              {types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<DB, std::remove_pointer_t<T>>::type>::describe, &process_pointer_content},
+              {types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>::describe, &process_pointer_content},
             };
           } else {
             return std::array<inst::ProcessorInst, 0>{};
@@ -663,7 +671,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   code += R"(
         static types::st::Unit<DB> getSizeType(
           const T& t,
-          typename TypeHandler<DB, T>::type returnArg) {
+          typename TypeHandler<Ctx, T>::type returnArg) {
               if constexpr(std::is_pointer_v<T>) {
                 JLOG("ptr val @");
                 JLOGPTR(t);
@@ -671,7 +679,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
                 if (t && pointers.add((uintptr_t)t)) {
                   return r0.template delegate<1>([&t](auto ret) {
                     if constexpr (!std::is_void<std::remove_pointer_t<T>>::value) {
-                      return TypeHandler<DB, std::remove_pointer_t<T>>::getSizeType(*t, ret);
+                      return TypeHandler<Ctx, std::remove_pointer_t<T>>::getSizeType(*t, ret);
                     } else {
                       return ret;
                     }
@@ -687,8 +695,9 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   )";
 
   code += R"(
-    template <typename DB>
-    class TypeHandler<DB, void> {
+    template <typename Ctx>
+    class TypeHandler<Ctx, void> {
+        using DB = typename Ctx::DataBuffer;
       public:
         using type = types::st::Unit<DB>;
 )";
@@ -711,21 +720,21 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
   tail = tail.delegate([&container, i](auto ret) {
-      return TypeHandler<DB, T0>::getSizeType(container.vals[i], ret);
+      return TypeHandler<Ctx, T0>::getSizeType(container.vals[i], ret);
   });
 }
 return tail.finish();
 )";
   oiArray.codegen.processors.emplace_back(ContainerInfo::Processor{
-      .type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>",
+      .type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>",
       .func = R"(
 static constexpr std::array<std::string_view, 1> names{"TODO"};
 static constexpr auto childField = inst::Field{
   sizeof(T0),
   "[]",
   names,
-  TypeHandler<DB, T0>::fields,
-  TypeHandler<DB, T0>::processors,
+  TypeHandler<Ctx, T0>::fields,
+  TypeHandler<Ctx, T0>::processors,
 };
 
 el.exclusive_size = 0;
diff --git a/types/array_type.toml b/types/array_type.toml
index 6d73322..126af6a 100644
--- a/types/array_type.toml
+++ b/types/array_type.toml
@@ -34,7 +34,7 @@ traversal_func = """
 
     for (auto & it: container) {
         tail = tail.delegate([&it](auto ret) {
-            return TypeHandler<DB, T0>::getSizeType(it, ret);
+            return TypeHandler<Ctx, T0>::getSizeType(it, ret);
         });
     }
 
@@ -42,9 +42,9 @@ traversal_func = """
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 size_t size = std::get<ParsedData::List>(d.val).length;
 el.exclusive_size = N0 == 0 ? 1 : 0;
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index fb54d22..f838eee 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -39,7 +39,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -53,7 +53,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 static constexpr size_t element_size = sizeof(std::_List_node<T0>);
@@ -62,7 +62,7 @@ static_assert(false && "No known element_size for list. See types/cxx11_list_typ
 #endif
 
 static constexpr std::array<inst::Field, 1> child_field{
-  make_field<DB, T0>("*"),
+  make_field<Ctx, T0>("*"),
 };
 static constexpr inst::Field element{
   element_size,
@@ -72,7 +72,7 @@ static constexpr inst::Field element{
   child_field,
   std::array<inst::ProcessorInst, 0>{},
 };
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats{
diff --git a/types/cxx11_string_type.toml b/types/cxx11_string_type.toml
index 94e7183..c88debd 100644
--- a/types/cxx11_string_type.toml
+++ b/types/cxx11_string_type.toml
@@ -37,8 +37,9 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
 """
 
 extra = """
-template <typename DB, typename CharT, typename Traits, typename Allocator>
-class CaptureKeyHandler<DB, std::__cxx11::basic_string<CharT, Traits, Allocator>> {
+template <typename Ctx, typename CharT, typename Traits, typename Allocator>
+class CaptureKeyHandler<Ctx, std::__cxx11::basic_string<CharT, Traits, Allocator>> {
+  using DB = typename Ctx::DataBuffer;
  public:
   // List of characters
   using type = types::st::List<DB, types::st::VarInt<DB>>;
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index c8c84f2..faf2e9d 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -44,10 +44,10 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -69,8 +69,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 static constexpr size_t element_size = sizeof(typename container_type::value_type);
@@ -84,8 +84,8 @@ el.container_stats->length = list.length;
 el.exclusive_size += allocationSize - list.length * element_size;
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr inst::Field element{
diff --git a/types/f14_fast_set.toml b/types/f14_fast_set.toml
index 8516f8c..b66284b 100644
--- a/types/f14_fast_set.toml
+++ b/types/f14_fast_set.toml
@@ -43,7 +43,7 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<DB>(entry, ret);
+    return OIInternal::getSizeType<Ctx>(entry, ret);
   });
 }
 
@@ -64,7 +64,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 
 [[codegen.processor]]
 type = """
-types::st::List<DB, typename TypeHandler<DB, T0>::type>
+types::st::List<DB, typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto allocationSize = el.pointer.value();
@@ -75,7 +75,7 @@ el.container_stats->length = list.length;
 
 el.exclusive_size += allocationSize - list.length * sizeof(T0);
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index 93b53e9..a53c3e6 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -44,10 +44,10 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -69,8 +69,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 static constexpr size_t element_size = sizeof(typename container_type::value_type);
@@ -84,8 +84,8 @@ el.container_stats->length = list.length;
 el.exclusive_size += allocationSize - list.length * element_size;
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr inst::Field element{
diff --git a/types/f14_node_set.toml b/types/f14_node_set.toml
index 511c2f8..3ed0b5d 100644
--- a/types/f14_node_set.toml
+++ b/types/f14_node_set.toml
@@ -43,7 +43,7 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<DB>(entry, ret);
+    return OIInternal::getSizeType<Ctx>(entry, ret);
   });
 }
 
@@ -64,7 +64,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 
 [[codegen.processor]]
 type = """
-types::st::List<DB, typename TypeHandler<DB, T0>::type>
+types::st::List<DB, typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto allocationSize = el.pointer.value();
@@ -75,7 +75,7 @@ el.container_stats->length = list.length;
 
 el.exclusive_size += allocationSize - list.length * sizeof(T0);
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index ec4f187..8072abf 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -44,10 +44,10 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -69,8 +69,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 static constexpr size_t element_size = sizeof(typename container_type::value_type);
@@ -84,8 +84,8 @@ el.container_stats->length = list.length;
 el.exclusive_size += allocationSize - list.length * element_size;
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr inst::Field element{
diff --git a/types/f14_value_set.toml b/types/f14_value_set.toml
index 20aae30..dd4070d 100644
--- a/types/f14_value_set.toml
+++ b/types/f14_value_set.toml
@@ -43,7 +43,7 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<DB>(entry, ret);
+    return OIInternal::getSizeType<Ctx>(entry, ret);
   });
 }
 
@@ -64,7 +64,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 
 [[codegen.processor]]
 type = """
-types::st::List<DB, typename TypeHandler<DB, T0>::type>
+types::st::List<DB, typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto allocationSize = el.pointer.value();
@@ -75,7 +75,7 @@ el.container_stats->length = list.length;
 
 el.exclusive_size += allocationSize - list.length * sizeof(T0);
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index 5e29d82..ec7ac93 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -44,10 +44,10 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -69,8 +69,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 static constexpr size_t element_size = sizeof(typename container_type::value_type);
@@ -84,8 +84,8 @@ el.container_stats->length = list.length;
 el.exclusive_size += allocationSize - list.length * element_size;
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr inst::Field element{
diff --git a/types/f14_vector_set.toml b/types/f14_vector_set.toml
index 1aed210..e98b6d8 100644
--- a/types/f14_vector_set.toml
+++ b/types/f14_vector_set.toml
@@ -43,7 +43,7 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<DB>(entry, ret);
+    return OIInternal::getSizeType<Ctx>(entry, ret);
   });
 }
 
@@ -64,7 +64,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 
 [[codegen.processor]]
 type = """
-types::st::List<DB, typename TypeHandler<DB, T0>::type>
+types::st::List<DB, typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto allocationSize = el.pointer.value();
@@ -75,7 +75,7 @@ el.container_stats->length = list.length;
 
 el.exclusive_size += allocationSize - list.length * sizeof(T0);
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/list_type.toml b/types/list_type.toml
index 9ea8ef7..86d457e 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -39,7 +39,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -53,7 +53,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 static constexpr size_t element_size = sizeof(std::_List_node<T0>);
@@ -62,7 +62,7 @@ static_assert(false && "No known element_size for list. See types/cxx11_list_typ
 #endif
 
 static constexpr std::array<inst::Field, 1> child_field{
-  make_field<DB, T0>("*"),
+  make_field<Ctx, T0>("*"),
 };
 static constexpr inst::Field element{
   element_size,
@@ -72,7 +72,7 @@ static constexpr inst::Field element{
   child_field,
   std::array<inst::ProcessorInst, 0>{},
 };
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats{
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index 90d424d..887df6a 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -43,11 +43,11 @@ traversal_func = '''
 
     for (const auto& kv : container) {
       tail = tail.delegate([&kv](auto ret) {
-        auto start = maybeCaptureKey<captureKeys, DB, T0>(kv.first, ret);
-        auto next = start.delegate([&kv](typename TypeHandler<DB, T0>::type ret) {
-          return OIInternal::getSizeType<DB>(kv.first, ret);
+        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
+        auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
+          return OIInternal::getSizeType<Ctx>(kv.first, ret);
         });
-        return OIInternal::getSizeType<DB>(kv.second, next);
+        return OIInternal::getSizeType<Ctx>(kv.second, next);
       });
     }
 
@@ -68,22 +68,22 @@ el.container_stats.emplace(result::Element::ContainerStats{ .capacity = std::get
 type = '''
 std::conditional_t<captureKeys,
   types::st::List<DB, types::st::Pair<DB,
-    typename CaptureKeyHandler<DB, T0>::type,
+    typename CaptureKeyHandler<Ctx, T0>::type,
     types::st::Pair<DB,
-      typename TypeHandler<DB, T0>::type,
-      typename TypeHandler<DB, T1>::type>>>,
+      typename TypeHandler<Ctx, T0>::type,
+      typename TypeHandler<Ctx, T1>::type>>>,
   types::st::List<DB, types::st::Pair<DB,
-    typename TypeHandler<DB, T0>::type,
-    typename TypeHandler<DB, T1>::type>>>
+    typename TypeHandler<Ctx, T0>::type,
+    typename TypeHandler<Ctx, T1>::type>>>
 '''
 func = '''
 using element_type = std::pair<T0, T1>;
 
 static constexpr std::array<inst::Field, 2> entryFields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
-static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, DB, T0>();
+static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, Ctx, T0>();
 static constexpr auto entry = inst::Field {
   sizeof(element_type),
   sizeof(element_type) - sizeof(T0) - sizeof(T1),
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index 7b1e622..e05a1e3 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -40,10 +40,10 @@ auto tail = returnArg
 
 for (const auto &entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -57,8 +57,8 @@ func = "el.pointer = std::get<ParsedData::VarInt>(d.val).value;"
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 #ifdef __GLIBCXX__
@@ -108,8 +108,8 @@ static_assert(false && "No known element_size for sets. See types/multi_map_type
 #endif
 
 static constexpr std::array<inst::Field, 2> elementFields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 static constexpr auto element = inst::Field {
   element_size,
diff --git a/types/multi_set_type.toml b/types/multi_set_type.toml
index 34163e8..00f8239 100644
--- a/types/multi_set_type.toml
+++ b/types/multi_set_type.toml
@@ -43,7 +43,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 // vector<bool>
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -57,7 +57,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 /* We don't have access to the _Rb_tree_node struct, so we manually re-create it
@@ -103,7 +103,7 @@ static constexpr size_t element_size = sizeof(OI__tree_node);
 static_assert(false && "No known element_size for multisets. See types/multi_set_type.toml");
 #endif
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats {
diff --git a/types/optional_type.toml b/types/optional_type.toml
index 2178743..3bd9cff 100644
--- a/types/optional_type.toml
+++ b/types/optional_type.toml
@@ -31,7 +31,7 @@ void getSizeType(const %1%<T>& container, size_t& returnArg) {
 traversal_func = """
 if (container.has_value()) {
   return returnArg.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<DB>(*container, ret);
+    return OIInternal::getSizeType<Ctx>(*container, ret);
   });
 } else {
   return returnArg.template delegate<0>(std::identity());
@@ -39,15 +39,15 @@ if (container.has_value()) {
 """
 
 [[codegen.processor]]
-type = "types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<DB, T0>::type>"
+type = "types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, T0>::type>"
 func = """
 static constexpr std::array<std::string_view, 1> names{"TODO"};
 static constexpr auto elementField = inst::Field{
   sizeof(T0),
   "el",
   names,
-  TypeHandler<DB, T0>::fields,
-  TypeHandler<DB, T0>::processors,
+  TypeHandler<Ctx, T0>::fields,
+  TypeHandler<Ctx, T0>::processors,
 };
 
 auto sum = std::get<ParsedData::Sum>(d.val);
diff --git a/types/pair_type.toml b/types/pair_type.toml
index 570d289..ba5bc38 100644
--- a/types/pair_type.toml
+++ b/types/pair_type.toml
@@ -27,19 +27,19 @@ void getSizeType(const %1%<P,Q> &container, size_t& returnArg)
 """
 
 traversal_func = """
-    return OIInternal::getSizeType<DB>(
+    return OIInternal::getSizeType<Ctx>(
         container.second,
         returnArg.delegate([&container](auto ret) {
-            return OIInternal::getSizeType<DB>(container.first, ret);
+            return OIInternal::getSizeType<Ctx>(container.first, ret);
         })
     );
 """
 
 [[codegen.processor]]
-type = "types::st::Pair<DB, typename TypeHandler<DB, T0>::type, typename TypeHandler<DB, T1>::type>"
+type = "types::st::Pair<DB, typename TypeHandler<Ctx, T0>::type, typename TypeHandler<Ctx, T1>::type>"
 func = """
-static constexpr auto firstField = make_field<DB, T0>("first");
-static constexpr auto secondField = make_field<DB, T1>("second");
+static constexpr auto firstField = make_field<Ctx, T0>("first");
+static constexpr auto secondField = make_field<Ctx, T1>("second");
 
 el.exclusive_size = sizeof(std::pair<T0, T1>) - sizeof(T0) - sizeof(T1);
 stack_ins(secondField);
diff --git a/types/seq_type.toml b/types/seq_type.toml
index 0ccfee8..88ddd1d 100644
--- a/types/seq_type.toml
+++ b/types/seq_type.toml
@@ -44,7 +44,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 // vector<bool>
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -64,9 +64,9 @@ el.container_stats.emplace(result::Element::ContainerStats{ .capacity = std::get
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats->length = list.length;
diff --git a/types/set_type.toml b/types/set_type.toml
index 43206f6..e29611e 100644
--- a/types/set_type.toml
+++ b/types/set_type.toml
@@ -44,7 +44,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 // vector<bool>
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -58,7 +58,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 /* We don't have access to the _Rb_tree_node struct, so we manually re-create it
@@ -104,7 +104,7 @@ static constexpr size_t element_size = sizeof(OI__tree_node);
 static_assert(false && "No known element_size for sets. See types/set_type.toml");
 #endif
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats {
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 693f871..125fd71 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -45,7 +45,7 @@ if constexpr (std::is_void<T0>::value) {
     return tail.template delegate<0>(std::identity());
 
   return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<DB>(*container, ret);
+    return OIInternal::getSizeType<Ctx>(*container, ret);
   });
 }
 """
@@ -60,7 +60,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<DB, T0>::type>
+  typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 #ifdef __GLIBCXX__
@@ -84,10 +84,10 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = sum.index, // 0 for empty containers/void, 1 otherwise
 });
 
-// Must be in a `if constexpr` or the compiler will complain about make_field<DB, void>
+// Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
 if constexpr (!std::is_void<T0>::value) {
   if (sum.index == 1) {
-    static constexpr auto element = make_field<DB, T0>("ptr_val");
+    static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);
   }
 }
diff --git a/types/small_vec_type.toml b/types/small_vec_type.toml
index aca3e79..d129285 100644
--- a/types/small_vec_type.toml
+++ b/types/small_vec_type.toml
@@ -56,8 +56,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&it: container) {
-  tail = tail.delegate([&it](typename TypeHandler<DB, T0>::type ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+  tail = tail.delegate([&it](typename TypeHandler<Ctx, T0>::type ret) {
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -81,7 +81,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 // Reading the `uses_intern_storage` boolean that was stored in `pointer` by the processor above.
 bool uses_intern_storage = std::exchange(el.pointer.value(), (uintptr_t)0);
@@ -97,7 +97,7 @@ if (uses_intern_storage) {
   el.exclusive_size += (el.container_stats->capacity - el.container_stats->length) * sizeof(T0);
 }
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/sorted_vec_set_type.toml b/types/sorted_vec_set_type.toml
index f17a0f7..e8b9483 100644
--- a/types/sorted_vec_set_type.toml
+++ b/types/sorted_vec_set_type.toml
@@ -38,7 +38,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 
 for (const auto& el : container) {
   tail = tail.delegate([&el](auto ret) {
-    return OIInternal::getSizeType<DB>(el, ret);
+    return OIInternal::getSizeType<Ctx>(el, ret);
   });
 }
 
@@ -56,9 +56,9 @@ el.container_stats.emplace(result::Element::ContainerStats{ .capacity = std::get
 '''
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats->length = list.length;
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index 61be6be..df06cc7 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -44,11 +44,11 @@ auto tail = returnArg
 
 for (const auto &entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, DB, T0>(key, ret);
-    auto next =  start.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(key, ret);
+    auto next =  start.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -64,14 +64,14 @@ type = """
 std::conditional_t<captureKeys,
   types::st::List<DB,
     types::st::Pair<DB,
-      typename CaptureKeyHandler<DB, T0>::type,
+      typename CaptureKeyHandler<Ctx, T0>::type,
       types::st::Pair<DB,
-        typename TypeHandler<DB, T0>::type,
-        typename TypeHandler<DB, T1>::type>>>,
+        typename TypeHandler<Ctx, T0>::type,
+        typename TypeHandler<Ctx, T1>::type>>>,
   types::st::List<DB,
     types::st::Pair<DB,
-      typename TypeHandler<DB, T0>::type,
-      typename TypeHandler<DB, T1>::type>>
+      typename TypeHandler<Ctx, T0>::type,
+      typename TypeHandler<Ctx, T1>::type>>
 >
 """
 func = """
@@ -122,11 +122,11 @@ static_assert(false && "No known element_size for sets. See types/std_map_type.t
 #endif
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
-static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, DB, T0>();
+static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, Ctx, T0>();
 
 static constexpr inst::Field element{
   element_size,
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index 541576d..27142a1 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -47,11 +47,11 @@ auto tail = returnArg
 
 for (const auto& kv : container) {
   tail = tail.delegate([&kv](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, DB, T0>(kv.first, ret);
-    auto next = start.delegate([&kv](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(kv.first, ret);
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
+    auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(kv.first, ret);
     });
-    return OIInternal::getSizeType<DB>(kv.second, next);
+    return OIInternal::getSizeType<Ctx>(kv.second, next);
   });
 }
 
@@ -79,13 +79,13 @@ type = """
 std::conditional_t<captureKeys,
  types::st::List<DB,
    types::st::Pair<DB,
-     typename CaptureKeyHandler<DB, T0>::type,
+     typename CaptureKeyHandler<Ctx, T0>::type,
      types::st::Pair<DB,
-       typename TypeHandler<DB, T0>::type,
-       typename TypeHandler<DB, T1>::type>>>,
+       typename TypeHandler<Ctx, T0>::type,
+       typename TypeHandler<Ctx, T1>::type>>>,
  types::st::List<DB, types::st::Pair<DB,
-   typename TypeHandler<DB, T0>::type,
-   typename TypeHandler<DB, T1>::type>>
+   typename TypeHandler<Ctx, T0>::type,
+   typename TypeHandler<Ctx, T1>::type>>
 >
 """
 func = """
@@ -122,11 +122,11 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
-static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, DB, T0>();
+static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, Ctx, T0>();
 
 static constexpr auto element = inst::Field{
   element_size,
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index a548623..261a0be 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -47,7 +47,7 @@ auto tail = returnArg
 
 for (const auto &it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -73,8 +73,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 #ifdef __GLIBCXX__
@@ -110,8 +110,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr auto element = inst::Field{
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index 35155f9..58364cf 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -46,7 +46,7 @@ if constexpr (std::is_void<T0>::value) {
     return tail.template delegate<0>(std::identity());
 
   return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<DB>(*container, ret);
+    return OIInternal::getSizeType<Ctx>(*container, ret);
   });
 }
 """
@@ -61,7 +61,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<DB, T0>::type>
+  typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto sum = std::get<ParsedData::Sum>(d.val);
@@ -70,10 +70,10 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = sum.index, // 0 for empty containers/void, 1 otherwise
 });
 
-// Must be in a `if constexpr` or the compiler will complain about make_field<DB, void>
+// Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
 if constexpr (!std::is_void<T0>::value) {
   if (sum.index == 1) {
-    static constexpr auto element = make_field<DB, T0>("ptr_val");
+    static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);
   }
 }
diff --git a/types/unordered_multiset_type.toml b/types/unordered_multiset_type.toml
index 9d1449a..5c2e96b 100644
--- a/types/unordered_multiset_type.toml
+++ b/types/unordered_multiset_type.toml
@@ -45,7 +45,7 @@ auto tail = returnArg
 
 for (const auto &it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -69,7 +69,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 /* Use libstdc++ implementation __details to compute the size of Nodes and Buckets.
@@ -100,7 +100,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = list.length,
 });
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/unordered_set_type.toml b/types/unordered_set_type.toml
index 62e0190..f8d26c9 100644
--- a/types/unordered_set_type.toml
+++ b/types/unordered_set_type.toml
@@ -45,7 +45,7 @@ auto tail = returnArg
 
 for (const auto &it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -69,7 +69,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 /* Use libstdc++ implementation __details to compute the size of Nodes and Buckets.
@@ -100,7 +100,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = list.length,
 });
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """

From 143805dfe2d234d5443e4771ab4555c76ab74492 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 15 Nov 2023 06:14:52 -0800
Subject: [PATCH 029/188] tbv2: add dynamic context passed through all
 functions

Previously for we had some shared state between all requests, noticeably the
pointers set. This change adds a by reference value to all requests which can
hold additional mutable state. The pointers set is moved into this mutable
state for OIL, which means each concurrent request will have its own pointer
set. Doing things this way allows more features to be added in the future
without such a big code modification.

Closes #404

Test plan:
- CI
---
 oi/CodeGen.cpp                         | 25 ++++++++++++++++++-------
 oi/FuncGen.cpp                         | 26 ++++++++++++++++----------
 oi/OICodeGen.cpp                       |  8 +++++++-
 oi/OITraceCode.cpp                     |  7 ++++---
 types/array_type.toml                  |  4 ++--
 types/cxx11_list_type.toml             |  4 ++--
 types/f14_fast_map.toml                |  8 ++++----
 types/f14_fast_set.toml                |  4 ++--
 types/f14_node_map.toml                |  8 ++++----
 types/f14_node_set.toml                |  4 ++--
 types/f14_value_map.toml               |  8 ++++----
 types/f14_value_set.toml               |  4 ++--
 types/f14_vector_map.toml              |  8 ++++----
 types/f14_vector_set.toml              |  4 ++--
 types/fb_string_type.toml              |  4 ++--
 types/folly_iobuf_queue_type.toml      |  2 +-
 types/list_type.toml                   |  4 ++--
 types/map_seq_type.toml                | 10 +++++-----
 types/multi_map_type.toml              |  8 ++++----
 types/multi_set_type.toml              |  4 ++--
 types/optional_type.toml               |  4 ++--
 types/pair_type.toml                   |  6 +++---
 types/ref_wrapper_type.toml            |  2 +-
 types/seq_type.toml                    |  4 ++--
 types/set_type.toml                    |  4 ++--
 types/shrd_ptr_type.toml               |  8 ++++----
 types/small_vec_type.toml              |  4 ++--
 types/sorted_vec_set_type.toml         |  4 ++--
 types/std_map_type.toml                | 10 +++++-----
 types/std_unordered_map_type.toml      | 10 +++++-----
 types/std_unordered_multimap_type.toml |  4 ++--
 types/uniq_ptr_type.toml               |  8 ++++----
 types/unordered_multiset_type.toml     |  4 ++--
 types/unordered_set_type.toml          |  4 ++--
 34 files changed, 127 insertions(+), 103 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index cfa87e7..78b370e 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -128,8 +128,9 @@ void addIncludes(const TypeGraph& typeGraph,
     includes.emplace("oi/types/st.h");
   }
   if (features[Feature::Library]) {
-    includes.emplace("vector");
+    includes.emplace("memory");
     includes.emplace("oi/IntrospectionResult.h");
+    includes.emplace("vector");
   }
   if (features[Feature::JitTiming]) {
     includes.emplace("chrono");
@@ -428,7 +429,7 @@ void addStandardGetSizeFuncDefs(std::string& code) {
       JLOG("ptr val @");
       JLOGPTR(s_ptr);
       StoreData((uintptr_t)(s_ptr), returnArg);
-      if (s_ptr && pointers.add((uintptr_t)s_ptr)) {
+      if (s_ptr && ctx.pointers.add((uintptr_t)s_ptr)) {
         StoreData(1, returnArg);
         getSizeType(*(s_ptr), returnArg);
       } else {
@@ -649,7 +650,8 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
 
   code += "  static types::st::Unit<DB> ";
   code += funcName;
-  code += "(\n      const ";
+  code += "(\n      Ctx& ctx,\n";
+  code += "    const ";
   code += c.name();
   code += "& t,\n      typename TypeHandler<Ctx, ";
   code += c.name();
@@ -683,7 +685,8 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
     } else {
       code += "delegate";
     }
-    code += "([&t](auto ret) { return OIInternal::getSizeType<Ctx>(t.";
+    code +=
+        "([&ctx, &t](auto ret) { return OIInternal::getSizeType<Ctx>(ctx, t.";
     code += member.name;
     code += ", ret); })";
   }
@@ -921,6 +924,7 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   code += ";\n";
 
   code += "  static types::st::Unit<DB> getSizeType(\n";
+  code += "      Ctx& ctx,\n";
   code += "      const ";
   code += containerWithTypes;
   code += "& container,\n";
@@ -979,7 +983,7 @@ void addCaptureKeySupport(std::string& code) {
     };
 
     template <bool CaptureKeys, typename Ctx, typename T>
-    auto maybeCaptureKey(const T& key, auto returnArg) {
+    auto maybeCaptureKey(Ctx& ctx, const T& key, auto returnArg) {
       if constexpr (CaptureKeys) {
         return returnArg.delegate([&key](auto ret) {
           return CaptureKeyHandler<Ctx, T>::captureKey(key, ret);
@@ -1041,10 +1045,10 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
   code += R"(
     template <typename Ctx, typename T>
     types::st::Unit<typename Ctx::DataBuffer>
-    getSizeType(const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
+    getSizeType(Ctx& ctx, const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
       JLOG("obj @");
       JLOGPTR(&t);
-      return TypeHandler<Ctx, T>::getSizeType(t, returnArg);
+      return TypeHandler<Ctx, T>::getSizeType(ctx, t, returnArg);
     }
 )";
 
@@ -1233,6 +1237,13 @@ void CodeGen::generate(
   if (config_.features[Feature::CaptureThriftIsset]) {
     genDefsThrift(typeGraph, code);
   }
+  if (!config_.features[Feature::TreeBuilderV2]) {
+    code += "namespace {\n";
+    code += "static struct Context {\n";
+    code += "  PointerHashSet<(1 << 20) / sizeof(uintptr_t)> pointers;\n";
+    code += "} ctx;\n";
+    code += "} // namespace\n";
+  }
 
   /*
    * The purpose of the anonymous namespace within `OIInternal` is that
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 1a0315f..8c5dc37 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -262,19 +262,24 @@ void __attribute__((used, retain)) introspect_%2$016x(
     std::vector<uint8_t>& v)
 #pragma GCC diagnostic pop
 {
-  pointers.initialize();
-  pointers.add((uintptr_t)&t);
-
   v.clear();
   v.reserve(4096);
 
+  auto pointers = std::make_unique<PointerHashSet<(1 << 20) / sizeof(uintptr_t)>>();
+  pointers->initialize();
+
   struct Context {
     using DataBuffer = DataBuffer::BackInserter<std::vector<uint8_t>>;
+
+    PointerHashSet<(1 << 20) / sizeof(uintptr_t)>& pointers;
   };
+  Context ctx{ .pointers = *pointers };
+  ctx.pointers.add((uintptr_t)&t);
+
   using ContentType = OIInternal::TypeHandler<Context, OIInternal::__ROOT_TYPE__>::type;
 
   ContentType ret{Context::DataBuffer{v}};
-  OIInternal::getSizeType<Context>(t, ret);
+  OIInternal::getSizeType<Context>(ctx, t, ret);
 }
 )";
 
@@ -319,8 +324,8 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
     func += "      const auto startTime = std::chrono::steady_clock::now();\n";
   }
   func += R"(
-      pointers.initialize();
-      pointers.add((uintptr_t)&t);
+      ctx.pointers.initialize();
+      ctx.pointers.add((uintptr_t)&t);
       auto data = reinterpret_cast<uintptr_t*>(dataBase);
 
       size_t dataSegOffset = 0;
@@ -417,7 +422,7 @@ void FuncGen::DefineTopLevelGetSizeSmartPtr(std::string& testCode,
     func += "      const auto startTime = std::chrono::steady_clock::now();\n";
   }
   func += R"(
-      pointers.initialize();
+      ctx.pointers.initialize();
       auto data = reinterpret_cast<uintptr_t*>(dataBase);
 
       size_t dataSegOffset = 0;
@@ -670,13 +675,14 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   }
   code += R"(
         static types::st::Unit<DB> getSizeType(
+          Ctx& ctx,
           const T& t,
           typename TypeHandler<Ctx, T>::type returnArg) {
               if constexpr(std::is_pointer_v<T>) {
                 JLOG("ptr val @");
                 JLOGPTR(t);
                 auto r0 = returnArg.write((uintptr_t)t);
-                if (t && pointers.add((uintptr_t)t)) {
+                if (t && ctx.pointers.add((uintptr_t)t)) {
                   return r0.template delegate<1>([&t](auto ret) {
                     if constexpr (!std::is_void<std::remove_pointer_t<T>>::value) {
                       return TypeHandler<Ctx, std::remove_pointer_t<T>>::getSizeType(*t, ret);
@@ -719,8 +725,8 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
   oiArray.codegen.traversalFunc = R"(
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
-  tail = tail.delegate([&container, i](auto ret) {
-      return TypeHandler<Ctx, T0>::getSizeType(container.vals[i], ret);
+  tail = tail.delegate([&ctx, &container, i](auto ret) {
+      return TypeHandler<Ctx, T0>::getSizeType(ctx, container.vals[i], ret);
   });
 }
 return tail.finish();
diff --git a/oi/OICodeGen.cpp b/oi/OICodeGen.cpp
index ee18429..b9cfcbb 100644
--- a/oi/OICodeGen.cpp
+++ b/oi/OICodeGen.cpp
@@ -3049,6 +3049,12 @@ bool OICodeGen::generateJitCode(std::string& code) {
     #define SAVE_DATA(val)    StoreData(val, returnArg)
   )");
 
+  code.append("namespace {\n");
+  code.append("static struct Context {\n");
+  code.append("  PointerHashSet<(1 << 20) / sizeof(uintptr_t)> pointers;\n");
+  code.append("} ctx;\n");
+  code.append("} // namespace\n");
+
   FuncGen::DefineJitLog(code, config.features);
 
   // The purpose of the anonymous namespace within `OIInternal` is that
@@ -3267,7 +3273,7 @@ bool OICodeGen::generateJitCode(std::string& code) {
       JLOG("ptr val @");
       JLOGPTR(s_ptr);
       StoreData((uintptr_t)(s_ptr), returnArg);
-      if (s_ptr && pointers.add((uintptr_t)s_ptr)) {
+      if (s_ptr && ctx.pointers.add((uintptr_t)s_ptr)) {
         StoreData(1, returnArg);
         getSizeType(*(s_ptr), returnArg);
       } else {
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index fa0beda..495c781 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -40,10 +40,11 @@ constexpr int oidMagicId = 0x01DE8;
 
 namespace {
 
-class {
+template <size_t Size>
+class PointerHashSet {
  private:
   // 1 MiB of pointers
-  std::array<uintptr_t, (1 << 20) / sizeof(uintptr_t)> data;
+  std::array<uintptr_t, Size> data;
 
   // twang_mix64 hash function, taken from Folly where it is used
   // as the default hash function for 64-bit integers
@@ -84,7 +85,7 @@ class {
   bool add(const auto* p) {
     return add((uintptr_t)p);
   }
-} static pointers;
+};
 
 }  // namespace
 
diff --git a/types/array_type.toml b/types/array_type.toml
index 126af6a..054d0c6 100644
--- a/types/array_type.toml
+++ b/types/array_type.toml
@@ -33,8 +33,8 @@ traversal_func = """
     auto tail = returnArg.write(container.size());
 
     for (auto & it: container) {
-        tail = tail.delegate([&it](auto ret) {
-            return TypeHandler<Ctx, T0>::getSizeType(it, ret);
+        tail = tail.delegate([&ctx, &it](auto ret) {
+            return TypeHandler<Ctx, T0>::getSizeType(ctx, it, ret);
         });
     }
 
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index f838eee..6788750 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -38,8 +38,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index faf2e9d..06c6f72 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_fast_set.toml b/types/f14_fast_set.toml
index b66284b..fb0017c 100644
--- a/types/f14_fast_set.toml
+++ b/types/f14_fast_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index a53c3e6..de1dcda 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_node_set.toml b/types/f14_node_set.toml
index 3ed0b5d..224789e 100644
--- a/types/f14_node_set.toml
+++ b/types/f14_node_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index 8072abf..c09bbf0 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_value_set.toml b/types/f14_value_set.toml
index dd4070d..f37f7d0 100644
--- a/types/f14_value_set.toml
+++ b/types/f14_value_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index ec7ac93..a673bbd 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_vector_set.toml b/types/f14_vector_set.toml
index e98b6d8..d85f36d 100644
--- a/types/f14_vector_set.toml
+++ b/types/f14_vector_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/fb_string_type.toml b/types/fb_string_type.toml
index e0679d3..169be87 100644
--- a/types/fb_string_type.toml
+++ b/types/fb_string_type.toml
@@ -29,7 +29,7 @@ void getSizeType(const %1%<E, T, A, Storage> &container, size_t& returnArg)
         &&
       ((uintptr_t)container.data() >= (uintptr_t)&container);
 
-    if (!inlined && pointers.add((uintptr_t)container.data())) {
+    if (!inlined && ctx.pointers.add((uintptr_t)container.data())) {
       SAVE_SIZE(container.capacity() * sizeof(T));
       SAVE_DATA(1);
     } else {
@@ -60,7 +60,7 @@ if (isStorageInline(container)) {
   category = Category::InlinedStorage;
 } else if (capacity < minLargeSize) {
   category = Category::OwnedHeapStorage;
-} else if (pointers.add(container.data())) {
+} else if (ctx.pointers.add(container.data())) {
   category = Category::ReferenceCountedStorage;
 } else {
   category = Category::AlreadyAttributed;
diff --git a/types/folly_iobuf_queue_type.toml b/types/folly_iobuf_queue_type.toml
index 0af134d..b9ef72c 100644
--- a/types/folly_iobuf_queue_type.toml
+++ b/types/folly_iobuf_queue_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1% &container, size_t& returnArg)
 
     const folly::IOBuf *head = container.front();
     SAVE_DATA((uintptr_t)head);
-    if (head && pointers.add((uintptr_t)head)) {
+    if (head && ctx.pointers.add((uintptr_t)head)) {
         SAVE_DATA(1);
         getSizeType(*head, returnArg);
     } else {
diff --git a/types/list_type.toml b/types/list_type.toml
index 86d457e..3e0da3c 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -38,8 +38,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index 887df6a..a09efff 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -42,12 +42,12 @@ traversal_func = '''
                     .write(container.size());
 
     for (const auto& kv : container) {
-      tail = tail.delegate([&kv](auto ret) {
-        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
-        auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
-          return OIInternal::getSizeType<Ctx>(kv.first, ret);
+      tail = tail.delegate([&ctx, &kv](auto ret) {
+        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, kv.first, ret);
+        auto next = start.delegate([&ctx, &kv](typename TypeHandler<Ctx, T0>::type ret) {
+          return OIInternal::getSizeType<Ctx>(ctx, kv.first, ret);
         });
-        return OIInternal::getSizeType<Ctx>(kv.second, next);
+        return OIInternal::getSizeType<Ctx>(ctx, kv.second, next);
       });
     }
 
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index e05a1e3..a5d628f 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -39,11 +39,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/multi_set_type.toml b/types/multi_set_type.toml
index 00f8239..f238356 100644
--- a/types/multi_set_type.toml
+++ b/types/multi_set_type.toml
@@ -42,8 +42,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/optional_type.toml b/types/optional_type.toml
index 3bd9cff..eeba64e 100644
--- a/types/optional_type.toml
+++ b/types/optional_type.toml
@@ -30,8 +30,8 @@ void getSizeType(const %1%<T>& container, size_t& returnArg) {
 
 traversal_func = """
 if (container.has_value()) {
-  return returnArg.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return returnArg.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 } else {
   return returnArg.template delegate<0>(std::identity());
diff --git a/types/pair_type.toml b/types/pair_type.toml
index ba5bc38..f157144 100644
--- a/types/pair_type.toml
+++ b/types/pair_type.toml
@@ -27,10 +27,10 @@ void getSizeType(const %1%<P,Q> &container, size_t& returnArg)
 """
 
 traversal_func = """
-    return OIInternal::getSizeType<Ctx>(
+    return OIInternal::getSizeType<Ctx>(ctx, 
         container.second,
-        returnArg.delegate([&container](auto ret) {
-            return OIInternal::getSizeType<Ctx>(container.first, ret);
+        returnArg.delegate([&ctx, &container](auto ret) {
+            return OIInternal::getSizeType<Ctx>(ctx, container.first, ret);
         })
     );
 """
diff --git a/types/ref_wrapper_type.toml b/types/ref_wrapper_type.toml
index 949b385..9ed5094 100644
--- a/types/ref_wrapper_type.toml
+++ b/types/ref_wrapper_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &ref, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
     SAVE_DATA((uintptr_t)&(ref.get()));
-    if (pointers.add((uintptr_t)&ref.get())) {
+    if (ctx.pointers.add((uintptr_t)&ref.get())) {
         SAVE_DATA(1);
         getSizeType(ref.get(), returnArg);
     } else {
diff --git a/types/seq_type.toml b/types/seq_type.toml
index 88ddd1d..f561d0b 100644
--- a/types/seq_type.toml
+++ b/types/seq_type.toml
@@ -43,8 +43,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/set_type.toml b/types/set_type.toml
index e29611e..9d3bc3c 100644
--- a/types/set_type.toml
+++ b/types/set_type.toml
@@ -43,8 +43,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 125fd71..bb3a622 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -24,7 +24,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
     if constexpr (!std::is_void<T>::value) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
-        if (s_ptr && pointers.add((uintptr_t)(s_ptr.get()))) {
+        if (s_ptr && ctx.pointers.add((uintptr_t)(s_ptr.get()))) {
             SAVE_DATA(1);
             getSizeType(*(s_ptr.get()), returnArg);
         } else {
@@ -40,12 +40,12 @@ auto tail = returnArg.write((uintptr_t)container.get());
 if constexpr (std::is_void<T0>::value) {
   return tail.template delegate<0>(std::identity());
 } else {
-  bool do_visit = container && pointers.add((uintptr_t)container.get());
+  bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
   if (!do_visit)
     return tail.template delegate<0>(std::identity());
 
-  return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return tail.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 }
 """
diff --git a/types/small_vec_type.toml b/types/small_vec_type.toml
index d129285..a5f1694 100644
--- a/types/small_vec_type.toml
+++ b/types/small_vec_type.toml
@@ -56,8 +56,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&it: container) {
-  tail = tail.delegate([&it](typename TypeHandler<Ctx, T0>::type ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](typename TypeHandler<Ctx, T0>::type ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/sorted_vec_set_type.toml b/types/sorted_vec_set_type.toml
index e8b9483..a803d61 100644
--- a/types/sorted_vec_set_type.toml
+++ b/types/sorted_vec_set_type.toml
@@ -37,8 +37,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (const auto& el : container) {
-  tail = tail.delegate([&el](auto ret) {
-    return OIInternal::getSizeType<Ctx>(el, ret);
+  tail = tail.delegate([&ctx, &el](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, el, ret);
   });
 }
 
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index df06cc7..ee652ec 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -43,12 +43,12 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(key, ret);
-    auto next =  start.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, key, ret);
+    auto next =  start.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index 27142a1..5b2402f 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -46,12 +46,12 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto& kv : container) {
-  tail = tail.delegate([&kv](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
-    auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(kv.first, ret);
+  tail = tail.delegate([&ctx, &kv](auto ret) {
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, kv.first, ret);
+    auto next = start.delegate([&ctx, &kv](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, kv.first, ret);
     });
-    return OIInternal::getSizeType<Ctx>(kv.second, next);
+    return OIInternal::getSizeType<Ctx>(ctx, kv.second, next);
   });
 }
 
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index 261a0be..a6f7671 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -46,8 +46,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index 58364cf..f723f80 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -25,7 +25,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
     if constexpr (!std::is_void<T>::value) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
-        if (u_ptr && pointers.add((uintptr_t)(u_ptr.get()))) {
+        if (u_ptr && ctx.pointers.add((uintptr_t)(u_ptr.get()))) {
             SAVE_DATA(1);
             getSizeType(*(u_ptr.get()), returnArg);
         } else {
@@ -41,12 +41,12 @@ auto tail = returnArg.write((uintptr_t)container.get());
 if constexpr (std::is_void<T0>::value) {
   return tail.template delegate<0>(std::identity());
 } else {
-  bool do_visit = container && pointers.add((uintptr_t)container.get());
+  bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
   if (!do_visit)
     return tail.template delegate<0>(std::identity());
 
-  return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return tail.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 }
 """
diff --git a/types/unordered_multiset_type.toml b/types/unordered_multiset_type.toml
index 5c2e96b..bfcf203 100644
--- a/types/unordered_multiset_type.toml
+++ b/types/unordered_multiset_type.toml
@@ -44,8 +44,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/unordered_set_type.toml b/types/unordered_set_type.toml
index f8d26c9..d9a827a 100644
--- a/types/unordered_set_type.toml
+++ b/types/unordered_set_type.toml
@@ -44,8 +44,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 

From f8ef60aa4f9fe1fd40f5458ba8a41126a65d0b76 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 15 Nov 2023 06:27:43 -0800
Subject: [PATCH 030/188] tbv2: replace DB template param with Ctx

TreeBuilder v2 adds a DB template parameter to every function. This is used as
part of the static type to decide what type of DataBuffer is being used:
currently `BackInserterDataBuffer<std::vector<uint8_t>>` for OIL and it would
be `DataSegmentDataBuffer` for OID.

This change replaces the `DB` template parameter with a more general `Ctx`. Due
to issues with dependent naming it also adds a `using DB` to each `TypeHandler`
which has the same function as before. This allows us to add more "static
context" (typedefs and constants) to functions without changing this signature
again, because changing the signature of everything is a massive pain.

Currently this change achieves nothing because Ctx contains only DB in a static
wrapper. In the next change I'm going to pass a reference of type Ctx around to
add a "dynamic context" to invocations which will contain the pointer array. In
future we'll then be able to add either static or dynamic context without any
signature adjustments.

Test plan:
- CI
---
 oi/CodeGen.cpp                         | 59 ++++++++++++++------------
 oi/FuncGen.cpp                         | 47 +++++++++++---------
 types/array_type.toml                  |  6 +--
 types/cxx11_list_type.toml             |  8 ++--
 types/cxx11_string_type.toml           |  5 ++-
 types/f14_fast_map.toml                | 14 +++---
 types/f14_fast_set.toml                |  6 +--
 types/f14_node_map.toml                | 14 +++---
 types/f14_node_set.toml                |  6 +--
 types/f14_value_map.toml               | 14 +++---
 types/f14_value_set.toml               |  6 +--
 types/f14_vector_map.toml              | 14 +++---
 types/f14_vector_set.toml              |  6 +--
 types/list_type.toml                   |  8 ++--
 types/map_seq_type.toml                | 24 +++++------
 types/multi_map_type.toml              | 14 +++---
 types/multi_set_type.toml              |  6 +--
 types/optional_type.toml               |  8 ++--
 types/pair_type.toml                   | 10 ++---
 types/seq_type.toml                    |  6 +--
 types/set_type.toml                    |  6 +--
 types/shrd_ptr_type.toml               |  8 ++--
 types/small_vec_type.toml              |  8 ++--
 types/sorted_vec_set_type.toml         |  6 +--
 types/std_map_type.toml                | 24 +++++------
 types/std_unordered_map_type.toml      | 24 +++++------
 types/std_unordered_multimap_type.toml | 10 ++---
 types/uniq_ptr_type.toml               |  8 ++--
 types/unordered_multiset_type.toml     |  6 +--
 types/unordered_set_type.toml          |  6 +--
 30 files changed, 200 insertions(+), 187 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 73725e4..cfa87e7 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -651,7 +651,7 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
   code += funcName;
   code += "(\n      const ";
   code += c.name();
-  code += "& t,\n      typename TypeHandler<DB, ";
+  code += "& t,\n      typename TypeHandler<Ctx, ";
   code += c.name();
   code += ">::type returnArg) {\n";
 
@@ -683,7 +683,7 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
     } else {
       code += "delegate";
     }
-    code += "([&t](auto ret) { return OIInternal::getSizeType<DB>(t.";
+    code += "([&t](auto ret) { return OIInternal::getSizeType<Ctx>(t.";
     code += member.name;
     code += ", ret); })";
   }
@@ -695,7 +695,7 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
 }
 
 // Generate the static type for the class's representation in the data buffer.
-// For `class { int a,b,c; }` we generate (DB omitted for clarity):
+// For `class { int a,b,c; }` we generate (Ctx/DB omitted for clarity):
 // Pair<TypeHandler<int>::type,
 //   Pair<TypeHandler<int>::type,
 //     TypeHandler<int>::type
@@ -734,7 +734,7 @@ void CodeGen::genClassStaticType(const Class& c, std::string& code) {
     }
 
     code +=
-        (boost::format("typename TypeHandler<DB, decltype(%1%::%2%)>::type") %
+        (boost::format("typename TypeHandler<Ctx, decltype(%1%::%2%)>::type") %
          c.name() % member.name)
             .str();
 
@@ -787,8 +787,8 @@ void CodeGen::genClassTreeBuilderInstructions(const Class& c,
     code += "      inst::Field{sizeof(" + fullName + "), " +
             std::to_string(calculateExclusiveSize(m.type())) + ",\"" +
             m.inputName + "\", member_" + std::to_string(index) +
-            "_type_names, TypeHandler<DB, decltype(" + fullName +
-            ")>::fields, TypeHandler<DB, decltype(" + fullName +
+            "_type_names, TypeHandler<Ctx, decltype(" + fullName +
+            ")>::fields, TypeHandler<Ctx, decltype(" + fullName +
             ")>::processors},\n";
   }
   code += "  };\n";
@@ -820,10 +820,11 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) {
                    .str();
   }
 
-  code += "template <typename DB>\n";
-  code += "class TypeHandler<DB, ";
+  code += "template <typename Ctx>\n";
+  code += "class TypeHandler<Ctx, ";
   code += c.name();
   code += "> {\n";
+  code += "  using DB = typename Ctx::DataBuffer;\n";
   code += helpers;
   code += " public:\n";
   code += "  using type = ";
@@ -877,7 +878,7 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     containerWithTypes = "OICaptureKeys<" + containerWithTypes + ">";
   }
 
-  code += "template <typename DB";
+  code += "template <typename Ctx";
   types = 0, values = 0;
   for (const auto& p : templateParams) {
     if (p.value) {
@@ -889,9 +890,10 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     }
   }
   code += ">\n";
-  code += "struct TypeHandler<DB, ";
+  code += "struct TypeHandler<Ctx, ";
   code += containerWithTypes;
   code += "> {\n";
+  code += "  using DB = typename Ctx::DataBuffer;\n";
 
   if (c.captureKeys) {
     code += "  static constexpr bool captureKeys = true;\n";
@@ -922,7 +924,7 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   code += "      const ";
   code += containerWithTypes;
   code += "& container,\n";
-  code += "      typename TypeHandler<DB, ";
+  code += "      typename TypeHandler<Ctx, ";
   code += containerWithTypes;
   code += ">::type returnArg) {\n";
   code += func;  // has rubbish indentation
@@ -961,8 +963,9 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
 
 void addCaptureKeySupport(std::string& code) {
   code += R"(
-    template <typename DB, typename T>
+    template <typename Ctx, typename T>
     class CaptureKeyHandler {
+      using DB = typename Ctx::DataBuffer;
      public:
       using type = types::st::Sum<DB, types::st::VarInt<DB>, types::st::VarInt<DB>>;
 
@@ -975,24 +978,24 @@ void addCaptureKeySupport(std::string& code) {
       }
     };
 
-    template <bool CaptureKeys, typename DB, typename T>
+    template <bool CaptureKeys, typename Ctx, typename T>
     auto maybeCaptureKey(const T& key, auto returnArg) {
       if constexpr (CaptureKeys) {
         return returnArg.delegate([&key](auto ret) {
-          return CaptureKeyHandler<DB, T>::captureKey(key, ret);
+          return CaptureKeyHandler<Ctx, T>::captureKey(key, ret);
         });
       } else {
         return returnArg;
       }
     }
 
-    template <typename DB, typename T>
+    template <typename Ctx, typename T>
     static constexpr inst::ProcessorInst CaptureKeysProcessor{
-      CaptureKeyHandler<DB, T>::type::describe,
+      CaptureKeyHandler<Ctx, T>::type::describe,
       [](result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
         if constexpr (std::is_same_v<
-            typename CaptureKeyHandler<DB, T>::type,
-            types::st::List<DB, types::st::VarInt<DB>>>) {
+            typename CaptureKeyHandler<Ctx, T>::type,
+            types::st::List<typename Ctx::DataBuffer, types::st::VarInt<typename Ctx::DataBuffer>>>) {
           // String
           auto& str = el.data.emplace<std::string>();
           auto list = std::get<ParsedData::List>(d.val);
@@ -1013,11 +1016,11 @@ void addCaptureKeySupport(std::string& code) {
       }
     };
 
-    template <bool CaptureKeys, typename DB, typename T>
+    template <bool CaptureKeys, typename Ctx, typename T>
     static constexpr auto maybeCaptureKeysProcessor() {
       if constexpr (CaptureKeys) {
         return std::array<inst::ProcessorInst, 1>{
-          CaptureKeysProcessor<DB, T>,
+          CaptureKeysProcessor<Ctx, T>,
         };
       }
       else {
@@ -1034,28 +1037,28 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
     addCaptureKeySupport(code);
 
   // Provide a wrapper function, getSizeType, to infer T instead of having to
-  // explicitly specify it with TypeHandler<DB, T>::getSizeType every time.
+  // explicitly specify it with TypeHandler<Ctx, T>::getSizeType every time.
   code += R"(
-    template <typename DB, typename T>
-    types::st::Unit<DB>
-    getSizeType(const T &t, typename TypeHandler<DB, T>::type returnArg) {
+    template <typename Ctx, typename T>
+    types::st::Unit<typename Ctx::DataBuffer>
+    getSizeType(const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
       JLOG("obj @");
       JLOGPTR(&t);
-      return TypeHandler<DB, T>::getSizeType(t, returnArg);
+      return TypeHandler<Ctx, T>::getSizeType(t, returnArg);
     }
 )";
 
   if (features[Feature::TreeBuilderV2]) {
     code += R"(
-template <typename DB, typename T>
+template <typename Ctx, typename T>
 constexpr inst::Field make_field(std::string_view name) {
   return inst::Field{
     sizeof(T),
     ExclusiveSizeProvider<T>::size,
     name,
     NameProvider<T>::names,
-    TypeHandler<DB, T>::fields,
-    TypeHandler<DB, T>::processors,
+    TypeHandler<Ctx, T>::fields,
+    TypeHandler<Ctx, T>::processors,
   };
 }
 )";
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 7a1dd09..1a0315f 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -268,11 +268,13 @@ void __attribute__((used, retain)) introspect_%2$016x(
   v.clear();
   v.reserve(4096);
 
-  using DataBufferType = DataBuffer::BackInserter<std::vector<uint8_t>>;
-  using ContentType = OIInternal::TypeHandler<DataBufferType, OIInternal::__ROOT_TYPE__>::type;
+  struct Context {
+    using DataBuffer = DataBuffer::BackInserter<std::vector<uint8_t>>;
+  };
+  using ContentType = OIInternal::TypeHandler<Context, OIInternal::__ROOT_TYPE__>::type;
 
-  ContentType ret{DataBufferType{v}};
-  OIInternal::getSizeType<DataBufferType>(t, ret);
+  ContentType ret{Context::DataBuffer{v}};
+  OIInternal::getSizeType<Context>(t, ret);
 }
 )";
 
@@ -364,6 +366,9 @@ void FuncGen::DefineTreeBuilderInstructions(
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunknown-attributes"
 namespace {
+struct FakeContext {
+  using DataBuffer = int;
+};
 const std::array<std::string_view, )";
   code += std::to_string(typeNames.size());
   code += "> typeNames";
@@ -382,8 +387,10 @@ const std::array<std::string_view, )";
   code += ", \"a0\", typeNames";
   code += typeHash;
   code +=
-      ", OIInternal::TypeHandler<int, OIInternal::__ROOT_TYPE__>::fields, "
-      "OIInternal::TypeHandler<int, OIInternal::__ROOT_TYPE__>::processors};\n";
+      ", OIInternal::TypeHandler<FakeContext, "
+      "OIInternal::__ROOT_TYPE__>::fields, "
+      "OIInternal::TypeHandler<FakeContext, "
+      "OIInternal::__ROOT_TYPE__>::processors};\n";
   code += "} // namespace\n";
   code +=
       "extern const exporters::inst::Inst __attribute__((used, retain)) "
@@ -595,14 +602,15 @@ class BackInserter {
  */
 void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   code += R"(
-    template <typename DB, typename T>
+    template <typename Ctx, typename T>
     struct TypeHandler {
+        using DB = typename Ctx::DataBuffer;
       private:
         static auto choose_type() {
             if constexpr(std::is_pointer_v<T>) {
                 return std::type_identity<types::st::Pair<DB,
                   types::st::VarInt<DB>,
-                  types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<DB, std::remove_pointer_t<T>>::type>
+                  types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>
                 >>();
             } else {
                 return std::type_identity<types::st::Unit<DB>>();
@@ -623,8 +631,8 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
             sizeof(T),
             "*",
             names,
-            TypeHandler<DB, T>::fields,
-            TypeHandler<DB, T>::processors,
+            TypeHandler<Ctx, T>::fields,
+            TypeHandler<Ctx, T>::processors,
           };
 
           const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
@@ -649,7 +657,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
           if constexpr(std::is_pointer_v<T>) {
             return std::array<inst::ProcessorInst, 2>{
               {types::st::VarInt<DB>::describe, &process_pointer},
-              {types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<DB, std::remove_pointer_t<T>>::type>::describe, &process_pointer_content},
+              {types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>::describe, &process_pointer_content},
             };
           } else {
             return std::array<inst::ProcessorInst, 0>{};
@@ -663,7 +671,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   code += R"(
         static types::st::Unit<DB> getSizeType(
           const T& t,
-          typename TypeHandler<DB, T>::type returnArg) {
+          typename TypeHandler<Ctx, T>::type returnArg) {
               if constexpr(std::is_pointer_v<T>) {
                 JLOG("ptr val @");
                 JLOGPTR(t);
@@ -671,7 +679,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
                 if (t && pointers.add((uintptr_t)t)) {
                   return r0.template delegate<1>([&t](auto ret) {
                     if constexpr (!std::is_void<std::remove_pointer_t<T>>::value) {
-                      return TypeHandler<DB, std::remove_pointer_t<T>>::getSizeType(*t, ret);
+                      return TypeHandler<Ctx, std::remove_pointer_t<T>>::getSizeType(*t, ret);
                     } else {
                       return ret;
                     }
@@ -687,8 +695,9 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   )";
 
   code += R"(
-    template <typename DB>
-    class TypeHandler<DB, void> {
+    template <typename Ctx>
+    class TypeHandler<Ctx, void> {
+        using DB = typename Ctx::DataBuffer;
       public:
         using type = types::st::Unit<DB>;
 )";
@@ -711,21 +720,21 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
   tail = tail.delegate([&container, i](auto ret) {
-      return TypeHandler<DB, T0>::getSizeType(container.vals[i], ret);
+      return TypeHandler<Ctx, T0>::getSizeType(container.vals[i], ret);
   });
 }
 return tail.finish();
 )";
   oiArray.codegen.processors.emplace_back(ContainerInfo::Processor{
-      .type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>",
+      .type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>",
       .func = R"(
 static constexpr std::array<std::string_view, 1> names{"TODO"};
 static constexpr auto childField = inst::Field{
   sizeof(T0),
   "[]",
   names,
-  TypeHandler<DB, T0>::fields,
-  TypeHandler<DB, T0>::processors,
+  TypeHandler<Ctx, T0>::fields,
+  TypeHandler<Ctx, T0>::processors,
 };
 
 el.exclusive_size = 0;
diff --git a/types/array_type.toml b/types/array_type.toml
index 6d73322..126af6a 100644
--- a/types/array_type.toml
+++ b/types/array_type.toml
@@ -34,7 +34,7 @@ traversal_func = """
 
     for (auto & it: container) {
         tail = tail.delegate([&it](auto ret) {
-            return TypeHandler<DB, T0>::getSizeType(it, ret);
+            return TypeHandler<Ctx, T0>::getSizeType(it, ret);
         });
     }
 
@@ -42,9 +42,9 @@ traversal_func = """
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 size_t size = std::get<ParsedData::List>(d.val).length;
 el.exclusive_size = N0 == 0 ? 1 : 0;
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index fb54d22..f838eee 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -39,7 +39,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -53,7 +53,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 static constexpr size_t element_size = sizeof(std::_List_node<T0>);
@@ -62,7 +62,7 @@ static_assert(false && "No known element_size for list. See types/cxx11_list_typ
 #endif
 
 static constexpr std::array<inst::Field, 1> child_field{
-  make_field<DB, T0>("*"),
+  make_field<Ctx, T0>("*"),
 };
 static constexpr inst::Field element{
   element_size,
@@ -72,7 +72,7 @@ static constexpr inst::Field element{
   child_field,
   std::array<inst::ProcessorInst, 0>{},
 };
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats{
diff --git a/types/cxx11_string_type.toml b/types/cxx11_string_type.toml
index 94e7183..c88debd 100644
--- a/types/cxx11_string_type.toml
+++ b/types/cxx11_string_type.toml
@@ -37,8 +37,9 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
 """
 
 extra = """
-template <typename DB, typename CharT, typename Traits, typename Allocator>
-class CaptureKeyHandler<DB, std::__cxx11::basic_string<CharT, Traits, Allocator>> {
+template <typename Ctx, typename CharT, typename Traits, typename Allocator>
+class CaptureKeyHandler<Ctx, std::__cxx11::basic_string<CharT, Traits, Allocator>> {
+  using DB = typename Ctx::DataBuffer;
  public:
   // List of characters
   using type = types::st::List<DB, types::st::VarInt<DB>>;
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index c8c84f2..faf2e9d 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -44,10 +44,10 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -69,8 +69,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 static constexpr size_t element_size = sizeof(typename container_type::value_type);
@@ -84,8 +84,8 @@ el.container_stats->length = list.length;
 el.exclusive_size += allocationSize - list.length * element_size;
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr inst::Field element{
diff --git a/types/f14_fast_set.toml b/types/f14_fast_set.toml
index 8516f8c..b66284b 100644
--- a/types/f14_fast_set.toml
+++ b/types/f14_fast_set.toml
@@ -43,7 +43,7 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<DB>(entry, ret);
+    return OIInternal::getSizeType<Ctx>(entry, ret);
   });
 }
 
@@ -64,7 +64,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 
 [[codegen.processor]]
 type = """
-types::st::List<DB, typename TypeHandler<DB, T0>::type>
+types::st::List<DB, typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto allocationSize = el.pointer.value();
@@ -75,7 +75,7 @@ el.container_stats->length = list.length;
 
 el.exclusive_size += allocationSize - list.length * sizeof(T0);
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index 93b53e9..a53c3e6 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -44,10 +44,10 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -69,8 +69,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 static constexpr size_t element_size = sizeof(typename container_type::value_type);
@@ -84,8 +84,8 @@ el.container_stats->length = list.length;
 el.exclusive_size += allocationSize - list.length * element_size;
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr inst::Field element{
diff --git a/types/f14_node_set.toml b/types/f14_node_set.toml
index 511c2f8..3ed0b5d 100644
--- a/types/f14_node_set.toml
+++ b/types/f14_node_set.toml
@@ -43,7 +43,7 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<DB>(entry, ret);
+    return OIInternal::getSizeType<Ctx>(entry, ret);
   });
 }
 
@@ -64,7 +64,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 
 [[codegen.processor]]
 type = """
-types::st::List<DB, typename TypeHandler<DB, T0>::type>
+types::st::List<DB, typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto allocationSize = el.pointer.value();
@@ -75,7 +75,7 @@ el.container_stats->length = list.length;
 
 el.exclusive_size += allocationSize - list.length * sizeof(T0);
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index ec4f187..8072abf 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -44,10 +44,10 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -69,8 +69,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 static constexpr size_t element_size = sizeof(typename container_type::value_type);
@@ -84,8 +84,8 @@ el.container_stats->length = list.length;
 el.exclusive_size += allocationSize - list.length * element_size;
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr inst::Field element{
diff --git a/types/f14_value_set.toml b/types/f14_value_set.toml
index 20aae30..dd4070d 100644
--- a/types/f14_value_set.toml
+++ b/types/f14_value_set.toml
@@ -43,7 +43,7 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<DB>(entry, ret);
+    return OIInternal::getSizeType<Ctx>(entry, ret);
   });
 }
 
@@ -64,7 +64,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 
 [[codegen.processor]]
 type = """
-types::st::List<DB, typename TypeHandler<DB, T0>::type>
+types::st::List<DB, typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto allocationSize = el.pointer.value();
@@ -75,7 +75,7 @@ el.container_stats->length = list.length;
 
 el.exclusive_size += allocationSize - list.length * sizeof(T0);
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index 5e29d82..ec7ac93 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -44,10 +44,10 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -69,8 +69,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 static constexpr size_t element_size = sizeof(typename container_type::value_type);
@@ -84,8 +84,8 @@ el.container_stats->length = list.length;
 el.exclusive_size += allocationSize - list.length * element_size;
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr inst::Field element{
diff --git a/types/f14_vector_set.toml b/types/f14_vector_set.toml
index 1aed210..e98b6d8 100644
--- a/types/f14_vector_set.toml
+++ b/types/f14_vector_set.toml
@@ -43,7 +43,7 @@ auto tail = returnArg
 
 for (auto &&entry: container) {
   tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<DB>(entry, ret);
+    return OIInternal::getSizeType<Ctx>(entry, ret);
   });
 }
 
@@ -64,7 +64,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 
 [[codegen.processor]]
 type = """
-types::st::List<DB, typename TypeHandler<DB, T0>::type>
+types::st::List<DB, typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto allocationSize = el.pointer.value();
@@ -75,7 +75,7 @@ el.container_stats->length = list.length;
 
 el.exclusive_size += allocationSize - list.length * sizeof(T0);
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/list_type.toml b/types/list_type.toml
index 9ea8ef7..86d457e 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -39,7 +39,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -53,7 +53,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 static constexpr size_t element_size = sizeof(std::_List_node<T0>);
@@ -62,7 +62,7 @@ static_assert(false && "No known element_size for list. See types/cxx11_list_typ
 #endif
 
 static constexpr std::array<inst::Field, 1> child_field{
-  make_field<DB, T0>("*"),
+  make_field<Ctx, T0>("*"),
 };
 static constexpr inst::Field element{
   element_size,
@@ -72,7 +72,7 @@ static constexpr inst::Field element{
   child_field,
   std::array<inst::ProcessorInst, 0>{},
 };
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats{
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index 90d424d..887df6a 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -43,11 +43,11 @@ traversal_func = '''
 
     for (const auto& kv : container) {
       tail = tail.delegate([&kv](auto ret) {
-        auto start = maybeCaptureKey<captureKeys, DB, T0>(kv.first, ret);
-        auto next = start.delegate([&kv](typename TypeHandler<DB, T0>::type ret) {
-          return OIInternal::getSizeType<DB>(kv.first, ret);
+        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
+        auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
+          return OIInternal::getSizeType<Ctx>(kv.first, ret);
         });
-        return OIInternal::getSizeType<DB>(kv.second, next);
+        return OIInternal::getSizeType<Ctx>(kv.second, next);
       });
     }
 
@@ -68,22 +68,22 @@ el.container_stats.emplace(result::Element::ContainerStats{ .capacity = std::get
 type = '''
 std::conditional_t<captureKeys,
   types::st::List<DB, types::st::Pair<DB,
-    typename CaptureKeyHandler<DB, T0>::type,
+    typename CaptureKeyHandler<Ctx, T0>::type,
     types::st::Pair<DB,
-      typename TypeHandler<DB, T0>::type,
-      typename TypeHandler<DB, T1>::type>>>,
+      typename TypeHandler<Ctx, T0>::type,
+      typename TypeHandler<Ctx, T1>::type>>>,
   types::st::List<DB, types::st::Pair<DB,
-    typename TypeHandler<DB, T0>::type,
-    typename TypeHandler<DB, T1>::type>>>
+    typename TypeHandler<Ctx, T0>::type,
+    typename TypeHandler<Ctx, T1>::type>>>
 '''
 func = '''
 using element_type = std::pair<T0, T1>;
 
 static constexpr std::array<inst::Field, 2> entryFields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
-static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, DB, T0>();
+static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, Ctx, T0>();
 static constexpr auto entry = inst::Field {
   sizeof(element_type),
   sizeof(element_type) - sizeof(T0) - sizeof(T1),
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index 7b1e622..e05a1e3 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -40,10 +40,10 @@ auto tail = returnArg
 
 for (const auto &entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -57,8 +57,8 @@ func = "el.pointer = std::get<ParsedData::VarInt>(d.val).value;"
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 #ifdef __GLIBCXX__
@@ -108,8 +108,8 @@ static_assert(false && "No known element_size for sets. See types/multi_map_type
 #endif
 
 static constexpr std::array<inst::Field, 2> elementFields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 static constexpr auto element = inst::Field {
   element_size,
diff --git a/types/multi_set_type.toml b/types/multi_set_type.toml
index 34163e8..00f8239 100644
--- a/types/multi_set_type.toml
+++ b/types/multi_set_type.toml
@@ -43,7 +43,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 // vector<bool>
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -57,7 +57,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 /* We don't have access to the _Rb_tree_node struct, so we manually re-create it
@@ -103,7 +103,7 @@ static constexpr size_t element_size = sizeof(OI__tree_node);
 static_assert(false && "No known element_size for multisets. See types/multi_set_type.toml");
 #endif
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats {
diff --git a/types/optional_type.toml b/types/optional_type.toml
index 2178743..3bd9cff 100644
--- a/types/optional_type.toml
+++ b/types/optional_type.toml
@@ -31,7 +31,7 @@ void getSizeType(const %1%<T>& container, size_t& returnArg) {
 traversal_func = """
 if (container.has_value()) {
   return returnArg.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<DB>(*container, ret);
+    return OIInternal::getSizeType<Ctx>(*container, ret);
   });
 } else {
   return returnArg.template delegate<0>(std::identity());
@@ -39,15 +39,15 @@ if (container.has_value()) {
 """
 
 [[codegen.processor]]
-type = "types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<DB, T0>::type>"
+type = "types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, T0>::type>"
 func = """
 static constexpr std::array<std::string_view, 1> names{"TODO"};
 static constexpr auto elementField = inst::Field{
   sizeof(T0),
   "el",
   names,
-  TypeHandler<DB, T0>::fields,
-  TypeHandler<DB, T0>::processors,
+  TypeHandler<Ctx, T0>::fields,
+  TypeHandler<Ctx, T0>::processors,
 };
 
 auto sum = std::get<ParsedData::Sum>(d.val);
diff --git a/types/pair_type.toml b/types/pair_type.toml
index 570d289..ba5bc38 100644
--- a/types/pair_type.toml
+++ b/types/pair_type.toml
@@ -27,19 +27,19 @@ void getSizeType(const %1%<P,Q> &container, size_t& returnArg)
 """
 
 traversal_func = """
-    return OIInternal::getSizeType<DB>(
+    return OIInternal::getSizeType<Ctx>(
         container.second,
         returnArg.delegate([&container](auto ret) {
-            return OIInternal::getSizeType<DB>(container.first, ret);
+            return OIInternal::getSizeType<Ctx>(container.first, ret);
         })
     );
 """
 
 [[codegen.processor]]
-type = "types::st::Pair<DB, typename TypeHandler<DB, T0>::type, typename TypeHandler<DB, T1>::type>"
+type = "types::st::Pair<DB, typename TypeHandler<Ctx, T0>::type, typename TypeHandler<Ctx, T1>::type>"
 func = """
-static constexpr auto firstField = make_field<DB, T0>("first");
-static constexpr auto secondField = make_field<DB, T1>("second");
+static constexpr auto firstField = make_field<Ctx, T0>("first");
+static constexpr auto secondField = make_field<Ctx, T1>("second");
 
 el.exclusive_size = sizeof(std::pair<T0, T1>) - sizeof(T0) - sizeof(T1);
 stack_ins(secondField);
diff --git a/types/seq_type.toml b/types/seq_type.toml
index 0ccfee8..88ddd1d 100644
--- a/types/seq_type.toml
+++ b/types/seq_type.toml
@@ -44,7 +44,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 // vector<bool>
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -64,9 +64,9 @@ el.container_stats.emplace(result::Element::ContainerStats{ .capacity = std::get
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats->length = list.length;
diff --git a/types/set_type.toml b/types/set_type.toml
index 43206f6..e29611e 100644
--- a/types/set_type.toml
+++ b/types/set_type.toml
@@ -44,7 +44,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 // vector<bool>
 for (auto&& it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -58,7 +58,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 /* We don't have access to the _Rb_tree_node struct, so we manually re-create it
@@ -104,7 +104,7 @@ static constexpr size_t element_size = sizeof(OI__tree_node);
 static_assert(false && "No known element_size for sets. See types/set_type.toml");
 #endif
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats {
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 693f871..125fd71 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -45,7 +45,7 @@ if constexpr (std::is_void<T0>::value) {
     return tail.template delegate<0>(std::identity());
 
   return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<DB>(*container, ret);
+    return OIInternal::getSizeType<Ctx>(*container, ret);
   });
 }
 """
@@ -60,7 +60,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<DB, T0>::type>
+  typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 #ifdef __GLIBCXX__
@@ -84,10 +84,10 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = sum.index, // 0 for empty containers/void, 1 otherwise
 });
 
-// Must be in a `if constexpr` or the compiler will complain about make_field<DB, void>
+// Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
 if constexpr (!std::is_void<T0>::value) {
   if (sum.index == 1) {
-    static constexpr auto element = make_field<DB, T0>("ptr_val");
+    static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);
   }
 }
diff --git a/types/small_vec_type.toml b/types/small_vec_type.toml
index aca3e79..d129285 100644
--- a/types/small_vec_type.toml
+++ b/types/small_vec_type.toml
@@ -56,8 +56,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&it: container) {
-  tail = tail.delegate([&it](typename TypeHandler<DB, T0>::type ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+  tail = tail.delegate([&it](typename TypeHandler<Ctx, T0>::type ret) {
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -81,7 +81,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 // Reading the `uses_intern_storage` boolean that was stored in `pointer` by the processor above.
 bool uses_intern_storage = std::exchange(el.pointer.value(), (uintptr_t)0);
@@ -97,7 +97,7 @@ if (uses_intern_storage) {
   el.exclusive_size += (el.container_stats->capacity - el.container_stats->length) * sizeof(T0);
 }
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/sorted_vec_set_type.toml b/types/sorted_vec_set_type.toml
index f17a0f7..e8b9483 100644
--- a/types/sorted_vec_set_type.toml
+++ b/types/sorted_vec_set_type.toml
@@ -38,7 +38,7 @@ auto tail = returnArg.write((uintptr_t)&container)
 
 for (const auto& el : container) {
   tail = tail.delegate([&el](auto ret) {
-    return OIInternal::getSizeType<DB>(el, ret);
+    return OIInternal::getSizeType<Ctx>(el, ret);
   });
 }
 
@@ -56,9 +56,9 @@ el.container_stats.emplace(result::Element::ContainerStats{ .capacity = std::get
 '''
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats->length = list.length;
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index 61be6be..df06cc7 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -44,11 +44,11 @@ auto tail = returnArg
 
 for (const auto &entry: container) {
   tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, DB, T0>(key, ret);
-    auto next =  start.delegate([&key](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(key, ret);
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(key, ret);
+    auto next =  start.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(key, ret);
     });
-    return OIInternal::getSizeType<DB>(value, next);
+    return OIInternal::getSizeType<Ctx>(value, next);
   });
 }
 
@@ -64,14 +64,14 @@ type = """
 std::conditional_t<captureKeys,
   types::st::List<DB,
     types::st::Pair<DB,
-      typename CaptureKeyHandler<DB, T0>::type,
+      typename CaptureKeyHandler<Ctx, T0>::type,
       types::st::Pair<DB,
-        typename TypeHandler<DB, T0>::type,
-        typename TypeHandler<DB, T1>::type>>>,
+        typename TypeHandler<Ctx, T0>::type,
+        typename TypeHandler<Ctx, T1>::type>>>,
   types::st::List<DB,
     types::st::Pair<DB,
-      typename TypeHandler<DB, T0>::type,
-      typename TypeHandler<DB, T1>::type>>
+      typename TypeHandler<Ctx, T0>::type,
+      typename TypeHandler<Ctx, T1>::type>>
 >
 """
 func = """
@@ -122,11 +122,11 @@ static_assert(false && "No known element_size for sets. See types/std_map_type.t
 #endif
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
-static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, DB, T0>();
+static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, Ctx, T0>();
 
 static constexpr inst::Field element{
   element_size,
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index 541576d..27142a1 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -47,11 +47,11 @@ auto tail = returnArg
 
 for (const auto& kv : container) {
   tail = tail.delegate([&kv](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, DB, T0>(kv.first, ret);
-    auto next = start.delegate([&kv](typename TypeHandler<DB, T0>::type ret) {
-      return OIInternal::getSizeType<DB>(kv.first, ret);
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
+    auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(kv.first, ret);
     });
-    return OIInternal::getSizeType<DB>(kv.second, next);
+    return OIInternal::getSizeType<Ctx>(kv.second, next);
   });
 }
 
@@ -79,13 +79,13 @@ type = """
 std::conditional_t<captureKeys,
  types::st::List<DB,
    types::st::Pair<DB,
-     typename CaptureKeyHandler<DB, T0>::type,
+     typename CaptureKeyHandler<Ctx, T0>::type,
      types::st::Pair<DB,
-       typename TypeHandler<DB, T0>::type,
-       typename TypeHandler<DB, T1>::type>>>,
+       typename TypeHandler<Ctx, T0>::type,
+       typename TypeHandler<Ctx, T1>::type>>>,
  types::st::List<DB, types::st::Pair<DB,
-   typename TypeHandler<DB, T0>::type,
-   typename TypeHandler<DB, T1>::type>>
+   typename TypeHandler<Ctx, T0>::type,
+   typename TypeHandler<Ctx, T1>::type>>
 >
 """
 func = """
@@ -122,11 +122,11 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
-static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, DB, T0>();
+static constexpr auto processors = maybeCaptureKeysProcessor<captureKeys, Ctx, T0>();
 
 static constexpr auto element = inst::Field{
   element_size,
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index a548623..261a0be 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -47,7 +47,7 @@ auto tail = returnArg
 
 for (const auto &it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -73,8 +73,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 [[codegen.processor]]
 type = """
 types::st::List<DB, types::st::Pair<DB,
-  typename TypeHandler<DB, T0>::type,
-  typename TypeHandler<DB, T1>::type>>
+  typename TypeHandler<Ctx, T0>::type,
+  typename TypeHandler<Ctx, T1>::type>>
 """
 func = """
 #ifdef __GLIBCXX__
@@ -110,8 +110,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 static constexpr std::array<inst::Field, 2> element_fields{
-  make_field<DB, T0>("key"),
-  make_field<DB, T1>("value"),
+  make_field<Ctx, T0>("key"),
+  make_field<Ctx, T1>("value"),
 };
 
 static constexpr auto element = inst::Field{
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index 35155f9..58364cf 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -46,7 +46,7 @@ if constexpr (std::is_void<T0>::value) {
     return tail.template delegate<0>(std::identity());
 
   return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<DB>(*container, ret);
+    return OIInternal::getSizeType<Ctx>(*container, ret);
   });
 }
 """
@@ -61,7 +61,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<DB, T0>::type>
+  typename TypeHandler<Ctx, T0>::type>
 """
 func = """
 auto sum = std::get<ParsedData::Sum>(d.val);
@@ -70,10 +70,10 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = sum.index, // 0 for empty containers/void, 1 otherwise
 });
 
-// Must be in a `if constexpr` or the compiler will complain about make_field<DB, void>
+// Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
 if constexpr (!std::is_void<T0>::value) {
   if (sum.index == 1) {
-    static constexpr auto element = make_field<DB, T0>("ptr_val");
+    static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);
   }
 }
diff --git a/types/unordered_multiset_type.toml b/types/unordered_multiset_type.toml
index 9d1449a..5c2e96b 100644
--- a/types/unordered_multiset_type.toml
+++ b/types/unordered_multiset_type.toml
@@ -45,7 +45,7 @@ auto tail = returnArg
 
 for (const auto &it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -69,7 +69,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 /* Use libstdc++ implementation __details to compute the size of Nodes and Buckets.
@@ -100,7 +100,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = list.length,
 });
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """
diff --git a/types/unordered_set_type.toml b/types/unordered_set_type.toml
index 62e0190..f8d26c9 100644
--- a/types/unordered_set_type.toml
+++ b/types/unordered_set_type.toml
@@ -45,7 +45,7 @@ auto tail = returnArg
 
 for (const auto &it : container) {
   tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<DB>(it, ret);
+    return OIInternal::getSizeType<Ctx>(it, ret);
   });
 }
 
@@ -69,7 +69,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 """
 
 [[codegen.processor]]
-type = "types::st::List<DB, typename TypeHandler<DB, T0>::type>"
+type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>"
 func = """
 #ifdef __GLIBCXX__
 /* Use libstdc++ implementation __details to compute the size of Nodes and Buckets.
@@ -100,7 +100,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = list.length,
 });
 
-static constexpr auto childField = make_field<DB, T0>("[]");
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 for (size_t i = 0; i < list.length; i++)
   stack_ins(childField);
 """

From f96d6a87f2d4201070dcd68150c0d5a54e956f0e Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 15 Nov 2023 06:27:43 -0800
Subject: [PATCH 031/188] tbv2: add dynamic context passed through all
 functions

Previously for we had some shared state between all requests, noticeably the
pointers set. This change adds a by reference value to all requests which can
hold additional mutable state. The pointers set is moved into this mutable
state for OIL, which means each concurrent request will have its own pointer
set. Doing things this way allows more features to be added in the future
without such a big code modification.

Closes #404

Test plan:
- CI
---
 oi/CodeGen.cpp                         | 25 ++++++++++++++++++-------
 oi/FuncGen.cpp                         | 26 ++++++++++++++++----------
 oi/OICodeGen.cpp                       |  8 +++++++-
 oi/OITraceCode.cpp                     |  7 ++++---
 types/array_type.toml                  |  4 ++--
 types/cxx11_list_type.toml             |  4 ++--
 types/f14_fast_map.toml                |  8 ++++----
 types/f14_fast_set.toml                |  4 ++--
 types/f14_node_map.toml                |  8 ++++----
 types/f14_node_set.toml                |  4 ++--
 types/f14_value_map.toml               |  8 ++++----
 types/f14_value_set.toml               |  4 ++--
 types/f14_vector_map.toml              |  8 ++++----
 types/f14_vector_set.toml              |  4 ++--
 types/fb_string_type.toml              |  4 ++--
 types/folly_iobuf_queue_type.toml      |  2 +-
 types/list_type.toml                   |  4 ++--
 types/map_seq_type.toml                | 10 +++++-----
 types/multi_map_type.toml              |  8 ++++----
 types/multi_set_type.toml              |  4 ++--
 types/optional_type.toml               |  4 ++--
 types/pair_type.toml                   |  6 +++---
 types/ref_wrapper_type.toml            |  2 +-
 types/seq_type.toml                    |  4 ++--
 types/set_type.toml                    |  4 ++--
 types/shrd_ptr_type.toml               |  8 ++++----
 types/small_vec_type.toml              |  4 ++--
 types/sorted_vec_set_type.toml         |  4 ++--
 types/std_map_type.toml                | 10 +++++-----
 types/std_unordered_map_type.toml      | 10 +++++-----
 types/std_unordered_multimap_type.toml |  4 ++--
 types/uniq_ptr_type.toml               |  8 ++++----
 types/unordered_multiset_type.toml     |  4 ++--
 types/unordered_set_type.toml          |  4 ++--
 34 files changed, 127 insertions(+), 103 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index cfa87e7..78b370e 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -128,8 +128,9 @@ void addIncludes(const TypeGraph& typeGraph,
     includes.emplace("oi/types/st.h");
   }
   if (features[Feature::Library]) {
-    includes.emplace("vector");
+    includes.emplace("memory");
     includes.emplace("oi/IntrospectionResult.h");
+    includes.emplace("vector");
   }
   if (features[Feature::JitTiming]) {
     includes.emplace("chrono");
@@ -428,7 +429,7 @@ void addStandardGetSizeFuncDefs(std::string& code) {
       JLOG("ptr val @");
       JLOGPTR(s_ptr);
       StoreData((uintptr_t)(s_ptr), returnArg);
-      if (s_ptr && pointers.add((uintptr_t)s_ptr)) {
+      if (s_ptr && ctx.pointers.add((uintptr_t)s_ptr)) {
         StoreData(1, returnArg);
         getSizeType(*(s_ptr), returnArg);
       } else {
@@ -649,7 +650,8 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
 
   code += "  static types::st::Unit<DB> ";
   code += funcName;
-  code += "(\n      const ";
+  code += "(\n      Ctx& ctx,\n";
+  code += "    const ";
   code += c.name();
   code += "& t,\n      typename TypeHandler<Ctx, ";
   code += c.name();
@@ -683,7 +685,8 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
     } else {
       code += "delegate";
     }
-    code += "([&t](auto ret) { return OIInternal::getSizeType<Ctx>(t.";
+    code +=
+        "([&ctx, &t](auto ret) { return OIInternal::getSizeType<Ctx>(ctx, t.";
     code += member.name;
     code += ", ret); })";
   }
@@ -921,6 +924,7 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   code += ";\n";
 
   code += "  static types::st::Unit<DB> getSizeType(\n";
+  code += "      Ctx& ctx,\n";
   code += "      const ";
   code += containerWithTypes;
   code += "& container,\n";
@@ -979,7 +983,7 @@ void addCaptureKeySupport(std::string& code) {
     };
 
     template <bool CaptureKeys, typename Ctx, typename T>
-    auto maybeCaptureKey(const T& key, auto returnArg) {
+    auto maybeCaptureKey(Ctx& ctx, const T& key, auto returnArg) {
       if constexpr (CaptureKeys) {
         return returnArg.delegate([&key](auto ret) {
           return CaptureKeyHandler<Ctx, T>::captureKey(key, ret);
@@ -1041,10 +1045,10 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
   code += R"(
     template <typename Ctx, typename T>
     types::st::Unit<typename Ctx::DataBuffer>
-    getSizeType(const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
+    getSizeType(Ctx& ctx, const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
       JLOG("obj @");
       JLOGPTR(&t);
-      return TypeHandler<Ctx, T>::getSizeType(t, returnArg);
+      return TypeHandler<Ctx, T>::getSizeType(ctx, t, returnArg);
     }
 )";
 
@@ -1233,6 +1237,13 @@ void CodeGen::generate(
   if (config_.features[Feature::CaptureThriftIsset]) {
     genDefsThrift(typeGraph, code);
   }
+  if (!config_.features[Feature::TreeBuilderV2]) {
+    code += "namespace {\n";
+    code += "static struct Context {\n";
+    code += "  PointerHashSet<(1 << 20) / sizeof(uintptr_t)> pointers;\n";
+    code += "} ctx;\n";
+    code += "} // namespace\n";
+  }
 
   /*
    * The purpose of the anonymous namespace within `OIInternal` is that
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 1a0315f..8c5dc37 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -262,19 +262,24 @@ void __attribute__((used, retain)) introspect_%2$016x(
     std::vector<uint8_t>& v)
 #pragma GCC diagnostic pop
 {
-  pointers.initialize();
-  pointers.add((uintptr_t)&t);
-
   v.clear();
   v.reserve(4096);
 
+  auto pointers = std::make_unique<PointerHashSet<(1 << 20) / sizeof(uintptr_t)>>();
+  pointers->initialize();
+
   struct Context {
     using DataBuffer = DataBuffer::BackInserter<std::vector<uint8_t>>;
+
+    PointerHashSet<(1 << 20) / sizeof(uintptr_t)>& pointers;
   };
+  Context ctx{ .pointers = *pointers };
+  ctx.pointers.add((uintptr_t)&t);
+
   using ContentType = OIInternal::TypeHandler<Context, OIInternal::__ROOT_TYPE__>::type;
 
   ContentType ret{Context::DataBuffer{v}};
-  OIInternal::getSizeType<Context>(t, ret);
+  OIInternal::getSizeType<Context>(ctx, t, ret);
 }
 )";
 
@@ -319,8 +324,8 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
     func += "      const auto startTime = std::chrono::steady_clock::now();\n";
   }
   func += R"(
-      pointers.initialize();
-      pointers.add((uintptr_t)&t);
+      ctx.pointers.initialize();
+      ctx.pointers.add((uintptr_t)&t);
       auto data = reinterpret_cast<uintptr_t*>(dataBase);
 
       size_t dataSegOffset = 0;
@@ -417,7 +422,7 @@ void FuncGen::DefineTopLevelGetSizeSmartPtr(std::string& testCode,
     func += "      const auto startTime = std::chrono::steady_clock::now();\n";
   }
   func += R"(
-      pointers.initialize();
+      ctx.pointers.initialize();
       auto data = reinterpret_cast<uintptr_t*>(dataBase);
 
       size_t dataSegOffset = 0;
@@ -670,13 +675,14 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   }
   code += R"(
         static types::st::Unit<DB> getSizeType(
+          Ctx& ctx,
           const T& t,
           typename TypeHandler<Ctx, T>::type returnArg) {
               if constexpr(std::is_pointer_v<T>) {
                 JLOG("ptr val @");
                 JLOGPTR(t);
                 auto r0 = returnArg.write((uintptr_t)t);
-                if (t && pointers.add((uintptr_t)t)) {
+                if (t && ctx.pointers.add((uintptr_t)t)) {
                   return r0.template delegate<1>([&t](auto ret) {
                     if constexpr (!std::is_void<std::remove_pointer_t<T>>::value) {
                       return TypeHandler<Ctx, std::remove_pointer_t<T>>::getSizeType(*t, ret);
@@ -719,8 +725,8 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
   oiArray.codegen.traversalFunc = R"(
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
-  tail = tail.delegate([&container, i](auto ret) {
-      return TypeHandler<Ctx, T0>::getSizeType(container.vals[i], ret);
+  tail = tail.delegate([&ctx, &container, i](auto ret) {
+      return TypeHandler<Ctx, T0>::getSizeType(ctx, container.vals[i], ret);
   });
 }
 return tail.finish();
diff --git a/oi/OICodeGen.cpp b/oi/OICodeGen.cpp
index ee18429..b9cfcbb 100644
--- a/oi/OICodeGen.cpp
+++ b/oi/OICodeGen.cpp
@@ -3049,6 +3049,12 @@ bool OICodeGen::generateJitCode(std::string& code) {
     #define SAVE_DATA(val)    StoreData(val, returnArg)
   )");
 
+  code.append("namespace {\n");
+  code.append("static struct Context {\n");
+  code.append("  PointerHashSet<(1 << 20) / sizeof(uintptr_t)> pointers;\n");
+  code.append("} ctx;\n");
+  code.append("} // namespace\n");
+
   FuncGen::DefineJitLog(code, config.features);
 
   // The purpose of the anonymous namespace within `OIInternal` is that
@@ -3267,7 +3273,7 @@ bool OICodeGen::generateJitCode(std::string& code) {
       JLOG("ptr val @");
       JLOGPTR(s_ptr);
       StoreData((uintptr_t)(s_ptr), returnArg);
-      if (s_ptr && pointers.add((uintptr_t)s_ptr)) {
+      if (s_ptr && ctx.pointers.add((uintptr_t)s_ptr)) {
         StoreData(1, returnArg);
         getSizeType(*(s_ptr), returnArg);
       } else {
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index fa0beda..495c781 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -40,10 +40,11 @@ constexpr int oidMagicId = 0x01DE8;
 
 namespace {
 
-class {
+template <size_t Size>
+class PointerHashSet {
  private:
   // 1 MiB of pointers
-  std::array<uintptr_t, (1 << 20) / sizeof(uintptr_t)> data;
+  std::array<uintptr_t, Size> data;
 
   // twang_mix64 hash function, taken from Folly where it is used
   // as the default hash function for 64-bit integers
@@ -84,7 +85,7 @@ class {
   bool add(const auto* p) {
     return add((uintptr_t)p);
   }
-} static pointers;
+};
 
 }  // namespace
 
diff --git a/types/array_type.toml b/types/array_type.toml
index 126af6a..054d0c6 100644
--- a/types/array_type.toml
+++ b/types/array_type.toml
@@ -33,8 +33,8 @@ traversal_func = """
     auto tail = returnArg.write(container.size());
 
     for (auto & it: container) {
-        tail = tail.delegate([&it](auto ret) {
-            return TypeHandler<Ctx, T0>::getSizeType(it, ret);
+        tail = tail.delegate([&ctx, &it](auto ret) {
+            return TypeHandler<Ctx, T0>::getSizeType(ctx, it, ret);
         });
     }
 
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index f838eee..6788750 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -38,8 +38,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index faf2e9d..06c6f72 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_fast_set.toml b/types/f14_fast_set.toml
index b66284b..fb0017c 100644
--- a/types/f14_fast_set.toml
+++ b/types/f14_fast_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index a53c3e6..de1dcda 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_node_set.toml b/types/f14_node_set.toml
index 3ed0b5d..224789e 100644
--- a/types/f14_node_set.toml
+++ b/types/f14_node_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index 8072abf..c09bbf0 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_value_set.toml b/types/f14_value_set.toml
index dd4070d..f37f7d0 100644
--- a/types/f14_value_set.toml
+++ b/types/f14_value_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index ec7ac93..a673bbd 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_vector_set.toml b/types/f14_vector_set.toml
index e98b6d8..d85f36d 100644
--- a/types/f14_vector_set.toml
+++ b/types/f14_vector_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/fb_string_type.toml b/types/fb_string_type.toml
index e0679d3..169be87 100644
--- a/types/fb_string_type.toml
+++ b/types/fb_string_type.toml
@@ -29,7 +29,7 @@ void getSizeType(const %1%<E, T, A, Storage> &container, size_t& returnArg)
         &&
       ((uintptr_t)container.data() >= (uintptr_t)&container);
 
-    if (!inlined && pointers.add((uintptr_t)container.data())) {
+    if (!inlined && ctx.pointers.add((uintptr_t)container.data())) {
       SAVE_SIZE(container.capacity() * sizeof(T));
       SAVE_DATA(1);
     } else {
@@ -60,7 +60,7 @@ if (isStorageInline(container)) {
   category = Category::InlinedStorage;
 } else if (capacity < minLargeSize) {
   category = Category::OwnedHeapStorage;
-} else if (pointers.add(container.data())) {
+} else if (ctx.pointers.add(container.data())) {
   category = Category::ReferenceCountedStorage;
 } else {
   category = Category::AlreadyAttributed;
diff --git a/types/folly_iobuf_queue_type.toml b/types/folly_iobuf_queue_type.toml
index 0af134d..b9ef72c 100644
--- a/types/folly_iobuf_queue_type.toml
+++ b/types/folly_iobuf_queue_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1% &container, size_t& returnArg)
 
     const folly::IOBuf *head = container.front();
     SAVE_DATA((uintptr_t)head);
-    if (head && pointers.add((uintptr_t)head)) {
+    if (head && ctx.pointers.add((uintptr_t)head)) {
         SAVE_DATA(1);
         getSizeType(*head, returnArg);
     } else {
diff --git a/types/list_type.toml b/types/list_type.toml
index 86d457e..3e0da3c 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -38,8 +38,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index 887df6a..a09efff 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -42,12 +42,12 @@ traversal_func = '''
                     .write(container.size());
 
     for (const auto& kv : container) {
-      tail = tail.delegate([&kv](auto ret) {
-        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
-        auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
-          return OIInternal::getSizeType<Ctx>(kv.first, ret);
+      tail = tail.delegate([&ctx, &kv](auto ret) {
+        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, kv.first, ret);
+        auto next = start.delegate([&ctx, &kv](typename TypeHandler<Ctx, T0>::type ret) {
+          return OIInternal::getSizeType<Ctx>(ctx, kv.first, ret);
         });
-        return OIInternal::getSizeType<Ctx>(kv.second, next);
+        return OIInternal::getSizeType<Ctx>(ctx, kv.second, next);
       });
     }
 
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index e05a1e3..a5d628f 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -39,11 +39,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/multi_set_type.toml b/types/multi_set_type.toml
index 00f8239..f238356 100644
--- a/types/multi_set_type.toml
+++ b/types/multi_set_type.toml
@@ -42,8 +42,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/optional_type.toml b/types/optional_type.toml
index 3bd9cff..eeba64e 100644
--- a/types/optional_type.toml
+++ b/types/optional_type.toml
@@ -30,8 +30,8 @@ void getSizeType(const %1%<T>& container, size_t& returnArg) {
 
 traversal_func = """
 if (container.has_value()) {
-  return returnArg.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return returnArg.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 } else {
   return returnArg.template delegate<0>(std::identity());
diff --git a/types/pair_type.toml b/types/pair_type.toml
index ba5bc38..f157144 100644
--- a/types/pair_type.toml
+++ b/types/pair_type.toml
@@ -27,10 +27,10 @@ void getSizeType(const %1%<P,Q> &container, size_t& returnArg)
 """
 
 traversal_func = """
-    return OIInternal::getSizeType<Ctx>(
+    return OIInternal::getSizeType<Ctx>(ctx, 
         container.second,
-        returnArg.delegate([&container](auto ret) {
-            return OIInternal::getSizeType<Ctx>(container.first, ret);
+        returnArg.delegate([&ctx, &container](auto ret) {
+            return OIInternal::getSizeType<Ctx>(ctx, container.first, ret);
         })
     );
 """
diff --git a/types/ref_wrapper_type.toml b/types/ref_wrapper_type.toml
index 949b385..9ed5094 100644
--- a/types/ref_wrapper_type.toml
+++ b/types/ref_wrapper_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &ref, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
     SAVE_DATA((uintptr_t)&(ref.get()));
-    if (pointers.add((uintptr_t)&ref.get())) {
+    if (ctx.pointers.add((uintptr_t)&ref.get())) {
         SAVE_DATA(1);
         getSizeType(ref.get(), returnArg);
     } else {
diff --git a/types/seq_type.toml b/types/seq_type.toml
index 88ddd1d..f561d0b 100644
--- a/types/seq_type.toml
+++ b/types/seq_type.toml
@@ -43,8 +43,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/set_type.toml b/types/set_type.toml
index e29611e..9d3bc3c 100644
--- a/types/set_type.toml
+++ b/types/set_type.toml
@@ -43,8 +43,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 125fd71..bb3a622 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -24,7 +24,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
     if constexpr (!std::is_void<T>::value) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
-        if (s_ptr && pointers.add((uintptr_t)(s_ptr.get()))) {
+        if (s_ptr && ctx.pointers.add((uintptr_t)(s_ptr.get()))) {
             SAVE_DATA(1);
             getSizeType(*(s_ptr.get()), returnArg);
         } else {
@@ -40,12 +40,12 @@ auto tail = returnArg.write((uintptr_t)container.get());
 if constexpr (std::is_void<T0>::value) {
   return tail.template delegate<0>(std::identity());
 } else {
-  bool do_visit = container && pointers.add((uintptr_t)container.get());
+  bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
   if (!do_visit)
     return tail.template delegate<0>(std::identity());
 
-  return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return tail.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 }
 """
diff --git a/types/small_vec_type.toml b/types/small_vec_type.toml
index d129285..a5f1694 100644
--- a/types/small_vec_type.toml
+++ b/types/small_vec_type.toml
@@ -56,8 +56,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&it: container) {
-  tail = tail.delegate([&it](typename TypeHandler<Ctx, T0>::type ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](typename TypeHandler<Ctx, T0>::type ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/sorted_vec_set_type.toml b/types/sorted_vec_set_type.toml
index e8b9483..a803d61 100644
--- a/types/sorted_vec_set_type.toml
+++ b/types/sorted_vec_set_type.toml
@@ -37,8 +37,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (const auto& el : container) {
-  tail = tail.delegate([&el](auto ret) {
-    return OIInternal::getSizeType<Ctx>(el, ret);
+  tail = tail.delegate([&ctx, &el](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, el, ret);
   });
 }
 
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index df06cc7..ee652ec 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -43,12 +43,12 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(key, ret);
-    auto next =  start.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, key, ret);
+    auto next =  start.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index 27142a1..5b2402f 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -46,12 +46,12 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto& kv : container) {
-  tail = tail.delegate([&kv](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
-    auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(kv.first, ret);
+  tail = tail.delegate([&ctx, &kv](auto ret) {
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, kv.first, ret);
+    auto next = start.delegate([&ctx, &kv](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, kv.first, ret);
     });
-    return OIInternal::getSizeType<Ctx>(kv.second, next);
+    return OIInternal::getSizeType<Ctx>(ctx, kv.second, next);
   });
 }
 
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index 261a0be..a6f7671 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -46,8 +46,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index 58364cf..f723f80 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -25,7 +25,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
     if constexpr (!std::is_void<T>::value) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
-        if (u_ptr && pointers.add((uintptr_t)(u_ptr.get()))) {
+        if (u_ptr && ctx.pointers.add((uintptr_t)(u_ptr.get()))) {
             SAVE_DATA(1);
             getSizeType(*(u_ptr.get()), returnArg);
         } else {
@@ -41,12 +41,12 @@ auto tail = returnArg.write((uintptr_t)container.get());
 if constexpr (std::is_void<T0>::value) {
   return tail.template delegate<0>(std::identity());
 } else {
-  bool do_visit = container && pointers.add((uintptr_t)container.get());
+  bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
   if (!do_visit)
     return tail.template delegate<0>(std::identity());
 
-  return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return tail.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 }
 """
diff --git a/types/unordered_multiset_type.toml b/types/unordered_multiset_type.toml
index 5c2e96b..bfcf203 100644
--- a/types/unordered_multiset_type.toml
+++ b/types/unordered_multiset_type.toml
@@ -44,8 +44,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/unordered_set_type.toml b/types/unordered_set_type.toml
index f8d26c9..d9a827a 100644
--- a/types/unordered_set_type.toml
+++ b/types/unordered_set_type.toml
@@ -44,8 +44,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 

From 55f113093ab4a42134b4ef8526e65ab6d8fa02ba Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 15 Nov 2023 12:02:32 -0800
Subject: [PATCH 032/188] tbv2: add dynamic context passed through all
 functions

Previously for we had some shared state between all requests, noticeably the
pointers set. This change adds a by reference value to all requests which can
hold additional mutable state. The pointers set is moved into this mutable
state for OIL, which means each concurrent request will have its own pointer
set. Doing things this way allows more features to be added in the future
without such a big code modification.

Closes #404

Test plan:
- CI
---
 oi/CodeGen.cpp                         | 25 ++++++++++++++++++-------
 oi/FuncGen.cpp                         | 26 ++++++++++++++++----------
 oi/OICodeGen.cpp                       |  8 +++++++-
 oi/OITraceCode.cpp                     |  7 ++++---
 types/array_type.toml                  |  4 ++--
 types/cxx11_list_type.toml             |  4 ++--
 types/f14_fast_map.toml                |  8 ++++----
 types/f14_fast_set.toml                |  4 ++--
 types/f14_node_map.toml                |  8 ++++----
 types/f14_node_set.toml                |  4 ++--
 types/f14_value_map.toml               |  8 ++++----
 types/f14_value_set.toml               |  4 ++--
 types/f14_vector_map.toml              |  8 ++++----
 types/f14_vector_set.toml              |  4 ++--
 types/fb_string_type.toml              |  4 ++--
 types/folly_iobuf_queue_type.toml      |  2 +-
 types/list_type.toml                   |  4 ++--
 types/map_seq_type.toml                | 10 +++++-----
 types/multi_map_type.toml              |  8 ++++----
 types/multi_set_type.toml              |  4 ++--
 types/optional_type.toml               |  4 ++--
 types/pair_type.toml                   |  6 +++---
 types/ref_wrapper_type.toml            |  2 +-
 types/seq_type.toml                    |  4 ++--
 types/set_type.toml                    |  4 ++--
 types/shrd_ptr_type.toml               |  8 ++++----
 types/small_vec_type.toml              |  4 ++--
 types/sorted_vec_set_type.toml         |  4 ++--
 types/std_map_type.toml                | 10 +++++-----
 types/std_unordered_map_type.toml      | 10 +++++-----
 types/std_unordered_multimap_type.toml |  4 ++--
 types/uniq_ptr_type.toml               |  8 ++++----
 types/unordered_multiset_type.toml     |  4 ++--
 types/unordered_set_type.toml          |  4 ++--
 34 files changed, 127 insertions(+), 103 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index cfa87e7..78b370e 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -128,8 +128,9 @@ void addIncludes(const TypeGraph& typeGraph,
     includes.emplace("oi/types/st.h");
   }
   if (features[Feature::Library]) {
-    includes.emplace("vector");
+    includes.emplace("memory");
     includes.emplace("oi/IntrospectionResult.h");
+    includes.emplace("vector");
   }
   if (features[Feature::JitTiming]) {
     includes.emplace("chrono");
@@ -428,7 +429,7 @@ void addStandardGetSizeFuncDefs(std::string& code) {
       JLOG("ptr val @");
       JLOGPTR(s_ptr);
       StoreData((uintptr_t)(s_ptr), returnArg);
-      if (s_ptr && pointers.add((uintptr_t)s_ptr)) {
+      if (s_ptr && ctx.pointers.add((uintptr_t)s_ptr)) {
         StoreData(1, returnArg);
         getSizeType(*(s_ptr), returnArg);
       } else {
@@ -649,7 +650,8 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
 
   code += "  static types::st::Unit<DB> ";
   code += funcName;
-  code += "(\n      const ";
+  code += "(\n      Ctx& ctx,\n";
+  code += "    const ";
   code += c.name();
   code += "& t,\n      typename TypeHandler<Ctx, ";
   code += c.name();
@@ -683,7 +685,8 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
     } else {
       code += "delegate";
     }
-    code += "([&t](auto ret) { return OIInternal::getSizeType<Ctx>(t.";
+    code +=
+        "([&ctx, &t](auto ret) { return OIInternal::getSizeType<Ctx>(ctx, t.";
     code += member.name;
     code += ", ret); })";
   }
@@ -921,6 +924,7 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   code += ";\n";
 
   code += "  static types::st::Unit<DB> getSizeType(\n";
+  code += "      Ctx& ctx,\n";
   code += "      const ";
   code += containerWithTypes;
   code += "& container,\n";
@@ -979,7 +983,7 @@ void addCaptureKeySupport(std::string& code) {
     };
 
     template <bool CaptureKeys, typename Ctx, typename T>
-    auto maybeCaptureKey(const T& key, auto returnArg) {
+    auto maybeCaptureKey(Ctx& ctx, const T& key, auto returnArg) {
       if constexpr (CaptureKeys) {
         return returnArg.delegate([&key](auto ret) {
           return CaptureKeyHandler<Ctx, T>::captureKey(key, ret);
@@ -1041,10 +1045,10 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
   code += R"(
     template <typename Ctx, typename T>
     types::st::Unit<typename Ctx::DataBuffer>
-    getSizeType(const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
+    getSizeType(Ctx& ctx, const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
       JLOG("obj @");
       JLOGPTR(&t);
-      return TypeHandler<Ctx, T>::getSizeType(t, returnArg);
+      return TypeHandler<Ctx, T>::getSizeType(ctx, t, returnArg);
     }
 )";
 
@@ -1233,6 +1237,13 @@ void CodeGen::generate(
   if (config_.features[Feature::CaptureThriftIsset]) {
     genDefsThrift(typeGraph, code);
   }
+  if (!config_.features[Feature::TreeBuilderV2]) {
+    code += "namespace {\n";
+    code += "static struct Context {\n";
+    code += "  PointerHashSet<(1 << 20) / sizeof(uintptr_t)> pointers;\n";
+    code += "} ctx;\n";
+    code += "} // namespace\n";
+  }
 
   /*
    * The purpose of the anonymous namespace within `OIInternal` is that
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 7cc2868..c9380d2 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -262,19 +262,24 @@ void __attribute__((used, retain)) introspect_%2$016x(
     std::vector<uint8_t>& v)
 #pragma GCC diagnostic pop
 {
-  pointers.initialize();
-  pointers.add((uintptr_t)&t);
-
   v.clear();
   v.reserve(4096);
 
+  auto pointers = std::make_unique<PointerHashSet<(1 << 20) / sizeof(uintptr_t)>>();
+  pointers->initialize();
+
   struct Context {
     using DataBuffer = DataBuffer::BackInserter<std::vector<uint8_t>>;
+
+    PointerHashSet<(1 << 20) / sizeof(uintptr_t)>& pointers;
   };
+  Context ctx{ .pointers = *pointers };
+  ctx.pointers.add((uintptr_t)&t);
+
   using ContentType = OIInternal::TypeHandler<Context, OIInternal::__ROOT_TYPE__>::type;
 
   ContentType ret{Context::DataBuffer{v}};
-  OIInternal::getSizeType<Context>(t, ret);
+  OIInternal::getSizeType<Context>(ctx, t, ret);
 }
 )";
 
@@ -319,8 +324,8 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
     func += "      const auto startTime = std::chrono::steady_clock::now();\n";
   }
   func += R"(
-      pointers.initialize();
-      pointers.add((uintptr_t)&t);
+      ctx.pointers.initialize();
+      ctx.pointers.add((uintptr_t)&t);
       auto data = reinterpret_cast<uintptr_t*>(dataBase);
 
       size_t dataSegOffset = 0;
@@ -421,7 +426,7 @@ void FuncGen::DefineTopLevelGetSizeSmartPtr(std::string& testCode,
     func += "      const auto startTime = std::chrono::steady_clock::now();\n";
   }
   func += R"(
-      pointers.initialize();
+      ctx.pointers.initialize();
       auto data = reinterpret_cast<uintptr_t*>(dataBase);
 
       size_t dataSegOffset = 0;
@@ -678,13 +683,14 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   }
   code += R"(
         static types::st::Unit<DB> getSizeType(
+          Ctx& ctx,
           const T& t,
           typename TypeHandler<Ctx, T>::type returnArg) {
               if constexpr(std::is_pointer_v<T>) {
                 JLOG("ptr val @");
                 JLOGPTR(t);
                 auto r0 = returnArg.write((uintptr_t)t);
-                if (t && pointers.add((uintptr_t)t)) {
+                if (t && ctx.pointers.add((uintptr_t)t)) {
                   return r0.template delegate<1>([&t](auto ret) {
                     if constexpr (!std::is_void<std::remove_pointer_t<T>>::value) {
                       return TypeHandler<Ctx, std::remove_pointer_t<T>>::getSizeType(*t, ret);
@@ -727,8 +733,8 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
   oiArray.codegen.traversalFunc = R"(
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
-  tail = tail.delegate([&container, i](auto ret) {
-      return TypeHandler<Ctx, T0>::getSizeType(container.vals[i], ret);
+  tail = tail.delegate([&ctx, &container, i](auto ret) {
+      return TypeHandler<Ctx, T0>::getSizeType(ctx, container.vals[i], ret);
   });
 }
 return tail.finish();
diff --git a/oi/OICodeGen.cpp b/oi/OICodeGen.cpp
index ee18429..b9cfcbb 100644
--- a/oi/OICodeGen.cpp
+++ b/oi/OICodeGen.cpp
@@ -3049,6 +3049,12 @@ bool OICodeGen::generateJitCode(std::string& code) {
     #define SAVE_DATA(val)    StoreData(val, returnArg)
   )");
 
+  code.append("namespace {\n");
+  code.append("static struct Context {\n");
+  code.append("  PointerHashSet<(1 << 20) / sizeof(uintptr_t)> pointers;\n");
+  code.append("} ctx;\n");
+  code.append("} // namespace\n");
+
   FuncGen::DefineJitLog(code, config.features);
 
   // The purpose of the anonymous namespace within `OIInternal` is that
@@ -3267,7 +3273,7 @@ bool OICodeGen::generateJitCode(std::string& code) {
       JLOG("ptr val @");
       JLOGPTR(s_ptr);
       StoreData((uintptr_t)(s_ptr), returnArg);
-      if (s_ptr && pointers.add((uintptr_t)s_ptr)) {
+      if (s_ptr && ctx.pointers.add((uintptr_t)s_ptr)) {
         StoreData(1, returnArg);
         getSizeType(*(s_ptr), returnArg);
       } else {
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index 5cc443d..161a87a 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -40,10 +40,11 @@ constexpr int oidMagicId = 0x01DE8;
 
 namespace {
 
-class {
+template <size_t Size>
+class PointerHashSet {
  private:
   // 1 MiB of pointers
-  std::array<uintptr_t, (1 << 20) / sizeof(uintptr_t)> data;
+  std::array<uintptr_t, Size> data;
   size_t numEntries;
 
   /*
@@ -107,7 +108,7 @@ class {
   bool add(const auto* p) {
     return add((uintptr_t)p);
   }
-} static pointers;
+};
 
 }  // namespace
 
diff --git a/types/array_type.toml b/types/array_type.toml
index 126af6a..054d0c6 100644
--- a/types/array_type.toml
+++ b/types/array_type.toml
@@ -33,8 +33,8 @@ traversal_func = """
     auto tail = returnArg.write(container.size());
 
     for (auto & it: container) {
-        tail = tail.delegate([&it](auto ret) {
-            return TypeHandler<Ctx, T0>::getSizeType(it, ret);
+        tail = tail.delegate([&ctx, &it](auto ret) {
+            return TypeHandler<Ctx, T0>::getSizeType(ctx, it, ret);
         });
     }
 
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index f838eee..6788750 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -38,8 +38,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index faf2e9d..06c6f72 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_fast_set.toml b/types/f14_fast_set.toml
index b66284b..fb0017c 100644
--- a/types/f14_fast_set.toml
+++ b/types/f14_fast_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index a53c3e6..de1dcda 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_node_set.toml b/types/f14_node_set.toml
index 3ed0b5d..224789e 100644
--- a/types/f14_node_set.toml
+++ b/types/f14_node_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index 8072abf..c09bbf0 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_value_set.toml b/types/f14_value_set.toml
index dd4070d..f37f7d0 100644
--- a/types/f14_value_set.toml
+++ b/types/f14_value_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index ec7ac93..a673bbd 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_vector_set.toml b/types/f14_vector_set.toml
index e98b6d8..d85f36d 100644
--- a/types/f14_vector_set.toml
+++ b/types/f14_vector_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/fb_string_type.toml b/types/fb_string_type.toml
index e0679d3..169be87 100644
--- a/types/fb_string_type.toml
+++ b/types/fb_string_type.toml
@@ -29,7 +29,7 @@ void getSizeType(const %1%<E, T, A, Storage> &container, size_t& returnArg)
         &&
       ((uintptr_t)container.data() >= (uintptr_t)&container);
 
-    if (!inlined && pointers.add((uintptr_t)container.data())) {
+    if (!inlined && ctx.pointers.add((uintptr_t)container.data())) {
       SAVE_SIZE(container.capacity() * sizeof(T));
       SAVE_DATA(1);
     } else {
@@ -60,7 +60,7 @@ if (isStorageInline(container)) {
   category = Category::InlinedStorage;
 } else if (capacity < minLargeSize) {
   category = Category::OwnedHeapStorage;
-} else if (pointers.add(container.data())) {
+} else if (ctx.pointers.add(container.data())) {
   category = Category::ReferenceCountedStorage;
 } else {
   category = Category::AlreadyAttributed;
diff --git a/types/folly_iobuf_queue_type.toml b/types/folly_iobuf_queue_type.toml
index 0af134d..b9ef72c 100644
--- a/types/folly_iobuf_queue_type.toml
+++ b/types/folly_iobuf_queue_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1% &container, size_t& returnArg)
 
     const folly::IOBuf *head = container.front();
     SAVE_DATA((uintptr_t)head);
-    if (head && pointers.add((uintptr_t)head)) {
+    if (head && ctx.pointers.add((uintptr_t)head)) {
         SAVE_DATA(1);
         getSizeType(*head, returnArg);
     } else {
diff --git a/types/list_type.toml b/types/list_type.toml
index 86d457e..3e0da3c 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -38,8 +38,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index 887df6a..a09efff 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -42,12 +42,12 @@ traversal_func = '''
                     .write(container.size());
 
     for (const auto& kv : container) {
-      tail = tail.delegate([&kv](auto ret) {
-        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
-        auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
-          return OIInternal::getSizeType<Ctx>(kv.first, ret);
+      tail = tail.delegate([&ctx, &kv](auto ret) {
+        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, kv.first, ret);
+        auto next = start.delegate([&ctx, &kv](typename TypeHandler<Ctx, T0>::type ret) {
+          return OIInternal::getSizeType<Ctx>(ctx, kv.first, ret);
         });
-        return OIInternal::getSizeType<Ctx>(kv.second, next);
+        return OIInternal::getSizeType<Ctx>(ctx, kv.second, next);
       });
     }
 
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index e05a1e3..a5d628f 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -39,11 +39,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/multi_set_type.toml b/types/multi_set_type.toml
index 00f8239..f238356 100644
--- a/types/multi_set_type.toml
+++ b/types/multi_set_type.toml
@@ -42,8 +42,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/optional_type.toml b/types/optional_type.toml
index 3bd9cff..eeba64e 100644
--- a/types/optional_type.toml
+++ b/types/optional_type.toml
@@ -30,8 +30,8 @@ void getSizeType(const %1%<T>& container, size_t& returnArg) {
 
 traversal_func = """
 if (container.has_value()) {
-  return returnArg.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return returnArg.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 } else {
   return returnArg.template delegate<0>(std::identity());
diff --git a/types/pair_type.toml b/types/pair_type.toml
index ba5bc38..f157144 100644
--- a/types/pair_type.toml
+++ b/types/pair_type.toml
@@ -27,10 +27,10 @@ void getSizeType(const %1%<P,Q> &container, size_t& returnArg)
 """
 
 traversal_func = """
-    return OIInternal::getSizeType<Ctx>(
+    return OIInternal::getSizeType<Ctx>(ctx, 
         container.second,
-        returnArg.delegate([&container](auto ret) {
-            return OIInternal::getSizeType<Ctx>(container.first, ret);
+        returnArg.delegate([&ctx, &container](auto ret) {
+            return OIInternal::getSizeType<Ctx>(ctx, container.first, ret);
         })
     );
 """
diff --git a/types/ref_wrapper_type.toml b/types/ref_wrapper_type.toml
index 949b385..9ed5094 100644
--- a/types/ref_wrapper_type.toml
+++ b/types/ref_wrapper_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &ref, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
     SAVE_DATA((uintptr_t)&(ref.get()));
-    if (pointers.add((uintptr_t)&ref.get())) {
+    if (ctx.pointers.add((uintptr_t)&ref.get())) {
         SAVE_DATA(1);
         getSizeType(ref.get(), returnArg);
     } else {
diff --git a/types/seq_type.toml b/types/seq_type.toml
index 88ddd1d..f561d0b 100644
--- a/types/seq_type.toml
+++ b/types/seq_type.toml
@@ -43,8 +43,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/set_type.toml b/types/set_type.toml
index e29611e..9d3bc3c 100644
--- a/types/set_type.toml
+++ b/types/set_type.toml
@@ -43,8 +43,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 125fd71..bb3a622 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -24,7 +24,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
     if constexpr (!std::is_void<T>::value) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
-        if (s_ptr && pointers.add((uintptr_t)(s_ptr.get()))) {
+        if (s_ptr && ctx.pointers.add((uintptr_t)(s_ptr.get()))) {
             SAVE_DATA(1);
             getSizeType(*(s_ptr.get()), returnArg);
         } else {
@@ -40,12 +40,12 @@ auto tail = returnArg.write((uintptr_t)container.get());
 if constexpr (std::is_void<T0>::value) {
   return tail.template delegate<0>(std::identity());
 } else {
-  bool do_visit = container && pointers.add((uintptr_t)container.get());
+  bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
   if (!do_visit)
     return tail.template delegate<0>(std::identity());
 
-  return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return tail.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 }
 """
diff --git a/types/small_vec_type.toml b/types/small_vec_type.toml
index d129285..a5f1694 100644
--- a/types/small_vec_type.toml
+++ b/types/small_vec_type.toml
@@ -56,8 +56,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&it: container) {
-  tail = tail.delegate([&it](typename TypeHandler<Ctx, T0>::type ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](typename TypeHandler<Ctx, T0>::type ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/sorted_vec_set_type.toml b/types/sorted_vec_set_type.toml
index e8b9483..a803d61 100644
--- a/types/sorted_vec_set_type.toml
+++ b/types/sorted_vec_set_type.toml
@@ -37,8 +37,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (const auto& el : container) {
-  tail = tail.delegate([&el](auto ret) {
-    return OIInternal::getSizeType<Ctx>(el, ret);
+  tail = tail.delegate([&ctx, &el](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, el, ret);
   });
 }
 
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index df06cc7..ee652ec 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -43,12 +43,12 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(key, ret);
-    auto next =  start.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, key, ret);
+    auto next =  start.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index 27142a1..5b2402f 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -46,12 +46,12 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto& kv : container) {
-  tail = tail.delegate([&kv](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
-    auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(kv.first, ret);
+  tail = tail.delegate([&ctx, &kv](auto ret) {
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, kv.first, ret);
+    auto next = start.delegate([&ctx, &kv](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, kv.first, ret);
     });
-    return OIInternal::getSizeType<Ctx>(kv.second, next);
+    return OIInternal::getSizeType<Ctx>(ctx, kv.second, next);
   });
 }
 
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index 261a0be..a6f7671 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -46,8 +46,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index 58364cf..f723f80 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -25,7 +25,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
     if constexpr (!std::is_void<T>::value) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
-        if (u_ptr && pointers.add((uintptr_t)(u_ptr.get()))) {
+        if (u_ptr && ctx.pointers.add((uintptr_t)(u_ptr.get()))) {
             SAVE_DATA(1);
             getSizeType(*(u_ptr.get()), returnArg);
         } else {
@@ -41,12 +41,12 @@ auto tail = returnArg.write((uintptr_t)container.get());
 if constexpr (std::is_void<T0>::value) {
   return tail.template delegate<0>(std::identity());
 } else {
-  bool do_visit = container && pointers.add((uintptr_t)container.get());
+  bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
   if (!do_visit)
     return tail.template delegate<0>(std::identity());
 
-  return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return tail.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 }
 """
diff --git a/types/unordered_multiset_type.toml b/types/unordered_multiset_type.toml
index 5c2e96b..bfcf203 100644
--- a/types/unordered_multiset_type.toml
+++ b/types/unordered_multiset_type.toml
@@ -44,8 +44,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/unordered_set_type.toml b/types/unordered_set_type.toml
index f8d26c9..d9a827a 100644
--- a/types/unordered_set_type.toml
+++ b/types/unordered_set_type.toml
@@ -44,8 +44,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 

From 1ed01c21977f9ad4d89d76007919fd3e008ccc1b Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 16 Nov 2023 03:35:24 -0800
Subject: [PATCH 033/188] tbv2: add dynamic context passed through all
 functions

Previously for we had some shared state between all requests, noticeably the
pointers set. This change adds a by reference value to all requests which can
hold additional mutable state. The pointers set is moved into this mutable
state for OIL, which means each concurrent request will have its own pointer
set. Doing things this way allows more features to be added in the future
without such a big code modification.

Closes #404

Test plan:
- CI
---
 oi/CodeGen.cpp                         | 25 +++++++++++++------
 oi/FuncGen.cpp                         | 34 +++++++++++++++-----------
 oi/OICodeGen.cpp                       |  8 +++++-
 oi/OITraceCode.cpp                     |  7 +++---
 types/array_type.toml                  |  4 +--
 types/cxx11_list_type.toml             |  4 +--
 types/f14_fast_map.toml                |  8 +++---
 types/f14_fast_set.toml                |  4 +--
 types/f14_node_map.toml                |  8 +++---
 types/f14_node_set.toml                |  4 +--
 types/f14_value_map.toml               |  8 +++---
 types/f14_value_set.toml               |  4 +--
 types/f14_vector_map.toml              |  8 +++---
 types/f14_vector_set.toml              |  4 +--
 types/fb_string_type.toml              |  4 +--
 types/folly_iobuf_queue_type.toml      |  2 +-
 types/list_type.toml                   |  4 +--
 types/map_seq_type.toml                | 10 ++++----
 types/multi_map_type.toml              |  8 +++---
 types/multi_set_type.toml              |  4 +--
 types/optional_type.toml               |  4 +--
 types/pair_type.toml                   |  6 ++---
 types/ref_wrapper_type.toml            |  2 +-
 types/seq_type.toml                    |  4 +--
 types/set_type.toml                    |  4 +--
 types/shrd_ptr_type.toml               |  8 +++---
 types/small_vec_type.toml              |  4 +--
 types/sorted_vec_set_type.toml         |  4 +--
 types/std_map_type.toml                | 10 ++++----
 types/std_unordered_map_type.toml      | 10 ++++----
 types/std_unordered_multimap_type.toml |  4 +--
 types/uniq_ptr_type.toml               |  8 +++---
 types/unordered_multiset_type.toml     |  4 +--
 types/unordered_set_type.toml          |  4 +--
 34 files changed, 131 insertions(+), 107 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index cfa87e7..bef102f 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -128,8 +128,9 @@ void addIncludes(const TypeGraph& typeGraph,
     includes.emplace("oi/types/st.h");
   }
   if (features[Feature::Library]) {
-    includes.emplace("vector");
+    includes.emplace("memory");
     includes.emplace("oi/IntrospectionResult.h");
+    includes.emplace("vector");
   }
   if (features[Feature::JitTiming]) {
     includes.emplace("chrono");
@@ -428,7 +429,7 @@ void addStandardGetSizeFuncDefs(std::string& code) {
       JLOG("ptr val @");
       JLOGPTR(s_ptr);
       StoreData((uintptr_t)(s_ptr), returnArg);
-      if (s_ptr && pointers.add((uintptr_t)s_ptr)) {
+      if (s_ptr && ctx.pointers.add((uintptr_t)s_ptr)) {
         StoreData(1, returnArg);
         getSizeType(*(s_ptr), returnArg);
       } else {
@@ -649,7 +650,8 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
 
   code += "  static types::st::Unit<DB> ";
   code += funcName;
-  code += "(\n      const ";
+  code += "(\n      Ctx& ctx,\n";
+  code += "    const ";
   code += c.name();
   code += "& t,\n      typename TypeHandler<Ctx, ";
   code += c.name();
@@ -683,7 +685,8 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
     } else {
       code += "delegate";
     }
-    code += "([&t](auto ret) { return OIInternal::getSizeType<Ctx>(t.";
+    code +=
+        "([&ctx, &t](auto ret) { return OIInternal::getSizeType<Ctx>(ctx, t.";
     code += member.name;
     code += ", ret); })";
   }
@@ -921,6 +924,7 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   code += ";\n";
 
   code += "  static types::st::Unit<DB> getSizeType(\n";
+  code += "      Ctx& ctx,\n";
   code += "      const ";
   code += containerWithTypes;
   code += "& container,\n";
@@ -979,7 +983,7 @@ void addCaptureKeySupport(std::string& code) {
     };
 
     template <bool CaptureKeys, typename Ctx, typename T>
-    auto maybeCaptureKey(const T& key, auto returnArg) {
+    auto maybeCaptureKey(Ctx& ctx, const T& key, auto returnArg) {
       if constexpr (CaptureKeys) {
         return returnArg.delegate([&key](auto ret) {
           return CaptureKeyHandler<Ctx, T>::captureKey(key, ret);
@@ -1041,10 +1045,10 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
   code += R"(
     template <typename Ctx, typename T>
     types::st::Unit<typename Ctx::DataBuffer>
-    getSizeType(const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
+    getSizeType(Ctx& ctx, const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
       JLOG("obj @");
       JLOGPTR(&t);
-      return TypeHandler<Ctx, T>::getSizeType(t, returnArg);
+      return TypeHandler<Ctx, T>::getSizeType(ctx, t, returnArg);
     }
 )";
 
@@ -1233,6 +1237,13 @@ void CodeGen::generate(
   if (config_.features[Feature::CaptureThriftIsset]) {
     genDefsThrift(typeGraph, code);
   }
+  if (!config_.features[Feature::TreeBuilderV2]) {
+    code += "namespace {\n";
+    code += "static struct Context {\n";
+    code += "  PointerHashSet<> pointers;\n";
+    code += "} ctx;\n";
+    code += "} // namespace\n";
+  }
 
   /*
    * The purpose of the anonymous namespace within `OIInternal` is that
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 7cc2868..0b1e293 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -262,19 +262,24 @@ void __attribute__((used, retain)) introspect_%2$016x(
     std::vector<uint8_t>& v)
 #pragma GCC diagnostic pop
 {
-  pointers.initialize();
-  pointers.add((uintptr_t)&t);
-
   v.clear();
   v.reserve(4096);
 
+  auto pointers = std::make_unique<PointerHashSet<>>();
+  pointers->initialize();
+
   struct Context {
     using DataBuffer = DataBuffer::BackInserter<std::vector<uint8_t>>;
+
+    PointerHashSet<>& pointers;
   };
+  Context ctx{ .pointers = *pointers };
+  ctx.pointers.add((uintptr_t)&t);
+
   using ContentType = OIInternal::TypeHandler<Context, OIInternal::__ROOT_TYPE__>::type;
 
   ContentType ret{Context::DataBuffer{v}};
-  OIInternal::getSizeType<Context>(t, ret);
+  OIInternal::getSizeType<Context>(ctx, t, ret);
 }
 )";
 
@@ -319,8 +324,8 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
     func += "      const auto startTime = std::chrono::steady_clock::now();\n";
   }
   func += R"(
-      pointers.initialize();
-      pointers.add((uintptr_t)&t);
+      ctx.pointers.initialize();
+      ctx.pointers.add((uintptr_t)&t);
       auto data = reinterpret_cast<uintptr_t*>(dataBase);
 
       size_t dataSegOffset = 0;
@@ -340,8 +345,8 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
       OIInternal::StoreData((uintptr_t)123456789, dataSegOffset);
       writtenSize = dataSegOffset;
       dataBase += dataSegOffset;
-      pointersSize = pointers.size();
-      pointersCapacity = pointers.capacity();
+      pointersSize = ctx.pointers.size();
+      pointersCapacity = ctx.pointers.capacity();
     )";
   if (features[Feature::JitTiming]) {
     func += R"(
@@ -421,7 +426,7 @@ void FuncGen::DefineTopLevelGetSizeSmartPtr(std::string& testCode,
     func += "      const auto startTime = std::chrono::steady_clock::now();\n";
   }
   func += R"(
-      pointers.initialize();
+      ctx.pointers.initialize();
       auto data = reinterpret_cast<uintptr_t*>(dataBase);
 
       size_t dataSegOffset = 0;
@@ -440,8 +445,8 @@ void FuncGen::DefineTopLevelGetSizeSmartPtr(std::string& testCode,
       OIInternal::StoreData((uintptr_t)123456789, dataSegOffset);
       writtenSize = dataSegOffset;
       dataBase += dataSegOffset;
-      pointersSize = pointers.size();
-      pointersCapacity = pointers.capacity();
+      pointersSize = ctx.pointers.size();
+      pointersCapacity = ctx.pointers.capacity();
     )";
   if (features[Feature::JitTiming]) {
     func += R"(
@@ -678,13 +683,14 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
   }
   code += R"(
         static types::st::Unit<DB> getSizeType(
+          Ctx& ctx,
           const T& t,
           typename TypeHandler<Ctx, T>::type returnArg) {
               if constexpr(std::is_pointer_v<T>) {
                 JLOG("ptr val @");
                 JLOGPTR(t);
                 auto r0 = returnArg.write((uintptr_t)t);
-                if (t && pointers.add((uintptr_t)t)) {
+                if (t && ctx.pointers.add((uintptr_t)t)) {
                   return r0.template delegate<1>([&t](auto ret) {
                     if constexpr (!std::is_void<std::remove_pointer_t<T>>::value) {
                       return TypeHandler<Ctx, std::remove_pointer_t<T>>::getSizeType(*t, ret);
@@ -727,8 +733,8 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
   oiArray.codegen.traversalFunc = R"(
 auto tail = returnArg.write(N0);
 for (size_t i=0; i<N0; i++) {
-  tail = tail.delegate([&container, i](auto ret) {
-      return TypeHandler<Ctx, T0>::getSizeType(container.vals[i], ret);
+  tail = tail.delegate([&ctx, &container, i](auto ret) {
+      return TypeHandler<Ctx, T0>::getSizeType(ctx, container.vals[i], ret);
   });
 }
 return tail.finish();
diff --git a/oi/OICodeGen.cpp b/oi/OICodeGen.cpp
index ee18429..eaa25ad 100644
--- a/oi/OICodeGen.cpp
+++ b/oi/OICodeGen.cpp
@@ -3049,6 +3049,12 @@ bool OICodeGen::generateJitCode(std::string& code) {
     #define SAVE_DATA(val)    StoreData(val, returnArg)
   )");
 
+  code.append("namespace {\n");
+  code.append("static struct Context {\n");
+  code.append("  PointerHashSet<> pointers;\n");
+  code.append("} ctx;\n");
+  code.append("} // namespace\n");
+
   FuncGen::DefineJitLog(code, config.features);
 
   // The purpose of the anonymous namespace within `OIInternal` is that
@@ -3267,7 +3273,7 @@ bool OICodeGen::generateJitCode(std::string& code) {
       JLOG("ptr val @");
       JLOGPTR(s_ptr);
       StoreData((uintptr_t)(s_ptr), returnArg);
-      if (s_ptr && pointers.add((uintptr_t)s_ptr)) {
+      if (s_ptr && ctx.pointers.add((uintptr_t)s_ptr)) {
         StoreData(1, returnArg);
         getSizeType(*(s_ptr), returnArg);
       } else {
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index 5cc443d..31a873b 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -40,10 +40,11 @@ constexpr int oidMagicId = 0x01DE8;
 
 namespace {
 
-class {
+template <size_t Size = (1 << 20) / sizeof(uintptr_t)>
+class PointerHashSet {
  private:
   // 1 MiB of pointers
-  std::array<uintptr_t, (1 << 20) / sizeof(uintptr_t)> data;
+  std::array<uintptr_t, Size> data;
   size_t numEntries;
 
   /*
@@ -107,7 +108,7 @@ class {
   bool add(const auto* p) {
     return add((uintptr_t)p);
   }
-} static pointers;
+};
 
 }  // namespace
 
diff --git a/types/array_type.toml b/types/array_type.toml
index 126af6a..054d0c6 100644
--- a/types/array_type.toml
+++ b/types/array_type.toml
@@ -33,8 +33,8 @@ traversal_func = """
     auto tail = returnArg.write(container.size());
 
     for (auto & it: container) {
-        tail = tail.delegate([&it](auto ret) {
-            return TypeHandler<Ctx, T0>::getSizeType(it, ret);
+        tail = tail.delegate([&ctx, &it](auto ret) {
+            return TypeHandler<Ctx, T0>::getSizeType(ctx, it, ret);
         });
     }
 
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index f838eee..6788750 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -38,8 +38,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index faf2e9d..06c6f72 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_fast_set.toml b/types/f14_fast_set.toml
index b66284b..fb0017c 100644
--- a/types/f14_fast_set.toml
+++ b/types/f14_fast_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index a53c3e6..de1dcda 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_node_set.toml b/types/f14_node_set.toml
index 3ed0b5d..224789e 100644
--- a/types/f14_node_set.toml
+++ b/types/f14_node_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index 8072abf..c09bbf0 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_value_set.toml b/types/f14_value_set.toml
index dd4070d..f37f7d0 100644
--- a/types/f14_value_set.toml
+++ b/types/f14_value_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index ec7ac93..a673bbd 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -43,11 +43,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/f14_vector_set.toml b/types/f14_vector_set.toml
index e98b6d8..d85f36d 100644
--- a/types/f14_vector_set.toml
+++ b/types/f14_vector_set.toml
@@ -42,8 +42,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&entry: container) {
-  tail = tail.delegate([&entry](auto ret) {
-    return OIInternal::getSizeType<Ctx>(entry, ret);
+  tail = tail.delegate([&ctx, &entry](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, entry, ret);
   });
 }
 
diff --git a/types/fb_string_type.toml b/types/fb_string_type.toml
index e0679d3..169be87 100644
--- a/types/fb_string_type.toml
+++ b/types/fb_string_type.toml
@@ -29,7 +29,7 @@ void getSizeType(const %1%<E, T, A, Storage> &container, size_t& returnArg)
         &&
       ((uintptr_t)container.data() >= (uintptr_t)&container);
 
-    if (!inlined && pointers.add((uintptr_t)container.data())) {
+    if (!inlined && ctx.pointers.add((uintptr_t)container.data())) {
       SAVE_SIZE(container.capacity() * sizeof(T));
       SAVE_DATA(1);
     } else {
@@ -60,7 +60,7 @@ if (isStorageInline(container)) {
   category = Category::InlinedStorage;
 } else if (capacity < minLargeSize) {
   category = Category::OwnedHeapStorage;
-} else if (pointers.add(container.data())) {
+} else if (ctx.pointers.add(container.data())) {
   category = Category::ReferenceCountedStorage;
 } else {
   category = Category::AlreadyAttributed;
diff --git a/types/folly_iobuf_queue_type.toml b/types/folly_iobuf_queue_type.toml
index 0af134d..b9ef72c 100644
--- a/types/folly_iobuf_queue_type.toml
+++ b/types/folly_iobuf_queue_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1% &container, size_t& returnArg)
 
     const folly::IOBuf *head = container.front();
     SAVE_DATA((uintptr_t)head);
-    if (head && pointers.add((uintptr_t)head)) {
+    if (head && ctx.pointers.add((uintptr_t)head)) {
         SAVE_DATA(1);
         getSizeType(*head, returnArg);
     } else {
diff --git a/types/list_type.toml b/types/list_type.toml
index 86d457e..3e0da3c 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -38,8 +38,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index 887df6a..a09efff 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -42,12 +42,12 @@ traversal_func = '''
                     .write(container.size());
 
     for (const auto& kv : container) {
-      tail = tail.delegate([&kv](auto ret) {
-        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
-        auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
-          return OIInternal::getSizeType<Ctx>(kv.first, ret);
+      tail = tail.delegate([&ctx, &kv](auto ret) {
+        auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, kv.first, ret);
+        auto next = start.delegate([&ctx, &kv](typename TypeHandler<Ctx, T0>::type ret) {
+          return OIInternal::getSizeType<Ctx>(ctx, kv.first, ret);
         });
-        return OIInternal::getSizeType<Ctx>(kv.second, next);
+        return OIInternal::getSizeType<Ctx>(ctx, kv.second, next);
       });
     }
 
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index e05a1e3..a5d628f 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -39,11 +39,11 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto next = ret.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto next = ret.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/multi_set_type.toml b/types/multi_set_type.toml
index 00f8239..f238356 100644
--- a/types/multi_set_type.toml
+++ b/types/multi_set_type.toml
@@ -42,8 +42,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/optional_type.toml b/types/optional_type.toml
index 3bd9cff..eeba64e 100644
--- a/types/optional_type.toml
+++ b/types/optional_type.toml
@@ -30,8 +30,8 @@ void getSizeType(const %1%<T>& container, size_t& returnArg) {
 
 traversal_func = """
 if (container.has_value()) {
-  return returnArg.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return returnArg.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 } else {
   return returnArg.template delegate<0>(std::identity());
diff --git a/types/pair_type.toml b/types/pair_type.toml
index ba5bc38..f157144 100644
--- a/types/pair_type.toml
+++ b/types/pair_type.toml
@@ -27,10 +27,10 @@ void getSizeType(const %1%<P,Q> &container, size_t& returnArg)
 """
 
 traversal_func = """
-    return OIInternal::getSizeType<Ctx>(
+    return OIInternal::getSizeType<Ctx>(ctx, 
         container.second,
-        returnArg.delegate([&container](auto ret) {
-            return OIInternal::getSizeType<Ctx>(container.first, ret);
+        returnArg.delegate([&ctx, &container](auto ret) {
+            return OIInternal::getSizeType<Ctx>(ctx, container.first, ret);
         })
     );
 """
diff --git a/types/ref_wrapper_type.toml b/types/ref_wrapper_type.toml
index 949b385..9ed5094 100644
--- a/types/ref_wrapper_type.toml
+++ b/types/ref_wrapper_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &ref, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
     SAVE_DATA((uintptr_t)&(ref.get()));
-    if (pointers.add((uintptr_t)&ref.get())) {
+    if (ctx.pointers.add((uintptr_t)&ref.get())) {
         SAVE_DATA(1);
         getSizeType(ref.get(), returnArg);
     } else {
diff --git a/types/seq_type.toml b/types/seq_type.toml
index 88ddd1d..f561d0b 100644
--- a/types/seq_type.toml
+++ b/types/seq_type.toml
@@ -43,8 +43,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/set_type.toml b/types/set_type.toml
index e29611e..9d3bc3c 100644
--- a/types/set_type.toml
+++ b/types/set_type.toml
@@ -43,8 +43,8 @@ auto tail = returnArg.write((uintptr_t)&container)
 // The double ampersand is needed otherwise this loop doesn't work with
 // vector<bool>
 for (auto&& it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 125fd71..bb3a622 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -24,7 +24,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
     if constexpr (!std::is_void<T>::value) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
-        if (s_ptr && pointers.add((uintptr_t)(s_ptr.get()))) {
+        if (s_ptr && ctx.pointers.add((uintptr_t)(s_ptr.get()))) {
             SAVE_DATA(1);
             getSizeType(*(s_ptr.get()), returnArg);
         } else {
@@ -40,12 +40,12 @@ auto tail = returnArg.write((uintptr_t)container.get());
 if constexpr (std::is_void<T0>::value) {
   return tail.template delegate<0>(std::identity());
 } else {
-  bool do_visit = container && pointers.add((uintptr_t)container.get());
+  bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
   if (!do_visit)
     return tail.template delegate<0>(std::identity());
 
-  return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return tail.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 }
 """
diff --git a/types/small_vec_type.toml b/types/small_vec_type.toml
index d129285..a5f1694 100644
--- a/types/small_vec_type.toml
+++ b/types/small_vec_type.toml
@@ -56,8 +56,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (auto &&it: container) {
-  tail = tail.delegate([&it](typename TypeHandler<Ctx, T0>::type ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](typename TypeHandler<Ctx, T0>::type ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/sorted_vec_set_type.toml b/types/sorted_vec_set_type.toml
index e8b9483..a803d61 100644
--- a/types/sorted_vec_set_type.toml
+++ b/types/sorted_vec_set_type.toml
@@ -37,8 +37,8 @@ auto tail = returnArg.write((uintptr_t)&container)
                 .write(container.size());
 
 for (const auto& el : container) {
-  tail = tail.delegate([&el](auto ret) {
-    return OIInternal::getSizeType<Ctx>(el, ret);
+  tail = tail.delegate([&ctx, &el](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, el, ret);
   });
 }
 
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index df06cc7..ee652ec 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -43,12 +43,12 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &entry: container) {
-  tail = tail.delegate([&key = entry.first, &value = entry.second](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(key, ret);
-    auto next =  start.delegate([&key](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(key, ret);
+  tail = tail.delegate([&ctx, &key = entry.first, &value = entry.second](auto ret) {
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, key, ret);
+    auto next =  start.delegate([&ctx, &key](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, key, ret);
     });
-    return OIInternal::getSizeType<Ctx>(value, next);
+    return OIInternal::getSizeType<Ctx>(ctx, value, next);
   });
 }
 
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index 27142a1..5b2402f 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -46,12 +46,12 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto& kv : container) {
-  tail = tail.delegate([&kv](auto ret) {
-    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(kv.first, ret);
-    auto next = start.delegate([&kv](typename TypeHandler<Ctx, T0>::type ret) {
-      return OIInternal::getSizeType<Ctx>(kv.first, ret);
+  tail = tail.delegate([&ctx, &kv](auto ret) {
+    auto start = maybeCaptureKey<captureKeys, Ctx, T0>(ctx, kv.first, ret);
+    auto next = start.delegate([&ctx, &kv](typename TypeHandler<Ctx, T0>::type ret) {
+      return OIInternal::getSizeType<Ctx>(ctx, kv.first, ret);
     });
-    return OIInternal::getSizeType<Ctx>(kv.second, next);
+    return OIInternal::getSizeType<Ctx>(ctx, kv.second, next);
   });
 }
 
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index 261a0be..a6f7671 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -46,8 +46,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index 58364cf..f723f80 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -25,7 +25,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
     if constexpr (!std::is_void<T>::value) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
-        if (u_ptr && pointers.add((uintptr_t)(u_ptr.get()))) {
+        if (u_ptr && ctx.pointers.add((uintptr_t)(u_ptr.get()))) {
             SAVE_DATA(1);
             getSizeType(*(u_ptr.get()), returnArg);
         } else {
@@ -41,12 +41,12 @@ auto tail = returnArg.write((uintptr_t)container.get());
 if constexpr (std::is_void<T0>::value) {
   return tail.template delegate<0>(std::identity());
 } else {
-  bool do_visit = container && pointers.add((uintptr_t)container.get());
+  bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
   if (!do_visit)
     return tail.template delegate<0>(std::identity());
 
-  return tail.template delegate<1>([&container](auto ret) {
-    return OIInternal::getSizeType<Ctx>(*container, ret);
+  return tail.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, *container, ret);
   });
 }
 """
diff --git a/types/unordered_multiset_type.toml b/types/unordered_multiset_type.toml
index 5c2e96b..bfcf203 100644
--- a/types/unordered_multiset_type.toml
+++ b/types/unordered_multiset_type.toml
@@ -44,8 +44,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 
diff --git a/types/unordered_set_type.toml b/types/unordered_set_type.toml
index f8d26c9..d9a827a 100644
--- a/types/unordered_set_type.toml
+++ b/types/unordered_set_type.toml
@@ -44,8 +44,8 @@ auto tail = returnArg
   .write(container.size());
 
 for (const auto &it : container) {
-  tail = tail.delegate([&it](auto ret) {
-    return OIInternal::getSizeType<Ctx>(it, ret);
+  tail = tail.delegate([&ctx, &it](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, it, ret);
   });
 }
 

From 390d30be54af16de7b968d0197e2b9bd29e8c996 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 16 Nov 2023 05:02:37 -0800
Subject: [PATCH 034/188] incomplete: name type in compiler errors

Summary:

We have a good type representation in the Type Graph of an incomplete type and
the underlying type that represents. However, this incomplete type still ends
up in the generated code as `void` which loses information. For example, a
container that can't contain void may fail to compile because it was
initialised with `void` but really its because the type it was supposed to be
initialised with (say, `Foo`) had incomplete debug information.

This change identifies that a type is incomplete in the output by generating it
as an incomplete type `struct Incomplete<struct Foo>`. This allows us to name
the type correctly in the TreeBuilder output and filter for incomplete types,
as well as getting appropriate compiler errors if it mustn't be incomplete.

Test Plan:

TODO
---
 oi/CodeGen.cpp                      |  9 ++++++
 oi/OITraceCode.cpp                  | 15 ++++++++++
 oi/type_graph/NameGen.cpp           |  5 ++++
 oi/type_graph/NameGen.h             |  1 +
 oi/type_graph/Printer.cpp           |  4 ++-
 oi/type_graph/TopoSorter.cpp        |  4 +++
 oi/type_graph/TopoSorter.h          |  1 +
 oi/type_graph/Types.cpp             |  2 --
 oi/type_graph/Types.h               | 45 ++++++++++++++++++++++++-----
 test/TypeGraphParser.cpp            |  5 ++--
 test/test_drgn_parser.cpp           |  4 +--
 test/test_enforce_compatibility.cpp |  4 +--
 test/test_flattener.cpp             |  2 +-
 test/test_name_gen.cpp              | 13 +++++++++
 types/shrd_ptr_type.toml            |  8 ++---
 types/uniq_ptr_type.toml            |  8 ++---
 16 files changed, 105 insertions(+), 25 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index cfa87e7..8cdbcd4 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -57,6 +57,7 @@ using type_graph::EnforceCompatibility;
 using type_graph::Enum;
 using type_graph::Flattener;
 using type_graph::IdentifyContainers;
+using type_graph::Incomplete;
 using type_graph::KeyCapture;
 using type_graph::Member;
 using type_graph::NameGen;
@@ -177,12 +178,20 @@ void genDeclsEnum(const Enum& e, std::string& code) {
   code += " {};\n";
 }
 
+void genDeclsIncomplete(const Incomplete& i, std::string& code) {
+  code += "template<> struct ";
+  code += i.name();
+  code += ";\n";
+}
+
 void genDecls(const TypeGraph& typeGraph, std::string& code) {
   for (const Type& t : typeGraph.finalTypes) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
       genDeclsClass(*c, code);
     } else if (const auto* e = dynamic_cast<const Enum*>(&t)) {
       genDeclsEnum(*e, code);
+    } else if (const auto* i = dynamic_cast<const Incomplete*>(&t)) {
+      genDeclsIncomplete(*i, code);
     }
   }
 }
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index 5cc443d..eed8fff 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -181,3 +181,18 @@ bool isStorageInline(const auto& c) {
   return (uintptr_t)std::data(c) < (uintptr_t)(&c + sizeof(c)) &&
          (uintptr_t)std::data(c) >= (uintptr_t)&c;
 }
+
+namespace OIInternal {
+namespace {
+
+template <typename T>
+struct Incomplete;
+
+template <typename T>
+constexpr bool oi_is_complete = true;
+
+template <typename T>
+constexpr bool oi_is_complete<Incomplete<T>> = false;
+
+}  // namespace
+}  // namespace OIInternal
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index c792766..115c06c 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -204,4 +204,9 @@ void NameGen::visit(CaptureKeys& c) {
   c.regenerateName();
 }
 
+void NameGen::visit(Incomplete& i) {
+  RecursiveVisitor::visit(i);
+  i.regenerateName();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..4857ded 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -48,6 +48,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
+  void visit(Incomplete& i) override;
 
   static const inline std::string AnonPrefix = "__oi_anon";
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index e7d70eb..fc97905 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -37,7 +37,9 @@ void Printer::print(const Type& type) {
 }
 
 void Printer::visit(const Incomplete& i) {
-  prefix();
+  if (prefix(i))
+    return;
+
   out_ << "Incomplete";
   if (auto underlyingType = i.underlyingType()) {
     out_ << std::endl;
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index 6ca10af..ff2715a 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -139,6 +139,10 @@ void TopoSorter::visit(CaptureKeys& c) {
   sortedTypes_.push_back(c);
 }
 
+void TopoSorter::visit(Incomplete& i) {
+  sortedTypes_.push_back(i);
+}
+
 /*
  * A type graph may contain cycles, so we need to slightly tweak the standard
  * topological sorting algorithm. Cycles can only be introduced by certain
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 374e943..44f2364 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -48,6 +48,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
+  void visit(Incomplete& i) override;
 
  private:
   std::unordered_set<Type*> visited_;
diff --git a/oi/type_graph/Types.cpp b/oi/type_graph/Types.cpp
index 8be091f..af39606 100644
--- a/oi/type_graph/Types.cpp
+++ b/oi/type_graph/Types.cpp
@@ -32,8 +32,6 @@ namespace oi::detail::type_graph {
 OI_TYPE_LIST
 #undef X
 
-const std::string Incomplete::kName = "void";
-
 std::string Primitive::getName(Kind kind) {
   switch (kind) {
     case Kind::Int8:
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f503886..25d1d03 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -29,6 +29,7 @@
  * debugging.
  */
 
+#include <algorithm>
 #include <cstddef>
 #include <map>
 #include <optional>
@@ -208,19 +209,20 @@ struct TemplateParam {
  */
 class Incomplete : public Type {
  public:
-  Incomplete(Type& underlyingType) : underlyingType_(underlyingType) {
+  Incomplete(NodeId id, Type& underlyingType)
+      : id_(id), underlyingType_(underlyingType) {
   }
 
-  Incomplete(std::string underlyingTypeName)
-      : underlyingType_(std::move(underlyingTypeName)) {
+  Incomplete(NodeId id, std::string underlyingTypeName)
+      : id_(id), underlyingType_(std::move(underlyingTypeName)) {
   }
 
-  static inline constexpr bool has_node_id = false;
+  static inline constexpr bool has_node_id = true;
 
   DECLARE_ACCEPT
 
   const std::string& name() const override {
-    return kName;
+    return name_;
   }
 
   std::string_view inputName() const override {
@@ -242,7 +244,35 @@ class Incomplete : public Type {
   }
 
   NodeId id() const override {
-    return -1;
+    return id_;
+  }
+
+  void regenerateName() {
+    std::string_view subName{std::visit(
+        [](const auto& el) -> std::string_view {
+          using T = std::decay_t<decltype(el)>;
+          if constexpr (std::is_same_v<T, std::string>) {
+            return el;
+          } else {
+            return el.get().name();
+          }
+        },
+        underlyingType_)};
+
+    constexpr std::string_view kPrefix{"Incomplete<struct "};
+    name_.clear();
+    name_.reserve(kPrefix.size() + subName.size() + 2);
+
+    name_ = kPrefix;
+    for (const auto c : subName) {
+      if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') ||
+          (c >= 'A' && c <= 'Z')) {
+        name_ += c;
+      } else {
+        name_ += '_';
+      }
+    }
+    name_ += ">";
   }
 
   std::optional<std::reference_wrapper<Type>> underlyingType() const {
@@ -254,8 +284,9 @@ class Incomplete : public Type {
   }
 
  private:
+  NodeId id_ = -1;
   std::variant<std::string, std::reference_wrapper<Type>> underlyingType_;
-  static const std::string kName;
+  std::string name_;
 };
 
 /*
diff --git a/test/TypeGraphParser.cpp b/test/TypeGraphParser.cpp
index 7b6f425..388965a 100644
--- a/test/TypeGraphParser.cpp
+++ b/test/TypeGraphParser.cpp
@@ -215,10 +215,11 @@ Type& TypeGraphParser::parseType(std::string_view& input, size_t rootIndent) {
       auto nameEndPos = line.find(']', nameStartPos);
       auto underlyingTypeName =
           line.substr(nameStartPos, nameEndPos - nameStartPos);
-      type = &typeGraph_.makeType<Incomplete>(std::string(underlyingTypeName));
+      type =
+          &typeGraph_.makeType<Incomplete>(id, std::string(underlyingTypeName));
     } else {
       auto& underlyingType = parseType(input, indent + 2);
-      type = &typeGraph_.makeType<Incomplete>(underlyingType);
+      type = &typeGraph_.makeType<Incomplete>(id, underlyingType);
     }
   } else if (nodeTypeName == "Class" || nodeTypeName == "Struct" ||
              nodeTypeName == "Union") {
diff --git a/test/test_drgn_parser.cpp b/test/test_drgn_parser.cpp
index a5ebe30..0d09b31 100644
--- a/test/test_drgn_parser.cpp
+++ b/test/test_drgn_parser.cpp
@@ -426,8 +426,8 @@ TEST_F(DrgnParserTest, PointerNoFollow) {
 
 TEST_F(DrgnParserTest, PointerIncomplete) {
   test("oid_test_case_pointers_incomplete_raw", R"(
-[0] Pointer
-      Incomplete: [IncompleteType]
+[1] Pointer
+[0]   Incomplete: [IncompleteType]
 )");
 }
 
diff --git a/test/test_enforce_compatibility.cpp b/test/test_enforce_compatibility.cpp
index 2632823..d91f356 100644
--- a/test/test_enforce_compatibility.cpp
+++ b/test/test_enforce_compatibility.cpp
@@ -38,8 +38,8 @@ TEST(EnforceCompatibilityTest, VoidPointer) {
        R"(
 [0] Class: MyClass (size: 8)
       Member: p (offset: 0)
-[1]     Pointer
-          Incomplete
+[2]     Pointer
+[1]       Incomplete
             Primitive: void
 )",
        R"(
diff --git a/test/test_flattener.cpp b/test/test_flattener.cpp
index eab3cf8..9f0c0aa 100644
--- a/test/test_flattener.cpp
+++ b/test/test_flattener.cpp
@@ -986,7 +986,7 @@ TEST(FlattenerTest, IncompleteParent) {
        R"(
 [0] Class: MyClass (size: 4)
       Parent (offset: 0)
-        Incomplete: [IncompleteParent]
+[1]     Incomplete: [IncompleteParent]
 )",
        R"(
 [0] Class: MyClass (size: 4)
diff --git a/test/test_name_gen.cpp b/test/test_name_gen.cpp
index 72177d7..d721f7d 100644
--- a/test/test_name_gen.cpp
+++ b/test/test_name_gen.cpp
@@ -475,3 +475,16 @@ TEST(NameGenTest, AnonymousTypes) {
   EXPECT_EQ(myenum.inputName(), "__oi_anon_1");
   EXPECT_EQ(mytypedef.inputName(), "");
 }
+
+TEST(NameGenTest, IncompleteTypes) {
+  auto myincompletevector = Incomplete{0, "std::vector<int>"};
+
+  auto myint = Primitive{Primitive::Kind::Int32};
+  auto myincompleteint = Incomplete{1, myint};
+
+  NameGen nameGen;
+  nameGen.generateNames({myincompletevector, myincompleteint});
+
+  EXPECT_EQ(myincompletevector.name(), "Incomplete<struct std__vector_int_>");
+  EXPECT_EQ(myincompleteint.name(), "Incomplete<struct int32_t>");
+}
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 125fd71..4372c76 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (!std::is_void<T>::value && oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
         if (s_ptr && pointers.add((uintptr_t)(s_ptr.get()))) {
@@ -37,7 +37,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (std::is_void<T0>::value || !oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && pointers.add((uintptr_t)container.get());
@@ -84,8 +84,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = sum.index, // 0 for empty containers/void, 1 otherwise
 });
 
-// Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+// Must be in a `if constexpr` or the compiler will complain about make_field<DB, void>
+if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index 58364cf..fde0adc 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T,Deleter>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (!std::is_void<T>::value && oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
         if (u_ptr && pointers.add((uintptr_t)(u_ptr.get()))) {
@@ -38,7 +38,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (std::is_void<T0>::value || !oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && pointers.add((uintptr_t)container.get());
@@ -70,8 +70,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = sum.index, // 0 for empty containers/void, 1 otherwise
 });
 
-// Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+// Must be in a `if constexpr` or the compiler will complain about make_field<DB, void>
+if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);

From 44afbc581d2ba103c9d63aff4b0a1cd302d00e5c Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 16 Nov 2023 07:24:03 -0800
Subject: [PATCH 035/188] incomplete: name type in compiler errors

Summary:

We have a good type representation in the Type Graph of an incomplete type and
the underlying type that represents. However, this incomplete type still ends
up in the generated code as `void` which loses information. For example, a
container that can't contain void may fail to compile because it was
initialised with `void` but really its because the type it was supposed to be
initialised with (say, `Foo`) had incomplete debug information.

This change identifies that a type is incomplete in the output by generating it
as an incomplete type `struct Incomplete<struct Foo>`. This allows us to name
the type correctly in the TreeBuilder output and filter for incomplete types,
as well as getting appropriate compiler errors if it mustn't be incomplete.

Test Plan:
- CI
- Added a unit test to namegen.
- Enabled and added an extra pointers_incomplete test.

This change is tricky to test because it isn't really user visible. The types
still use their `inputName` which is unchanged in any successful output - this
change is used so the compiler fails with a more detailed error.
---
 oi/CodeGen.cpp                            |  9 ++++
 oi/OITraceCode.cpp                        | 15 +++++++
 oi/type_graph/NameGen.cpp                 |  5 +++
 oi/type_graph/NameGen.h                   |  1 +
 oi/type_graph/Printer.cpp                 |  4 +-
 oi/type_graph/TopoSorter.cpp              |  4 ++
 oi/type_graph/TopoSorter.h                |  1 +
 oi/type_graph/Types.cpp                   |  2 -
 oi/type_graph/Types.h                     | 53 +++++++++++++++++------
 test/TypeGraphParser.cpp                  |  5 ++-
 test/integration/pointers_incomplete.toml | 32 ++++++++++++--
 test/test_drgn_parser.cpp                 |  4 +-
 test/test_enforce_compatibility.cpp       |  4 +-
 test/test_flattener.cpp                   |  2 +-
 test/test_name_gen.cpp                    | 13 ++++++
 types/shrd_ptr_type.toml                  |  8 ++--
 types/uniq_ptr_type.toml                  |  8 ++--
 17 files changed, 134 insertions(+), 36 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index cfa87e7..8cdbcd4 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -57,6 +57,7 @@ using type_graph::EnforceCompatibility;
 using type_graph::Enum;
 using type_graph::Flattener;
 using type_graph::IdentifyContainers;
+using type_graph::Incomplete;
 using type_graph::KeyCapture;
 using type_graph::Member;
 using type_graph::NameGen;
@@ -177,12 +178,20 @@ void genDeclsEnum(const Enum& e, std::string& code) {
   code += " {};\n";
 }
 
+void genDeclsIncomplete(const Incomplete& i, std::string& code) {
+  code += "template<> struct ";
+  code += i.name();
+  code += ";\n";
+}
+
 void genDecls(const TypeGraph& typeGraph, std::string& code) {
   for (const Type& t : typeGraph.finalTypes) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
       genDeclsClass(*c, code);
     } else if (const auto* e = dynamic_cast<const Enum*>(&t)) {
       genDeclsEnum(*e, code);
+    } else if (const auto* i = dynamic_cast<const Incomplete*>(&t)) {
+      genDeclsIncomplete(*i, code);
     }
   }
 }
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index 5cc443d..eed8fff 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -181,3 +181,18 @@ bool isStorageInline(const auto& c) {
   return (uintptr_t)std::data(c) < (uintptr_t)(&c + sizeof(c)) &&
          (uintptr_t)std::data(c) >= (uintptr_t)&c;
 }
+
+namespace OIInternal {
+namespace {
+
+template <typename T>
+struct Incomplete;
+
+template <typename T>
+constexpr bool oi_is_complete = true;
+
+template <typename T>
+constexpr bool oi_is_complete<Incomplete<T>> = false;
+
+}  // namespace
+}  // namespace OIInternal
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index c792766..115c06c 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -204,4 +204,9 @@ void NameGen::visit(CaptureKeys& c) {
   c.regenerateName();
 }
 
+void NameGen::visit(Incomplete& i) {
+  RecursiveVisitor::visit(i);
+  i.regenerateName();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..4857ded 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -48,6 +48,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
+  void visit(Incomplete& i) override;
 
   static const inline std::string AnonPrefix = "__oi_anon";
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index e7d70eb..fc97905 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -37,7 +37,9 @@ void Printer::print(const Type& type) {
 }
 
 void Printer::visit(const Incomplete& i) {
-  prefix();
+  if (prefix(i))
+    return;
+
   out_ << "Incomplete";
   if (auto underlyingType = i.underlyingType()) {
     out_ << std::endl;
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index 6ca10af..ff2715a 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -139,6 +139,10 @@ void TopoSorter::visit(CaptureKeys& c) {
   sortedTypes_.push_back(c);
 }
 
+void TopoSorter::visit(Incomplete& i) {
+  sortedTypes_.push_back(i);
+}
+
 /*
  * A type graph may contain cycles, so we need to slightly tweak the standard
  * topological sorting algorithm. Cycles can only be introduced by certain
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 374e943..44f2364 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -48,6 +48,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
+  void visit(Incomplete& i) override;
 
  private:
   std::unordered_set<Type*> visited_;
diff --git a/oi/type_graph/Types.cpp b/oi/type_graph/Types.cpp
index 8be091f..af39606 100644
--- a/oi/type_graph/Types.cpp
+++ b/oi/type_graph/Types.cpp
@@ -32,8 +32,6 @@ namespace oi::detail::type_graph {
 OI_TYPE_LIST
 #undef X
 
-const std::string Incomplete::kName = "void";
-
 std::string Primitive::getName(Kind kind) {
   switch (kind) {
     case Kind::Int8:
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f503886..46d5ef5 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -29,6 +29,7 @@
  * debugging.
  */
 
+#include <algorithm>
 #include <cstddef>
 #include <map>
 #include <optional>
@@ -208,29 +209,33 @@ struct TemplateParam {
  */
 class Incomplete : public Type {
  public:
-  Incomplete(Type& underlyingType) : underlyingType_(underlyingType) {
+  Incomplete(NodeId id, Type& underlyingType)
+      : id_(id), underlyingType_(underlyingType) {
   }
 
-  Incomplete(std::string underlyingTypeName)
-      : underlyingType_(std::move(underlyingTypeName)) {
+  Incomplete(NodeId id, std::string underlyingTypeName)
+      : id_(id), underlyingType_(std::move(underlyingTypeName)) {
   }
 
-  static inline constexpr bool has_node_id = false;
+  static inline constexpr bool has_node_id = true;
 
   DECLARE_ACCEPT
 
   const std::string& name() const override {
-    return kName;
+    return name_;
   }
 
   std::string_view inputName() const override {
-    if (std::holds_alternative<std::string>(underlyingType_)) {
-      return std::get<std::string>(underlyingType_);
-    }
-
-    return std::get<std::reference_wrapper<Type>>(underlyingType_)
-        .get()
-        .inputName();
+    return std::visit(
+        [](const auto& el) -> std::string_view {
+          using T = std::decay_t<decltype(el)>;
+          if constexpr (std::is_same_v<T, std::string>) {
+            return el;
+          } else {
+            return el.get().inputName();
+          }
+        },
+        underlyingType_);
   }
 
   size_t size() const override {
@@ -242,7 +247,26 @@ class Incomplete : public Type {
   }
 
   NodeId id() const override {
-    return -1;
+    return id_;
+  }
+
+  void regenerateName() {
+    std::string_view inputName = this->inputName();
+
+    constexpr std::string_view kPrefix{"Incomplete<struct "};
+    name_.clear();
+    name_.reserve(kPrefix.size() + inputName.size() + 2);
+
+    name_ = kPrefix;
+    for (const auto c : inputName) {
+      if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') ||
+          (c >= 'A' && c <= 'Z')) {
+        name_ += c;
+      } else {
+        name_ += '_';
+      }
+    }
+    name_ += ">";
   }
 
   std::optional<std::reference_wrapper<Type>> underlyingType() const {
@@ -254,8 +278,9 @@ class Incomplete : public Type {
   }
 
  private:
+  NodeId id_ = -1;
   std::variant<std::string, std::reference_wrapper<Type>> underlyingType_;
-  static const std::string kName;
+  std::string name_;
 };
 
 /*
diff --git a/test/TypeGraphParser.cpp b/test/TypeGraphParser.cpp
index 7b6f425..388965a 100644
--- a/test/TypeGraphParser.cpp
+++ b/test/TypeGraphParser.cpp
@@ -215,10 +215,11 @@ Type& TypeGraphParser::parseType(std::string_view& input, size_t rootIndent) {
       auto nameEndPos = line.find(']', nameStartPos);
       auto underlyingTypeName =
           line.substr(nameStartPos, nameEndPos - nameStartPos);
-      type = &typeGraph_.makeType<Incomplete>(std::string(underlyingTypeName));
+      type =
+          &typeGraph_.makeType<Incomplete>(id, std::string(underlyingTypeName));
     } else {
       auto& underlyingType = parseType(input, indent + 2);
-      type = &typeGraph_.makeType<Incomplete>(underlyingType);
+      type = &typeGraph_.makeType<Incomplete>(id, underlyingType);
     }
   } else if (nodeTypeName == "Class" || nodeTypeName == "Struct" ||
              nodeTypeName == "Union") {
diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml
index 437041c..aeebb24 100644
--- a/test/integration/pointers_incomplete.toml
+++ b/test/integration/pointers_incomplete.toml
@@ -55,7 +55,6 @@ definitions = '''
     }]'''
 
   [cases.unique_ptr]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299
     param_types = ["const std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>&"]
     setup = '''
       auto raw_ptr = static_cast<IncompleteType*>(::operator new(5));
@@ -63,28 +62,29 @@ definitions = '''
         raw_ptr, &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
   [cases.unique_ptr_null]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299
     param_types = ["const std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>&"]
     setup = '''
       return std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>(
         nullptr, &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.shared_ptr]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300
     param_types = ["const std::shared_ptr<IncompleteType>&"]
     setup = '''
       auto raw_ptr = static_cast<IncompleteType*>(::operator new(5));
       return std::shared_ptr<IncompleteType>(raw_ptr , &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
   [cases.shared_ptr_null]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300
     param_types = ["const std::shared_ptr<IncompleteType>"]
     setup = "return nullptr;"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.containing_struct]
     oil_disable = "oil can't chase raw pointers safely"
@@ -117,3 +117,27 @@ definitions = '''
         }
       ]
     }]'''
+
+  [cases.containing_struct_no_follow]
+    param_types = ["const IncompleteTypeContainer&"]
+    setup = "return IncompleteTypeContainer{};"
+    expect_json = '''[{
+      "staticSize": 88,
+      "members": [
+        { "name": "ptrundef", "staticSize": 8 },
+        { "name": "__makePad1", "staticSize": 1 },
+        { "name": "shundef", "staticSize": 16 },
+        { "name": "__makePad2", "staticSize": 1 },
+        { "name": "shoptundef",
+          "staticSize": 24,
+          "length": 0,
+          "capacity": 1
+        },
+        { "name": "__makePad3", "staticSize": 1 },
+        { "name": "optundef",
+          "staticSize": 16,
+          "length": 0,
+          "capacity": 1
+        }
+      ]
+    }]'''
diff --git a/test/test_drgn_parser.cpp b/test/test_drgn_parser.cpp
index a5ebe30..0d09b31 100644
--- a/test/test_drgn_parser.cpp
+++ b/test/test_drgn_parser.cpp
@@ -426,8 +426,8 @@ TEST_F(DrgnParserTest, PointerNoFollow) {
 
 TEST_F(DrgnParserTest, PointerIncomplete) {
   test("oid_test_case_pointers_incomplete_raw", R"(
-[0] Pointer
-      Incomplete: [IncompleteType]
+[1] Pointer
+[0]   Incomplete: [IncompleteType]
 )");
 }
 
diff --git a/test/test_enforce_compatibility.cpp b/test/test_enforce_compatibility.cpp
index 2632823..d91f356 100644
--- a/test/test_enforce_compatibility.cpp
+++ b/test/test_enforce_compatibility.cpp
@@ -38,8 +38,8 @@ TEST(EnforceCompatibilityTest, VoidPointer) {
        R"(
 [0] Class: MyClass (size: 8)
       Member: p (offset: 0)
-[1]     Pointer
-          Incomplete
+[2]     Pointer
+[1]       Incomplete
             Primitive: void
 )",
        R"(
diff --git a/test/test_flattener.cpp b/test/test_flattener.cpp
index eab3cf8..9f0c0aa 100644
--- a/test/test_flattener.cpp
+++ b/test/test_flattener.cpp
@@ -986,7 +986,7 @@ TEST(FlattenerTest, IncompleteParent) {
        R"(
 [0] Class: MyClass (size: 4)
       Parent (offset: 0)
-        Incomplete: [IncompleteParent]
+[1]     Incomplete: [IncompleteParent]
 )",
        R"(
 [0] Class: MyClass (size: 4)
diff --git a/test/test_name_gen.cpp b/test/test_name_gen.cpp
index 72177d7..d721f7d 100644
--- a/test/test_name_gen.cpp
+++ b/test/test_name_gen.cpp
@@ -475,3 +475,16 @@ TEST(NameGenTest, AnonymousTypes) {
   EXPECT_EQ(myenum.inputName(), "__oi_anon_1");
   EXPECT_EQ(mytypedef.inputName(), "");
 }
+
+TEST(NameGenTest, IncompleteTypes) {
+  auto myincompletevector = Incomplete{0, "std::vector<int>"};
+
+  auto myint = Primitive{Primitive::Kind::Int32};
+  auto myincompleteint = Incomplete{1, myint};
+
+  NameGen nameGen;
+  nameGen.generateNames({myincompletevector, myincompleteint});
+
+  EXPECT_EQ(myincompletevector.name(), "Incomplete<struct std__vector_int_>");
+  EXPECT_EQ(myincompleteint.name(), "Incomplete<struct int32_t>");
+}
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 125fd71..4372c76 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (!std::is_void<T>::value && oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
         if (s_ptr && pointers.add((uintptr_t)(s_ptr.get()))) {
@@ -37,7 +37,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (std::is_void<T0>::value || !oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && pointers.add((uintptr_t)container.get());
@@ -84,8 +84,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = sum.index, // 0 for empty containers/void, 1 otherwise
 });
 
-// Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+// Must be in a `if constexpr` or the compiler will complain about make_field<DB, void>
+if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index 58364cf..fde0adc 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T,Deleter>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (!std::is_void<T>::value && oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
         if (u_ptr && pointers.add((uintptr_t)(u_ptr.get()))) {
@@ -38,7 +38,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (std::is_void<T0>::value || !oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && pointers.add((uintptr_t)container.get());
@@ -70,8 +70,8 @@ el.container_stats.emplace(result::Element::ContainerStats {
   .length = sum.index, // 0 for empty containers/void, 1 otherwise
 });
 
-// Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+// Must be in a `if constexpr` or the compiler will complain about make_field<DB, void>
+if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);

From b5cdc2543a1e4321f66c3890b56cf7d90da0b0f3 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 16 Nov 2023 07:24:03 -0800
Subject: [PATCH 036/188] incomplete: name type in compiler errors

Summary:

We have a good type representation in the Type Graph of an incomplete type and
the underlying type that represents. However, this incomplete type still ends
up in the generated code as `void` which loses information. For example, a
container that can't contain void may fail to compile because it was
initialised with `void` but really its because the type it was supposed to be
initialised with (say, `Foo`) had incomplete debug information.

This change identifies that a type is incomplete in the output by generating it
as an incomplete type `struct Incomplete<struct Foo>`. This allows us to name
the type correctly in the TreeBuilder output and filter for incomplete types,
as well as getting appropriate compiler errors if it mustn't be incomplete.

Test Plan:
- CI
- Added a unit test to namegen.
- Enabled and added an extra pointers_incomplete test.

This change is tricky to test because it isn't really user visible. The types
still use their `inputName` which is unchanged in any successful output - this
change is used so the compiler fails with a more detailed error.
---
 oi/CodeGen.cpp                            |  9 ++++
 oi/OITraceCode.cpp                        | 15 +++++++
 oi/type_graph/NameGen.cpp                 |  5 +++
 oi/type_graph/NameGen.h                   |  1 +
 oi/type_graph/Printer.cpp                 |  4 +-
 oi/type_graph/TopoSorter.cpp              |  4 ++
 oi/type_graph/TopoSorter.h                |  1 +
 oi/type_graph/Types.cpp                   |  2 -
 oi/type_graph/Types.h                     | 53 +++++++++++++++++------
 test/TypeGraphParser.cpp                  |  5 ++-
 test/integration/pointers_incomplete.toml | 32 ++++++++++++--
 test/test_drgn_parser.cpp                 |  4 +-
 test/test_enforce_compatibility.cpp       |  4 +-
 test/test_flattener.cpp                   |  2 +-
 test/test_name_gen.cpp                    | 13 ++++++
 types/shrd_ptr_type.toml                  |  6 +--
 types/uniq_ptr_type.toml                  |  6 +--
 17 files changed, 132 insertions(+), 34 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index cfa87e7..8cdbcd4 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -57,6 +57,7 @@ using type_graph::EnforceCompatibility;
 using type_graph::Enum;
 using type_graph::Flattener;
 using type_graph::IdentifyContainers;
+using type_graph::Incomplete;
 using type_graph::KeyCapture;
 using type_graph::Member;
 using type_graph::NameGen;
@@ -177,12 +178,20 @@ void genDeclsEnum(const Enum& e, std::string& code) {
   code += " {};\n";
 }
 
+void genDeclsIncomplete(const Incomplete& i, std::string& code) {
+  code += "template<> struct ";
+  code += i.name();
+  code += ";\n";
+}
+
 void genDecls(const TypeGraph& typeGraph, std::string& code) {
   for (const Type& t : typeGraph.finalTypes) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
       genDeclsClass(*c, code);
     } else if (const auto* e = dynamic_cast<const Enum*>(&t)) {
       genDeclsEnum(*e, code);
+    } else if (const auto* i = dynamic_cast<const Incomplete*>(&t)) {
+      genDeclsIncomplete(*i, code);
     }
   }
 }
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index 5cc443d..eed8fff 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -181,3 +181,18 @@ bool isStorageInline(const auto& c) {
   return (uintptr_t)std::data(c) < (uintptr_t)(&c + sizeof(c)) &&
          (uintptr_t)std::data(c) >= (uintptr_t)&c;
 }
+
+namespace OIInternal {
+namespace {
+
+template <typename T>
+struct Incomplete;
+
+template <typename T>
+constexpr bool oi_is_complete = true;
+
+template <typename T>
+constexpr bool oi_is_complete<Incomplete<T>> = false;
+
+}  // namespace
+}  // namespace OIInternal
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index c792766..115c06c 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -204,4 +204,9 @@ void NameGen::visit(CaptureKeys& c) {
   c.regenerateName();
 }
 
+void NameGen::visit(Incomplete& i) {
+  RecursiveVisitor::visit(i);
+  i.regenerateName();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..4857ded 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -48,6 +48,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
+  void visit(Incomplete& i) override;
 
   static const inline std::string AnonPrefix = "__oi_anon";
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index e7d70eb..fc97905 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -37,7 +37,9 @@ void Printer::print(const Type& type) {
 }
 
 void Printer::visit(const Incomplete& i) {
-  prefix();
+  if (prefix(i))
+    return;
+
   out_ << "Incomplete";
   if (auto underlyingType = i.underlyingType()) {
     out_ << std::endl;
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index 6ca10af..ff2715a 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -139,6 +139,10 @@ void TopoSorter::visit(CaptureKeys& c) {
   sortedTypes_.push_back(c);
 }
 
+void TopoSorter::visit(Incomplete& i) {
+  sortedTypes_.push_back(i);
+}
+
 /*
  * A type graph may contain cycles, so we need to slightly tweak the standard
  * topological sorting algorithm. Cycles can only be introduced by certain
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 374e943..44f2364 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -48,6 +48,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
+  void visit(Incomplete& i) override;
 
  private:
   std::unordered_set<Type*> visited_;
diff --git a/oi/type_graph/Types.cpp b/oi/type_graph/Types.cpp
index 8be091f..af39606 100644
--- a/oi/type_graph/Types.cpp
+++ b/oi/type_graph/Types.cpp
@@ -32,8 +32,6 @@ namespace oi::detail::type_graph {
 OI_TYPE_LIST
 #undef X
 
-const std::string Incomplete::kName = "void";
-
 std::string Primitive::getName(Kind kind) {
   switch (kind) {
     case Kind::Int8:
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f503886..46d5ef5 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -29,6 +29,7 @@
  * debugging.
  */
 
+#include <algorithm>
 #include <cstddef>
 #include <map>
 #include <optional>
@@ -208,29 +209,33 @@ struct TemplateParam {
  */
 class Incomplete : public Type {
  public:
-  Incomplete(Type& underlyingType) : underlyingType_(underlyingType) {
+  Incomplete(NodeId id, Type& underlyingType)
+      : id_(id), underlyingType_(underlyingType) {
   }
 
-  Incomplete(std::string underlyingTypeName)
-      : underlyingType_(std::move(underlyingTypeName)) {
+  Incomplete(NodeId id, std::string underlyingTypeName)
+      : id_(id), underlyingType_(std::move(underlyingTypeName)) {
   }
 
-  static inline constexpr bool has_node_id = false;
+  static inline constexpr bool has_node_id = true;
 
   DECLARE_ACCEPT
 
   const std::string& name() const override {
-    return kName;
+    return name_;
   }
 
   std::string_view inputName() const override {
-    if (std::holds_alternative<std::string>(underlyingType_)) {
-      return std::get<std::string>(underlyingType_);
-    }
-
-    return std::get<std::reference_wrapper<Type>>(underlyingType_)
-        .get()
-        .inputName();
+    return std::visit(
+        [](const auto& el) -> std::string_view {
+          using T = std::decay_t<decltype(el)>;
+          if constexpr (std::is_same_v<T, std::string>) {
+            return el;
+          } else {
+            return el.get().inputName();
+          }
+        },
+        underlyingType_);
   }
 
   size_t size() const override {
@@ -242,7 +247,26 @@ class Incomplete : public Type {
   }
 
   NodeId id() const override {
-    return -1;
+    return id_;
+  }
+
+  void regenerateName() {
+    std::string_view inputName = this->inputName();
+
+    constexpr std::string_view kPrefix{"Incomplete<struct "};
+    name_.clear();
+    name_.reserve(kPrefix.size() + inputName.size() + 2);
+
+    name_ = kPrefix;
+    for (const auto c : inputName) {
+      if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') ||
+          (c >= 'A' && c <= 'Z')) {
+        name_ += c;
+      } else {
+        name_ += '_';
+      }
+    }
+    name_ += ">";
   }
 
   std::optional<std::reference_wrapper<Type>> underlyingType() const {
@@ -254,8 +278,9 @@ class Incomplete : public Type {
   }
 
  private:
+  NodeId id_ = -1;
   std::variant<std::string, std::reference_wrapper<Type>> underlyingType_;
-  static const std::string kName;
+  std::string name_;
 };
 
 /*
diff --git a/test/TypeGraphParser.cpp b/test/TypeGraphParser.cpp
index 7b6f425..388965a 100644
--- a/test/TypeGraphParser.cpp
+++ b/test/TypeGraphParser.cpp
@@ -215,10 +215,11 @@ Type& TypeGraphParser::parseType(std::string_view& input, size_t rootIndent) {
       auto nameEndPos = line.find(']', nameStartPos);
       auto underlyingTypeName =
           line.substr(nameStartPos, nameEndPos - nameStartPos);
-      type = &typeGraph_.makeType<Incomplete>(std::string(underlyingTypeName));
+      type =
+          &typeGraph_.makeType<Incomplete>(id, std::string(underlyingTypeName));
     } else {
       auto& underlyingType = parseType(input, indent + 2);
-      type = &typeGraph_.makeType<Incomplete>(underlyingType);
+      type = &typeGraph_.makeType<Incomplete>(id, underlyingType);
     }
   } else if (nodeTypeName == "Class" || nodeTypeName == "Struct" ||
              nodeTypeName == "Union") {
diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml
index 437041c..aeebb24 100644
--- a/test/integration/pointers_incomplete.toml
+++ b/test/integration/pointers_incomplete.toml
@@ -55,7 +55,6 @@ definitions = '''
     }]'''
 
   [cases.unique_ptr]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299
     param_types = ["const std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>&"]
     setup = '''
       auto raw_ptr = static_cast<IncompleteType*>(::operator new(5));
@@ -63,28 +62,29 @@ definitions = '''
         raw_ptr, &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
   [cases.unique_ptr_null]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299
     param_types = ["const std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>&"]
     setup = '''
       return std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>(
         nullptr, &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.shared_ptr]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300
     param_types = ["const std::shared_ptr<IncompleteType>&"]
     setup = '''
       auto raw_ptr = static_cast<IncompleteType*>(::operator new(5));
       return std::shared_ptr<IncompleteType>(raw_ptr , &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
   [cases.shared_ptr_null]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300
     param_types = ["const std::shared_ptr<IncompleteType>"]
     setup = "return nullptr;"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.containing_struct]
     oil_disable = "oil can't chase raw pointers safely"
@@ -117,3 +117,27 @@ definitions = '''
         }
       ]
     }]'''
+
+  [cases.containing_struct_no_follow]
+    param_types = ["const IncompleteTypeContainer&"]
+    setup = "return IncompleteTypeContainer{};"
+    expect_json = '''[{
+      "staticSize": 88,
+      "members": [
+        { "name": "ptrundef", "staticSize": 8 },
+        { "name": "__makePad1", "staticSize": 1 },
+        { "name": "shundef", "staticSize": 16 },
+        { "name": "__makePad2", "staticSize": 1 },
+        { "name": "shoptundef",
+          "staticSize": 24,
+          "length": 0,
+          "capacity": 1
+        },
+        { "name": "__makePad3", "staticSize": 1 },
+        { "name": "optundef",
+          "staticSize": 16,
+          "length": 0,
+          "capacity": 1
+        }
+      ]
+    }]'''
diff --git a/test/test_drgn_parser.cpp b/test/test_drgn_parser.cpp
index a5ebe30..0d09b31 100644
--- a/test/test_drgn_parser.cpp
+++ b/test/test_drgn_parser.cpp
@@ -426,8 +426,8 @@ TEST_F(DrgnParserTest, PointerNoFollow) {
 
 TEST_F(DrgnParserTest, PointerIncomplete) {
   test("oid_test_case_pointers_incomplete_raw", R"(
-[0] Pointer
-      Incomplete: [IncompleteType]
+[1] Pointer
+[0]   Incomplete: [IncompleteType]
 )");
 }
 
diff --git a/test/test_enforce_compatibility.cpp b/test/test_enforce_compatibility.cpp
index 2632823..d91f356 100644
--- a/test/test_enforce_compatibility.cpp
+++ b/test/test_enforce_compatibility.cpp
@@ -38,8 +38,8 @@ TEST(EnforceCompatibilityTest, VoidPointer) {
        R"(
 [0] Class: MyClass (size: 8)
       Member: p (offset: 0)
-[1]     Pointer
-          Incomplete
+[2]     Pointer
+[1]       Incomplete
             Primitive: void
 )",
        R"(
diff --git a/test/test_flattener.cpp b/test/test_flattener.cpp
index eab3cf8..9f0c0aa 100644
--- a/test/test_flattener.cpp
+++ b/test/test_flattener.cpp
@@ -986,7 +986,7 @@ TEST(FlattenerTest, IncompleteParent) {
        R"(
 [0] Class: MyClass (size: 4)
       Parent (offset: 0)
-        Incomplete: [IncompleteParent]
+[1]     Incomplete: [IncompleteParent]
 )",
        R"(
 [0] Class: MyClass (size: 4)
diff --git a/test/test_name_gen.cpp b/test/test_name_gen.cpp
index 72177d7..d721f7d 100644
--- a/test/test_name_gen.cpp
+++ b/test/test_name_gen.cpp
@@ -475,3 +475,16 @@ TEST(NameGenTest, AnonymousTypes) {
   EXPECT_EQ(myenum.inputName(), "__oi_anon_1");
   EXPECT_EQ(mytypedef.inputName(), "");
 }
+
+TEST(NameGenTest, IncompleteTypes) {
+  auto myincompletevector = Incomplete{0, "std::vector<int>"};
+
+  auto myint = Primitive{Primitive::Kind::Int32};
+  auto myincompleteint = Incomplete{1, myint};
+
+  NameGen nameGen;
+  nameGen.generateNames({myincompletevector, myincompleteint});
+
+  EXPECT_EQ(myincompletevector.name(), "Incomplete<struct std__vector_int_>");
+  EXPECT_EQ(myincompleteint.name(), "Incomplete<struct int32_t>");
+}
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 125fd71..6d1515a 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (!std::is_void<T>::value && oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
         if (s_ptr && pointers.add((uintptr_t)(s_ptr.get()))) {
@@ -37,7 +37,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (std::is_void<T0>::value || !oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && pointers.add((uintptr_t)container.get());
@@ -85,7 +85,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index 58364cf..7ee8335 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T,Deleter>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (!std::is_void<T>::value && oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
         if (u_ptr && pointers.add((uintptr_t)(u_ptr.get()))) {
@@ -38,7 +38,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (std::is_void<T0>::value || !oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && pointers.add((uintptr_t)container.get());
@@ -71,7 +71,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);

From 9023c06a5165e2c16775f812d0450d8d61f0bd0d Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Mon, 20 Nov 2023 09:13:23 -0800
Subject: [PATCH 037/188] jitlog: use a memfd and glog

---
 oi/OIDebugger.cpp | 145 +++++++++++++++++++++++++---------------------
 oi/OIDebugger.h   |   6 +-
 oi/Syscall.h      |   2 +
 3 files changed, 87 insertions(+), 66 deletions(-)

diff --git a/oi/OIDebugger.cpp b/oi/OIDebugger.cpp
index 07faec6..ef40ab6 100644
--- a/oi/OIDebugger.cpp
+++ b/oi/OIDebugger.cpp
@@ -29,6 +29,7 @@
 #include <cerrno>
 #include <chrono>
 #include <cstddef>
+#include <cstdio>
 #include <cstring>
 #include <numeric>
 #include <span>
@@ -195,61 +196,91 @@ bool OIDebugger::singleStepFunc(pid_t pid, uint64_t real_end) {
 }
 
 bool OIDebugger::setupLogFile(void) {
-  // 1. Copy the log file path in out text segment
-  // 2. Run the syscall
-  // 3. Store the resulting fd in the segmentConfigFile
-  if (!segConfig.existingConfig) {
-    auto logFilePath =
-        fs::path("/tmp") / ("oid-" + std::to_string(traceePid) + ".jit.log");
+  // 1. Open an anonymous memfd in the target with `memfd_create`.
+  // 2. Duplicate that fd to the debugger using `pidfd_getfd`.
+  // 3. Store the resulting fds in OIDebugger.
+  bool ret = true;
 
-    auto logFilePathLen = strlen(logFilePath.c_str()) + 1;
-    if (logFilePathLen > textSegSize) {
-      LOG(ERROR) << "The Log File's path " << logFilePath << " ("
-                 << logFilePathLen << ") is too long for the text segment ("
-                 << textSegSize << ")";
-      return false;
-    }
+  auto traceeFd = remoteSyscall<MemfdCreate>("jit.log", 0);
+  if (!traceeFd.has_value()) {
+    LOG(ERROR) << "Failed to create memory log file";
+    return false;
+  }
+  logFds.traceeFd = *traceeFd;
 
-    /*
-     * Using the text segment to store the path in the remote process' memory.
-     * The memory will be re-used anyway and the path will get overwritten.
-     */
-    if (!writeTargetMemory((void*)logFilePath.c_str(),
-                           (void*)segConfig.textSegBase,
-                           logFilePathLen)) {
-      LOG(ERROR) << "Failed to write Log File's path into target process";
-      return false;
-    }
-
-    /*
-     * Execute the `open(2)` syscall on the remote process.
-     * We use the O_SYNC flags to ensure each write we do make it on the disk.
-     * Another option would have been O_DSYNC, but since the logs get always
-     * appended to the end of the file, the file size always changes and
-     * we have to update the metadata everytime anyways.
-     * So O_SYNC won't incure a performance penalty, compared to O_DSYNC,
-     * and ensure we can use the metadata for future automation, etc.
-     */
-    auto fd = remoteSyscall<SysOpen>(segConfig.textSegBase,          // path
-                                     O_CREAT | O_APPEND | O_WRONLY,  // flags
-                                     S_IRUSR | S_IWUSR | S_IRGRP);   // mode
-    if (!fd.has_value()) {
-      LOG(ERROR) << "Failed to open Log File " << logFilePath;
-      return false;
-    }
-
-    segConfig.logFile = *fd;
+  auto traceePidFd = syscall(SYS_pidfd_open, traceePid, 0);
+  if (traceePidFd == -1) {
+    PLOG(ERROR) << "Failed to open child pidfd";
+    return false;
+  }
+  auto debuggerFd = syscall(SYS_pidfd_getfd, traceePidFd, *traceeFd, 0);
+  if (close(static_cast<int>(traceePidFd)) != 0) {
+    PLOG(ERROR) << "Failed to close pidfd";
+    ret = false;
+  }
+  if (debuggerFd == -1) {
+    PLOG(ERROR) << "Failed to duplicate child memfd to debugger";
+    return false;
   }
 
-  return true;
+  logFds.debuggerFd = static_cast<int>(debuggerFd);
+  return ret;
 }
 
 bool OIDebugger::cleanupLogFile(void) {
-  return remoteSyscall<SysClose>(segConfig.logFile).has_value();
+  bool ret = true;
+  if (logFds.traceeFd == -1)
+    return ret;
+
+  if (!remoteSyscall<SysClose>(logFds.traceeFd).has_value()) {
+    LOG(ERROR) << "Remote close failed";
+    ret = false;
+  }
+
+  if (logFds.debuggerFd == -1)
+    return ret;
+
+  FILE* logs = fdopen(logFds.debuggerFd, "r");
+  if (logs == NULL) {
+    PLOG(ERROR) << "Failed to fdopen jitlog";
+    return false;
+  }
+  if (fseek(logs, 0, SEEK_SET) != 0) {
+    PLOG(ERROR) << "Failed to fseek jitlog";
+    return false;
+  }
+
+  char* line = nullptr;
+  size_t read = 0;
+  VLOG(1) << "Outputting JIT logs:";
+  errno = 0;
+  while ((read = getline(&line, &read, logs)) != (size_t)-1) {
+    VLOG(1) << "JITLOG: " << line;
+  }
+  if (errno) {
+    PLOG(ERROR) << "getline";
+    return false;
+  }
+  VLOG(1) << "Finished outputting JIT logs.";
+
+  free(line);
+  if (fclose(logs) == -1) {
+    PLOG(ERROR) << "fclose";
+    return false;
+  }
+
+  return ret;
 }
 
 /* Set up traced process results and text segments */
 bool OIDebugger::segmentInit(void) {
+  if (generatorConfig.features[Feature::JitLogging]) {
+    if (!setupLogFile()) {
+      LOG(ERROR) << "setUpLogFile failed!!!";
+      return false;
+    }
+  }
+
   /*
    * TODO: change this. If setup_results_segment() fails we have to remove
    * the text segment.
@@ -260,11 +291,6 @@ bool OIDebugger::segmentInit(void) {
         LOG(ERROR) << "setUpSegment failed!!!";
         return false;
       }
-
-      if (!setupLogFile()) {
-        LOG(ERROR) << "setUpLogFile failed!!!";
-        return false;
-      }
     } else {
       if (!unmapSegment(SegType::data)) {
         LOG(ERROR) << "Failed to unmmap data segment";
@@ -345,8 +371,7 @@ void OIDebugger::createSegmentConfigFile(void) {
             << " dataSegBase: " << segConfig.dataSegBase
             << " dataSegSize: " << segConfig.dataSegSize
             << " replayInstBase: " << segConfig.replayInstBase
-            << " cookie: " << segConfig.cookie
-            << " logFile: " << segConfig.logFile;
+            << " cookie: " << segConfig.cookie;
 
     assert(segConfig.existingConfig);
   }
@@ -1767,11 +1792,6 @@ bool OIDebugger::unmapSegments(bool deleteSegConfFile) {
     ret = false;
   }
 
-  if (ret && !cleanupLogFile()) {
-    LOG(ERROR) << "Problem closing target process log file";
-    ret = false;
-  }
-
   deleteSegmentConfig(deleteSegConfFile);
 
   return ret;
@@ -1830,13 +1850,6 @@ bool OIDebugger::removeTraps(pid_t pid) {
     it = activeTraps.erase(it);
   }
 
-  if (generatorConfig.features[Feature::JitLogging]) {
-    /* Flush the JIT log, so it's always written on disk at least once */
-    if (!remoteSyscall<SysFsync>(segConfig.logFile).has_value()) {
-      LOG(ERROR) << "Failed to flush the JIT Log";
-    }
-  }
-
   /* Resume the main thread now, so it doesn't have to wait on restoreState */
   if (!contTargetThread(targetPid)) {
     return false;
@@ -2341,7 +2354,7 @@ bool OIDebugger::compileCode() {
     }
 
     int logFile =
-        generatorConfig.features[Feature::JitLogging] ? segConfig.logFile : 0;
+        generatorConfig.features[Feature::JitLogging] ? logFds.traceeFd : 0;
     if (!writeTargetMemory(
             &logFile, (void*)syntheticSymbols["logFile"], sizeof(logFile))) {
       LOG(ERROR) << "Failed to write logFile in probe's cookieValue";
@@ -2411,7 +2424,6 @@ void OIDebugger::restoreState(void) {
         VLOG(1) << "Couldn't interrupt target pid " << p
                 << " (Reason: " << strerror(errno) << ")";
       }
-
       VLOG(1) << "Waiting to stop PID : " << p;
 
       if (waitpid(p, 0, WSTOPPED) != p) {
@@ -2558,6 +2570,9 @@ void OIDebugger::restoreState(void) {
       }
     }
 
+    if (!cleanupLogFile())
+      LOG(ERROR) << "failed to cleanup log file!";
+
     if (ptrace(PTRACE_DETACH, p, 0L, 0L) < 0) {
       LOG(ERROR) << "restoreState Couldn't detach target pid " << p
                  << " (Reason: " << strerror(errno) << ")";
diff --git a/oi/OIDebugger.h b/oi/OIDebugger.h
index 3997616..793fab1 100644
--- a/oi/OIDebugger.h
+++ b/oi/OIDebugger.h
@@ -249,6 +249,11 @@ class OIDebugger {
   std::filesystem::path segConfigFilePath;
   std::filesystem::path customCodeFile;
 
+  struct {
+    int traceeFd = -1;
+    int debuggerFd = -1;
+  } logFds;
+
   struct c {
     uintptr_t textSegBase{};
     size_t textSegSize{};
@@ -259,7 +264,6 @@ class OIDebugger {
     uintptr_t dataSegBase{};
     size_t dataSegSize{};
     uintptr_t cookie{};
-    int logFile{};
   } segConfig{};
 
   /*
diff --git a/oi/Syscall.h b/oi/Syscall.h
index 521e5ec..59ce775 100644
--- a/oi/Syscall.h
+++ b/oi/Syscall.h
@@ -71,6 +71,8 @@ struct Syscall {
 using SysOpen = Syscall<"open", SYS_open, int, const char*, int, mode_t>;
 using SysClose = Syscall<"close", SYS_close, int, int>;
 using SysFsync = Syscall<"fsync", SYS_fsync, int, int>;
+using MemfdCreate =
+    Syscall<"memfd_create", SYS_memfd_create, int, const char*, unsigned int>;
 
 using SysMmap =
     Syscall<"mmap", SYS_mmap, void*, void*, size_t, int, int, int, off_t>;

From 12b4c4ed13766b1748ca0355cc60509bf57ab8af Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 21 Nov 2023 02:50:31 -0800
Subject: [PATCH 038/188] jitlog: use a memfd and glog

Summary:

Changes jitlog to use a memfd, an anonymous in memory file descriptor, rather
than a file on disk. Also clean up this fd at the end of an OID run rather than
leaving it in the hope it's valid next time.

A previous attempt to land this used a `char*` from the OID process space in
the remote target syscall. Somehow this works with our integration test target,
but not globally. Changed to use the previous behaviour of putting the syscall
arg in the empty text segment. In doing this I noticed that the text segment
wouldn't be initialised at this point on a fresh process, so we were copying
into effectively an uninitialised address. Move the jit log fd setup to after
the segment setup accordingly.

Test plan:
- CI
- Tested on an integration test target as before. Works.
- Created a new target that definitely doesn't have this string in (simple for
  loop). Failed before, works now.

Example:
```sh
$ OID_TEST_ARGS='-fjit-logging' stest OidIntegration.simple_struct
...
I1121 02:57:36.136890 500897 OIDebugger.cpp:269] Outputting JIT logs:
I1121 02:57:36.136896 500897 OIDebugger.cpp:272] JITLOG: SimpleStruct
@00007ffc639be180
I1121 02:57:36.136899 500897 OIDebugger.cpp:272] JITLOG: a @00007ffc639be180
I1121 02:57:36.136901 500897 OIDebugger.cpp:272] JITLOG: obj @00007ffc639be180
I1121 02:57:36.136904 500897 OIDebugger.cpp:272] JITLOG: b @00007ffc639be184
I1121 02:57:36.136905 500897 OIDebugger.cpp:272] JITLOG: obj @00007ffc639be184
I1121 02:57:36.136907 500897 OIDebugger.cpp:272] JITLOG: c @00007ffc639be188
I1121 02:57:36.136909 500897 OIDebugger.cpp:272] JITLOG: obj @00007ffc639be188
I1121 02:57:36.136911 500897 OIDebugger.cpp:278] Finished outputting JIT logs.
...
```
---
 oi/OIDebugger.cpp | 159 +++++++++++++++++++++++++++-------------------
 oi/OIDebugger.h   |   6 +-
 oi/Syscall.h      |   2 +
 3 files changed, 101 insertions(+), 66 deletions(-)

diff --git a/oi/OIDebugger.cpp b/oi/OIDebugger.cpp
index 07faec6..e984b47 100644
--- a/oi/OIDebugger.cpp
+++ b/oi/OIDebugger.cpp
@@ -29,6 +29,7 @@
 #include <cerrno>
 #include <chrono>
 #include <cstddef>
+#include <cstdio>
 #include <cstring>
 #include <numeric>
 #include <span>
@@ -195,57 +196,94 @@ bool OIDebugger::singleStepFunc(pid_t pid, uint64_t real_end) {
 }
 
 bool OIDebugger::setupLogFile(void) {
-  // 1. Copy the log file path in out text segment
-  // 2. Run the syscall
-  // 3. Store the resulting fd in the segmentConfigFile
-  if (!segConfig.existingConfig) {
-    auto logFilePath =
-        fs::path("/tmp") / ("oid-" + std::to_string(traceePid) + ".jit.log");
+  // 1. Copy the anonymous file name into our text segment.
+  // 2. Open an anonymous memfd in the target with `memfd_create`.
+  // 3. Duplicate that fd to the debugger using `pidfd_getfd`.
+  // 4. Store the resulting fds in OIDebugger.
+  bool ret = true;
 
-    auto logFilePathLen = strlen(logFilePath.c_str()) + 1;
-    if (logFilePathLen > textSegSize) {
-      LOG(ERROR) << "The Log File's path " << logFilePath << " ("
-                 << logFilePathLen << ") is too long for the text segment ("
-                 << textSegSize << ")";
-      return false;
-    }
+  static const std::string kAnonFileName{"jit.log"};
 
-    /*
-     * Using the text segment to store the path in the remote process' memory.
-     * The memory will be re-used anyway and the path will get overwritten.
-     */
-    if (!writeTargetMemory((void*)logFilePath.c_str(),
-                           (void*)segConfig.textSegBase,
-                           logFilePathLen)) {
-      LOG(ERROR) << "Failed to write Log File's path into target process";
-      return false;
-    }
-
-    /*
-     * Execute the `open(2)` syscall on the remote process.
-     * We use the O_SYNC flags to ensure each write we do make it on the disk.
-     * Another option would have been O_DSYNC, but since the logs get always
-     * appended to the end of the file, the file size always changes and
-     * we have to update the metadata everytime anyways.
-     * So O_SYNC won't incure a performance penalty, compared to O_DSYNC,
-     * and ensure we can use the metadata for future automation, etc.
-     */
-    auto fd = remoteSyscall<SysOpen>(segConfig.textSegBase,          // path
-                                     O_CREAT | O_APPEND | O_WRONLY,  // flags
-                                     S_IRUSR | S_IWUSR | S_IRGRP);   // mode
-    if (!fd.has_value()) {
-      LOG(ERROR) << "Failed to open Log File " << logFilePath;
-      return false;
-    }
-
-    segConfig.logFile = *fd;
+  // Copy the file name into the text segment so it can be referenced by the
+  // remote syscall. This will be overwritten later during relocation.
+  if (!writeTargetMemory(
+        (void*)(kAnonFileName.c_str()),
+        reinterpret_cast<void*>(segConfig.textSegBase),
+        kAnonFileName.length() + 1
+  )) {
+    LOG(ERROR) << "Failed to write log file's name into target process";
+    return false;
   }
 
-  return true;
+  auto traceeFd = remoteSyscall<MemfdCreate>(segConfig.textSegBase, 0);
+  if (!traceeFd.has_value()) {
+    LOG(ERROR) << "Failed to create memory log file";
+    return false;
+  }
+  logFds.traceeFd = *traceeFd;
+
+  auto traceePidFd = syscall(SYS_pidfd_open, traceePid, 0);
+  if (traceePidFd == -1) {
+    PLOG(ERROR) << "Failed to open child pidfd";
+    return false;
+  }
+  auto debuggerFd = syscall(SYS_pidfd_getfd, traceePidFd, *traceeFd, 0);
+  if (close(static_cast<int>(traceePidFd)) != 0) {
+    PLOG(ERROR) << "Failed to close pidfd";
+    ret = false;
+  }
+  if (debuggerFd == -1) {
+    PLOG(ERROR) << "Failed to duplicate child memfd to debugger";
+    return false;
+  }
+
+  logFds.debuggerFd = static_cast<int>(debuggerFd);
+  return ret;
 }
 
 bool OIDebugger::cleanupLogFile(void) {
-  return remoteSyscall<SysClose>(segConfig.logFile).has_value();
+  bool ret = true;
+  if (logFds.traceeFd == -1)
+    return ret;
+
+  if (!remoteSyscall<SysClose>(logFds.traceeFd).has_value()) {
+    LOG(ERROR) << "Remote close failed";
+    ret = false;
+  }
+
+  if (logFds.debuggerFd == -1)
+    return ret;
+
+  FILE* logs = fdopen(logFds.debuggerFd, "r");
+  if (logs == NULL) {
+    PLOG(ERROR) << "Failed to fdopen jitlog";
+    return false;
+  }
+  if (fseek(logs, 0, SEEK_SET) != 0) {
+    PLOG(ERROR) << "Failed to fseek jitlog";
+    return false;
+  }
+
+  char* line = nullptr;
+  size_t read = 0;
+  VLOG(1) << "Outputting JIT logs:";
+  errno = 0;
+  while ((read = getline(&line, &read, logs)) != (size_t)-1) {
+    VLOG(1) << "JITLOG: " << line;
+  }
+  if (errno) {
+    PLOG(ERROR) << "getline";
+    return false;
+  }
+  VLOG(1) << "Finished outputting JIT logs.";
+
+  free(line);
+  if (fclose(logs) == -1) {
+    PLOG(ERROR) << "fclose";
+    return false;
+  }
+
+  return ret;
 }
 
 /* Set up traced process results and text segments */
@@ -260,11 +298,6 @@ bool OIDebugger::segmentInit(void) {
         LOG(ERROR) << "setUpSegment failed!!!";
         return false;
       }
-
-      if (!setupLogFile()) {
-        LOG(ERROR) << "setUpLogFile failed!!!";
-        return false;
-      }
     } else {
       if (!unmapSegment(SegType::data)) {
         LOG(ERROR) << "Failed to unmmap data segment";
@@ -295,6 +328,13 @@ bool OIDebugger::segmentInit(void) {
   auto now = high_resolution_clock::now().time_since_epoch();
   segConfig.cookie = duration_cast<duration<uintptr_t, std::nano>>(now).count();
 
+  if (generatorConfig.features[Feature::JitLogging]) {
+    if (!setupLogFile()) {
+      LOG(ERROR) << "setUpLogFile failed!!!";
+      return false;
+    }
+  }
+
   return true;
 }
 
@@ -345,8 +385,7 @@ void OIDebugger::createSegmentConfigFile(void) {
             << " dataSegBase: " << segConfig.dataSegBase
             << " dataSegSize: " << segConfig.dataSegSize
             << " replayInstBase: " << segConfig.replayInstBase
-            << " cookie: " << segConfig.cookie
-            << " logFile: " << segConfig.logFile;
+            << " cookie: " << segConfig.cookie;
 
     assert(segConfig.existingConfig);
   }
@@ -1767,11 +1806,6 @@ bool OIDebugger::unmapSegments(bool deleteSegConfFile) {
     ret = false;
   }
 
-  if (ret && !cleanupLogFile()) {
-    LOG(ERROR) << "Problem closing target process log file";
-    ret = false;
-  }
-
   deleteSegmentConfig(deleteSegConfFile);
 
   return ret;
@@ -1830,13 +1864,6 @@ bool OIDebugger::removeTraps(pid_t pid) {
     it = activeTraps.erase(it);
   }
 
-  if (generatorConfig.features[Feature::JitLogging]) {
-    /* Flush the JIT log, so it's always written on disk at least once */
-    if (!remoteSyscall<SysFsync>(segConfig.logFile).has_value()) {
-      LOG(ERROR) << "Failed to flush the JIT Log";
-    }
-  }
-
   /* Resume the main thread now, so it doesn't have to wait on restoreState */
   if (!contTargetThread(targetPid)) {
     return false;
@@ -2341,7 +2368,7 @@ bool OIDebugger::compileCode() {
     }
 
     int logFile =
-        generatorConfig.features[Feature::JitLogging] ? segConfig.logFile : 0;
+        generatorConfig.features[Feature::JitLogging] ? logFds.traceeFd : 0;
     if (!writeTargetMemory(
             &logFile, (void*)syntheticSymbols["logFile"], sizeof(logFile))) {
       LOG(ERROR) << "Failed to write logFile in probe's cookieValue";
@@ -2411,7 +2438,6 @@ void OIDebugger::restoreState(void) {
         VLOG(1) << "Couldn't interrupt target pid " << p
                 << " (Reason: " << strerror(errno) << ")";
       }
-
       VLOG(1) << "Waiting to stop PID : " << p;
 
       if (waitpid(p, 0, WSTOPPED) != p) {
@@ -2558,6 +2584,9 @@ void OIDebugger::restoreState(void) {
       }
     }
 
+    if (!cleanupLogFile())
+      LOG(ERROR) << "failed to cleanup log file!";
+
     if (ptrace(PTRACE_DETACH, p, 0L, 0L) < 0) {
       LOG(ERROR) << "restoreState Couldn't detach target pid " << p
                  << " (Reason: " << strerror(errno) << ")";
diff --git a/oi/OIDebugger.h b/oi/OIDebugger.h
index 3997616..793fab1 100644
--- a/oi/OIDebugger.h
+++ b/oi/OIDebugger.h
@@ -249,6 +249,11 @@ class OIDebugger {
   std::filesystem::path segConfigFilePath;
   std::filesystem::path customCodeFile;
 
+  struct {
+    int traceeFd = -1;
+    int debuggerFd = -1;
+  } logFds;
+
   struct c {
     uintptr_t textSegBase{};
     size_t textSegSize{};
@@ -259,7 +264,6 @@ class OIDebugger {
     uintptr_t dataSegBase{};
     size_t dataSegSize{};
     uintptr_t cookie{};
-    int logFile{};
   } segConfig{};
 
   /*
diff --git a/oi/Syscall.h b/oi/Syscall.h
index 521e5ec..59ce775 100644
--- a/oi/Syscall.h
+++ b/oi/Syscall.h
@@ -71,6 +71,8 @@ struct Syscall {
 using SysOpen = Syscall<"open", SYS_open, int, const char*, int, mode_t>;
 using SysClose = Syscall<"close", SYS_close, int, int>;
 using SysFsync = Syscall<"fsync", SYS_fsync, int, int>;
+using MemfdCreate =
+    Syscall<"memfd_create", SYS_memfd_create, int, const char*, unsigned int>;
 
 using SysMmap =
     Syscall<"mmap", SYS_mmap, void*, void*, size_t, int, int, int, off_t>;

From eba6c87a8c71485357965d397dcc2cd593637151 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 21 Nov 2023 02:50:31 -0800
Subject: [PATCH 039/188] jitlog: use a memfd and glog

Summary:

Changes jitlog to use a memfd, an anonymous in memory file descriptor, rather
than a file on disk. Also clean up this fd at the end of an OID run rather than
leaving it in the hope it's valid next time.

A previous attempt to land this used a `char*` from the OID process space in
the remote target syscall. Somehow this works with our integration test target,
but not globally. Changed to use the previous behaviour of putting the syscall
arg in the empty text segment. In doing this I noticed that the text segment
wouldn't be initialised at this point on a fresh process, so we were copying
into effectively an uninitialised address. Move the jit log fd setup to after
the segment setup accordingly.

Test plan:
- CI
- Tested on an integration test target as before. Works.
- Created a new target that definitely doesn't have this string in (simple for
  loop). Failed before, works now.

Example:
```sh
$ OID_TEST_ARGS='-fjit-logging' stest OidIntegration.simple_struct
...
I1121 02:57:36.136890 500897 OIDebugger.cpp:269] Outputting JIT logs:
I1121 02:57:36.136896 500897 OIDebugger.cpp:272] JITLOG: SimpleStruct
@00007ffc639be180
I1121 02:57:36.136899 500897 OIDebugger.cpp:272] JITLOG: a @00007ffc639be180
I1121 02:57:36.136901 500897 OIDebugger.cpp:272] JITLOG: obj @00007ffc639be180
I1121 02:57:36.136904 500897 OIDebugger.cpp:272] JITLOG: b @00007ffc639be184
I1121 02:57:36.136905 500897 OIDebugger.cpp:272] JITLOG: obj @00007ffc639be184
I1121 02:57:36.136907 500897 OIDebugger.cpp:272] JITLOG: c @00007ffc639be188
I1121 02:57:36.136909 500897 OIDebugger.cpp:272] JITLOG: obj @00007ffc639be188
I1121 02:57:36.136911 500897 OIDebugger.cpp:278] Finished outputting JIT logs.
...
```
---
 oi/OIDebugger.cpp | 157 +++++++++++++++++++++++++++-------------------
 oi/OIDebugger.h   |   6 +-
 oi/Syscall.h      |   2 +
 3 files changed, 99 insertions(+), 66 deletions(-)

diff --git a/oi/OIDebugger.cpp b/oi/OIDebugger.cpp
index 07faec6..64dbdf8 100644
--- a/oi/OIDebugger.cpp
+++ b/oi/OIDebugger.cpp
@@ -29,6 +29,7 @@
 #include <cerrno>
 #include <chrono>
 #include <cstddef>
+#include <cstdio>
 #include <cstring>
 #include <numeric>
 #include <span>
@@ -195,57 +196,92 @@ bool OIDebugger::singleStepFunc(pid_t pid, uint64_t real_end) {
 }
 
 bool OIDebugger::setupLogFile(void) {
-  // 1. Copy the log file path in out text segment
-  // 2. Run the syscall
-  // 3. Store the resulting fd in the segmentConfigFile
-  if (!segConfig.existingConfig) {
-    auto logFilePath =
-        fs::path("/tmp") / ("oid-" + std::to_string(traceePid) + ".jit.log");
+  // 1. Copy the anonymous file name into our text segment.
+  // 2. Open an anonymous memfd in the target with `memfd_create`.
+  // 3. Duplicate that fd to the debugger using `pidfd_getfd`.
+  // 4. Store the resulting fds in OIDebugger.
+  bool ret = true;
 
-    auto logFilePathLen = strlen(logFilePath.c_str()) + 1;
-    if (logFilePathLen > textSegSize) {
-      LOG(ERROR) << "The Log File's path " << logFilePath << " ("
-                 << logFilePathLen << ") is too long for the text segment ("
-                 << textSegSize << ")";
-      return false;
-    }
+  static const std::string kAnonFileName{"jit.log"};
 
-    /*
-     * Using the text segment to store the path in the remote process' memory.
-     * The memory will be re-used anyway and the path will get overwritten.
-     */
-    if (!writeTargetMemory((void*)logFilePath.c_str(),
-                           (void*)segConfig.textSegBase,
-                           logFilePathLen)) {
-      LOG(ERROR) << "Failed to write Log File's path into target process";
-      return false;
-    }
-
-    /*
-     * Execute the `open(2)` syscall on the remote process.
-     * We use the O_SYNC flags to ensure each write we do make it on the disk.
-     * Another option would have been O_DSYNC, but since the logs get always
-     * appended to the end of the file, the file size always changes and
-     * we have to update the metadata everytime anyways.
-     * So O_SYNC won't incure a performance penalty, compared to O_DSYNC,
-     * and ensure we can use the metadata for future automation, etc.
-     */
-    auto fd = remoteSyscall<SysOpen>(segConfig.textSegBase,          // path
-                                     O_CREAT | O_APPEND | O_WRONLY,  // flags
-                                     S_IRUSR | S_IWUSR | S_IRGRP);   // mode
-    if (!fd.has_value()) {
-      LOG(ERROR) << "Failed to open Log File " << logFilePath;
-      return false;
-    }
-
-    segConfig.logFile = *fd;
+  // Copy the file name into the text segment so it can be referenced by the
+  // remote syscall. This will be overwritten later during relocation.
+  if (!writeTargetMemory((void*)(kAnonFileName.c_str()),
+                         reinterpret_cast<void*>(segConfig.textSegBase),
+                         kAnonFileName.length() + 1)) {
+    LOG(ERROR) << "Failed to write log file's name into target process";
+    return false;
   }
 
-  return true;
+  auto traceeFd = remoteSyscall<MemfdCreate>(segConfig.textSegBase, 0);
+  if (!traceeFd.has_value()) {
+    LOG(ERROR) << "Failed to create memory log file";
+    return false;
+  }
+  logFds.traceeFd = *traceeFd;
+
+  auto traceePidFd = syscall(SYS_pidfd_open, traceePid, 0);
+  if (traceePidFd == -1) {
+    PLOG(ERROR) << "Failed to open child pidfd";
+    return false;
+  }
+  auto debuggerFd = syscall(SYS_pidfd_getfd, traceePidFd, *traceeFd, 0);
+  if (close(static_cast<int>(traceePidFd)) != 0) {
+    PLOG(ERROR) << "Failed to close pidfd";
+    ret = false;
+  }
+  if (debuggerFd == -1) {
+    PLOG(ERROR) << "Failed to duplicate child memfd to debugger";
+    return false;
+  }
+
+  logFds.debuggerFd = static_cast<int>(debuggerFd);
+  return ret;
 }
 
 bool OIDebugger::cleanupLogFile(void) {
-  return remoteSyscall<SysClose>(segConfig.logFile).has_value();
+  bool ret = true;
+  if (logFds.traceeFd == -1)
+    return ret;
+
+  if (!remoteSyscall<SysClose>(logFds.traceeFd).has_value()) {
+    LOG(ERROR) << "Remote close failed";
+    ret = false;
+  }
+
+  if (logFds.debuggerFd == -1)
+    return ret;
+
+  FILE* logs = fdopen(logFds.debuggerFd, "r");
+  if (logs == NULL) {
+    PLOG(ERROR) << "Failed to fdopen jitlog";
+    return false;
+  }
+  if (fseek(logs, 0, SEEK_SET) != 0) {
+    PLOG(ERROR) << "Failed to fseek jitlog";
+    return false;
+  }
+
+  char* line = nullptr;
+  size_t read = 0;
+  VLOG(1) << "Outputting JIT logs:";
+  errno = 0;
+  while ((read = getline(&line, &read, logs)) != (size_t)-1) {
+    VLOG(1) << "JITLOG: " << line;
+  }
+  if (errno) {
+    PLOG(ERROR) << "getline";
+    return false;
+  }
+  VLOG(1) << "Finished outputting JIT logs.";
+
+  free(line);
+  if (fclose(logs) == -1) {
+    PLOG(ERROR) << "fclose";
+    return false;
+  }
+
+  return ret;
 }
 
 /* Set up traced process results and text segments */
@@ -260,11 +296,6 @@ bool OIDebugger::segmentInit(void) {
         LOG(ERROR) << "setUpSegment failed!!!";
         return false;
       }
-
-      if (!setupLogFile()) {
-        LOG(ERROR) << "setUpLogFile failed!!!";
-        return false;
-      }
     } else {
       if (!unmapSegment(SegType::data)) {
         LOG(ERROR) << "Failed to unmmap data segment";
@@ -295,6 +326,13 @@ bool OIDebugger::segmentInit(void) {
   auto now = high_resolution_clock::now().time_since_epoch();
   segConfig.cookie = duration_cast<duration<uintptr_t, std::nano>>(now).count();
 
+  if (generatorConfig.features[Feature::JitLogging]) {
+    if (!setupLogFile()) {
+      LOG(ERROR) << "setUpLogFile failed!!!";
+      return false;
+    }
+  }
+
   return true;
 }
 
@@ -345,8 +383,7 @@ void OIDebugger::createSegmentConfigFile(void) {
             << " dataSegBase: " << segConfig.dataSegBase
             << " dataSegSize: " << segConfig.dataSegSize
             << " replayInstBase: " << segConfig.replayInstBase
-            << " cookie: " << segConfig.cookie
-            << " logFile: " << segConfig.logFile;
+            << " cookie: " << segConfig.cookie;
 
     assert(segConfig.existingConfig);
   }
@@ -1767,11 +1804,6 @@ bool OIDebugger::unmapSegments(bool deleteSegConfFile) {
     ret = false;
   }
 
-  if (ret && !cleanupLogFile()) {
-    LOG(ERROR) << "Problem closing target process log file";
-    ret = false;
-  }
-
   deleteSegmentConfig(deleteSegConfFile);
 
   return ret;
@@ -1830,13 +1862,6 @@ bool OIDebugger::removeTraps(pid_t pid) {
     it = activeTraps.erase(it);
   }
 
-  if (generatorConfig.features[Feature::JitLogging]) {
-    /* Flush the JIT log, so it's always written on disk at least once */
-    if (!remoteSyscall<SysFsync>(segConfig.logFile).has_value()) {
-      LOG(ERROR) << "Failed to flush the JIT Log";
-    }
-  }
-
   /* Resume the main thread now, so it doesn't have to wait on restoreState */
   if (!contTargetThread(targetPid)) {
     return false;
@@ -2341,7 +2366,7 @@ bool OIDebugger::compileCode() {
     }
 
     int logFile =
-        generatorConfig.features[Feature::JitLogging] ? segConfig.logFile : 0;
+        generatorConfig.features[Feature::JitLogging] ? logFds.traceeFd : 0;
     if (!writeTargetMemory(
             &logFile, (void*)syntheticSymbols["logFile"], sizeof(logFile))) {
       LOG(ERROR) << "Failed to write logFile in probe's cookieValue";
@@ -2411,7 +2436,6 @@ void OIDebugger::restoreState(void) {
         VLOG(1) << "Couldn't interrupt target pid " << p
                 << " (Reason: " << strerror(errno) << ")";
       }
-
       VLOG(1) << "Waiting to stop PID : " << p;
 
       if (waitpid(p, 0, WSTOPPED) != p) {
@@ -2558,6 +2582,9 @@ void OIDebugger::restoreState(void) {
       }
     }
 
+    if (!cleanupLogFile())
+      LOG(ERROR) << "failed to cleanup log file!";
+
     if (ptrace(PTRACE_DETACH, p, 0L, 0L) < 0) {
       LOG(ERROR) << "restoreState Couldn't detach target pid " << p
                  << " (Reason: " << strerror(errno) << ")";
diff --git a/oi/OIDebugger.h b/oi/OIDebugger.h
index 3997616..793fab1 100644
--- a/oi/OIDebugger.h
+++ b/oi/OIDebugger.h
@@ -249,6 +249,11 @@ class OIDebugger {
   std::filesystem::path segConfigFilePath;
   std::filesystem::path customCodeFile;
 
+  struct {
+    int traceeFd = -1;
+    int debuggerFd = -1;
+  } logFds;
+
   struct c {
     uintptr_t textSegBase{};
     size_t textSegSize{};
@@ -259,7 +264,6 @@ class OIDebugger {
     uintptr_t dataSegBase{};
     size_t dataSegSize{};
     uintptr_t cookie{};
-    int logFile{};
   } segConfig{};
 
   /*
diff --git a/oi/Syscall.h b/oi/Syscall.h
index 521e5ec..59ce775 100644
--- a/oi/Syscall.h
+++ b/oi/Syscall.h
@@ -71,6 +71,8 @@ struct Syscall {
 using SysOpen = Syscall<"open", SYS_open, int, const char*, int, mode_t>;
 using SysClose = Syscall<"close", SYS_close, int, int>;
 using SysFsync = Syscall<"fsync", SYS_fsync, int, int>;
+using MemfdCreate =
+    Syscall<"memfd_create", SYS_memfd_create, int, const char*, unsigned int>;
 
 using SysMmap =
     Syscall<"mmap", SYS_mmap, void*, void*, size_t, int, int, int, off_t>;

From fceaed3d3130151c1bb1c7e815044433659b0266 Mon Sep 17 00:00:00 2001
From: Thierry Treyer <ttreyer@fb.com>
Date: Tue, 21 Nov 2023 06:32:37 -0800
Subject: [PATCH 040/188] drgn: Reconstruct template parameters

---
 extern/drgn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extern/drgn b/extern/drgn
index de24803..c894da7 160000
--- a/extern/drgn
+++ b/extern/drgn
@@ -1 +1 @@
-Subproject commit de2480353e73a48063f4eca4b620d72e80e5246f
+Subproject commit c894da76075b16e68cad4525786d025ddf10040a

From d3049b5a1e9a62ac7ab7e1f3299bfc87d419b276 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 5 Dec 2023 08:21:20 -0800
Subject: [PATCH 041/188] oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD
---
 CMakeLists.txt                          |  12 +-
 oi/CodeGen.cpp                          |  30 ++-
 oi/CodeGen.h                            |   8 +-
 oi/Config.cpp                           |   9 +-
 oi/ContainerInfo.cpp                    |   2 +-
 oi/OICodeGen.h                          |   2 +-
 oi/OIGenerator.cpp                      | 332 ++++++++++++------------
 oi/OIGenerator.h                        |  22 +-
 oi/type_graph/CMakeLists.txt            |   2 +
 oi/type_graph/ClangTypeParser.cpp       | 283 ++++++++++++++++++++
 oi/type_graph/ClangTypeParser.h         | 102 ++++++++
 oi/type_graph/NameGen.cpp               |   8 +
 oi/type_graph/NameGen.h                 |   1 +
 oi/type_graph/Printer.cpp               |  11 +
 oi/type_graph/Printer.h                 |   1 +
 oi/type_graph/RemoveTopLevelPointer.cpp |   4 +
 oi/type_graph/RemoveTopLevelPointer.h   |   1 +
 oi/type_graph/Types.h                   |  56 ++++
 oi/type_graph/Visitor.h                 |   7 +
 tools/OILGen.cpp                        | 142 +++-------
 20 files changed, 741 insertions(+), 294 deletions(-)
 create mode 100644 oi/type_graph/ClangTypeParser.cpp
 create mode 100644 oi/type_graph/ClangTypeParser.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f9ee7..9273682 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -104,6 +104,15 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(range-v3)
 
+### cxxopts
+FetchContent_Declare(
+  cxxopts
+  GIT_REPOSITORY https://github.com/jarro2783/cxxopts.git
+  GIT_TAG        eb787304d67ec22f7c3a184ee8b4c481d04357fd # v3.1.1
+  GIT_PROGRESS   TRUE
+)
+FetchContent_MakeAvailable(cxxopts)
+
 set_project_warnings()
 
 if (ASAN)
@@ -289,7 +298,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -363,6 +372,7 @@ add_executable(oilgen
   oi/OIGenerator.cpp
 )
 target_link_libraries(oilgen
+  cxxopts
   drgn_utils
   oicore
 )
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index bef102f..a7b59bb 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -539,7 +539,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1106,15 +1106,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
 }
 
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
-  try {
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
+  if (!registerContainers())
     return false;
-  }
 
   TypeGraph typeGraph;
   try {
@@ -1129,6 +1122,20 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
   return true;
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    size_t i = 0;
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1170,7 +1177,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(Flattener::createPass());
@@ -1286,7 +1293,8 @@ void CodeGen::generate(
   code += "\nusing __ROOT_TYPE__ = " + rootType.name() + ";\n";
   code += "} // namespace\n} // namespace OIInternal\n";
 
-  const auto typeName = SymbolService::getTypeName(drgnType);
+  // const auto typeName = SymbolService::getTypeName(drgnType);
+  const std::string typeName{"typeName"};
   if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
   } else {
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 186ec34..fb0ec20 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -40,8 +40,11 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config) : config_(config) {
+    assert(!config.features[Feature::PolymorphicInheritance]);
+  }
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   /*
@@ -53,6 +56,7 @@ class CodeGen {
                        std::string linkageName,
                        std::string& code);
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -66,7 +70,7 @@ class CodeGen {
 
  private:
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
diff --git a/oi/Config.cpp b/oi/Config.cpp
index 157b279..66798bb 100644
--- a/oi/Config.cpp
+++ b/oi/Config.cpp
@@ -22,6 +22,8 @@
 #include <boost/property_tree/ini_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 #include <filesystem>
+#include <range/v3/action/sort.hpp>
+#include <range/v3/action/unique.hpp>
 
 #include "oi/support/Toml.h"
 
@@ -54,6 +56,9 @@ std::optional<FeatureSet> processConfigFiles(
     enables |= *fs;
   }
 
+  ranges::actions::sort(generatorConfig.containerConfigPaths);
+  ranges::actions::unique(generatorConfig.containerConfigPaths);
+
   // Override anything from the config files with command line options
   for (auto [k, v] : featureMap) {
     enables[k] = v;
@@ -108,8 +113,8 @@ std::optional<FeatureSet> processConfigFile(
           if the right path is absolute, else append the right path to the
           left path.
           */
-          generatorConfig.containerConfigPaths.emplace(configDirectory /
-                                                       el.get());
+          generatorConfig.containerConfigPaths.emplace_back(configDirectory /
+                                                            el.get());
         }
       });
     }
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 99a7dc1..dbe10c6 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -202,7 +202,7 @@ std::regex getMatcher(const std::string& typeName) {
 ContainerInfo::ContainerInfo(const fs::path& path) {
   toml::table container;
   try {
-    container = toml::parse_file(std::string(path));
+    container = toml::parse_file(path.string());
   } catch (const toml::parse_error& err) {
     // Convert into a ContainerInfoError, just to avoid having to include
     // the huge TOML++ header in the caller's file. Use toml::parse_error's
diff --git a/oi/OICodeGen.h b/oi/OICodeGen.h
index a01548f..7226a4e 100644
--- a/oi/OICodeGen.h
+++ b/oi/OICodeGen.h
@@ -66,7 +66,7 @@ class OICodeGen {
     };
 
     FeatureSet features;
-    std::set<std::filesystem::path> containerConfigPaths;
+    std::vector<std::filesystem::path> containerConfigPaths;
     std::set<std::string> defaultHeaders;
     std::set<std::string> defaultNamespaces;
     std::vector<std::pair<std::string, std::string>> membersToStub;
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 1cb2a49..e4dbf98 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -16,171 +16,88 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
 #include <glog/logging.h>
 
 #include <boost/core/demangle.hpp>
 #include <fstream>
 #include <iostream>
+#include <range/v3/action/transform.hpp>
+#include <range/v3/core.hpp>
+#include <range/v3/view/cache1.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/drop_while.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>  // TODO: remove
 #include <string_view>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
-
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
-      templateArgsToSymbolsMap;
-
-  auto symbols = prog.find_all_symbols();
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
+class CreateTypeGraphConsumer;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(clang::CompilerInstance& inst_) : inst{inst_} {
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
-    if (val.first.empty() || val.second.empty()) {
-      continue;
-    }
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  type_graph::TypeGraph typeGraph;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
+
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  clang::CompilerInstance& inst;
+
+  friend CreateTypeGraphConsumer;
+};
+
+}  // namespace
+
+int OIGenerator::generate() {
+  clang::CompilerInstance inst;
+  inst.createDiagnostics();
+
+  auto inv = std::make_shared<clang::CompilerInvocation>();
+
+  std::vector<const char*> clangArgsCstr =
+      clangArgs |
+      ranges::views::transform([](const auto& s) { return s.c_str(); }) |
+      ranges::to<std::vector>();
+  if (!clang::CompilerInvocation::CreateFromArgs(
+          *inv, clangArgsCstr, inst.getDiagnostics())) {
+    LOG(ERROR) << "Failed to initialise compiler!";
+    return -1;
   }
 
-  return strongToWeakSymbols;
-}
+  inst.setInvocation(inv);
 
-std::unordered_map<std::string, drgn_qualified_type>
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> out;
-
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
-    std::string strongLinkageName;
-    {
-      const char* linkageNameCstr;
-      if (auto err = drgnplusplus::error(
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
-        // throw err;
-        continue;
-      }
-      strongLinkageName = linkageNameCstr;
-    }
-
-    std::string weakLinkageName;
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
+  CreateTypeGraphAction action{inst};
+  if (!inst.ExecuteAction(action)) {
+    LOG(ERROR) << "Action execution failed!";
+    return -1;
   }
-
-  return out;
-}
-
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
-                                      const OICompiler::Config& compilerConfig,
-                                      const drgn_qualified_type& type,
-                                      const std::string& linkageName,
-                                      SymbolService& symbols) {
-  CodeGen codegen{generatorConfig, symbols};
-
-  std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
-
-  std::string sourcePath = sourceFileDumpPath;
-  if (sourceFileDumpPath.empty()) {
-    // This is the path Clang acts as if it has compiled from e.g. for debug
-    // information. It does not need to exist.
-    sourcePath = "oil_jit.cpp";
-  } else {
-    std::ofstream outputFile(sourcePath);
-    outputFile << code;
-  }
-
-  OICompiler compiler{{}, compilerConfig};
-
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
-
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
-}
-
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
-  drgnplusplus::program prog;
-
-  {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(
-            drgn_program_load_debug_info(prog.get(),
-                                         std::data(objectPaths),
-                                         std::size(objectPaths),
-                                         false,
-                                         false))) {
-      LOG(ERROR) << "error loading debug info program: " << err;
-      throw err;
-    }
-  }
-
-  auto oilTypes = findOilTypesAndNames(prog);
+  if (action.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one type to generate for but we can't currently "
+        "handle this case");
 
   std::map<Feature, bool> featuresMap = {
       {Feature::TypeGraph, true},
@@ -203,27 +120,116 @@ int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
   generatorConfig.features = *features;
   compilerConfig.features = *features;
 
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
-    }
-  }
-
-  size_t successes = oilTypes.size() - failures;
-  LOG(INFO) << "object introspection generation complete. " << successes
-            << " successes and " << failures << " failures.";
-
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
+  CodeGen codegen{generatorConfig};
+  if (!codegen.registerContainers()) {
+    LOG(ERROR) << "Failed to register containers";
     return -1;
   }
-  return 0;
+  codegen.transform(action.typeGraph);
+
+  std::string code;
+  codegen.generate(action.typeGraph, code, nullptr);
+
+  std::string sourcePath = sourceFileDumpPath;
+  if (sourceFileDumpPath.empty()) {
+    // This is the path Clang acts as if it has compiled from e.g. for debug
+    // information. It does not need to exist.
+    sourcePath = "oil_jit.cpp";
+  } else {
+    std::ofstream outputFile(sourcePath);
+    outputFile << code;
+  }
+
+  OICompiler compiler{{}, compilerConfig};
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
 }
 
+namespace {
+
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  CreateTypeGraphAction& self;
+
+ public:
+  CreateTypeGraphConsumer(CreateTypeGraphAction& self_) : self(self_) {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
+    }
+
+    auto introspectImpl =
+        std::move(oi_namespaces) |
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
+        ranges::views::transform([](auto* p) {
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
+        }) |
+        ranges::views::filter([](auto* td) {
+          return td != nullptr && td->getName() == "introspectImpl";
+        }) |
+        ranges::views::take(1) | ranges::to<std::vector>();
+    if (introspectImpl.empty()) {
+      LOG(WARNING)
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
+             "you compile with `OIL_AOT_COMPILATION=1`?";
+      return;
+    }
+
+    auto nameToClangTypeMap =
+        ranges::views::single(introspectImpl[0]) |
+        ranges::views::for_each(
+            [](auto* td) { return td->specializations(); }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
+
+              assert(fd->getNumParams() == 1);
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{self.typeGraph, opts};
+
+    auto& Sema = self.inst.getSema();
+    self.nameToTypeMap =
+        nameToClangTypeMap |
+        ranges::views::transform(
+            [&parser, &Context, &Sema](
+                auto& p) -> std::pair<std::string, type_graph::Type*> {
+              return {p.first, &parser.parse(Context, Sema, *p.second)};
+            }) |
+        ranges::to<std::unordered_map>();
+
+    for (const auto& [name, type] : self.nameToTypeMap)
+      self.typeGraph.addRoot(*type);
+  }
+};
+
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(*this);
+}
+
+}  // namespace
 }  // namespace oi::detail
diff --git a/oi/OIGenerator.h b/oi/OIGenerator.h
index dd38384..586e642 100644
--- a/oi/OIGenerator.h
+++ b/oi/OIGenerator.h
@@ -24,10 +24,13 @@
 #include "oi/OICompiler.h"
 
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate();
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -44,6 +47,9 @@ class OIGenerator {
   void setUsePIC(bool pic_) {
     pic = pic_;
   }
+  void setClangArgs(std::vector<std::string> args_) {
+    clangArgs = std::move(args_);
+  }
 
  private:
   std::filesystem::path outputPath;
@@ -51,19 +57,7 @@ class OIGenerator {
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
   bool pic = false;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
+  std::vector<std::string> clangArgs;
 };
 
 }  // namespace oi::detail
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 940c031..46add03 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
   Flattener.cpp
@@ -26,3 +27,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
new file mode 100644
index 0000000..99f8002
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -0,0 +1,283 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <stdexcept>  // TODO: remove
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (requireCompleteType(*sema, ty))
+    return makeType<Incomplete>(ty, "incomplete (TODO naming)");
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(*ty.getAs<const clang::RecordType>());
+    case clang::Type::LValueReference:
+      return enumerateReference(*ty.getAs<const clang::LValueReferenceType>());
+    case clang::Type::Pointer:
+      return enumeratePointer(*ty.getAs<const clang::PointerType>());
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          *ty.getAs<const clang::SubstTemplateTypeParmType>());
+    case clang::Type::Builtin:
+      return enumeratePrimitive(*ty.getAs<const clang::BuiltinType>());
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(*ty.getAs<const clang::ElaboratedType>());
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          *ty.getAs<const clang::TemplateSpecializationType>());
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Class& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+  params.reserve(list.size());
+
+  for (const auto& arg : list.asArray()) {
+    params.emplace_back(enumerateTemplateParam(arg));
+  }
+}
+
+TemplateParam ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return {ttype, qualifiers};
+    }
+    default:
+      p.dump();
+      throw std::logic_error("unsupported template argument kind");
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function references
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      ty.dump();
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  // TODO: SEGVs if the type can't be completed
+  return sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      0);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
new file mode 100644
index 0000000..13530eb
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.h
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ElaboratedType;
+class LValueReferenceType;
+class RecordType;
+class SubstTemplateTypeParmType;
+class Type;
+class BuiltinType;
+class TemplateArgument;
+class TemplateSpecializationType;
+class PointerType;
+class ASTContext;
+class Sema;
+}  // namespace clang
+
+namespace oi::detail::type_graph {
+
+class Class;
+class Member;
+class Primitive;
+class Reference;
+struct TemplateParam;
+class Type;
+class TypeGraph;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+};
+
+/*
+ * SourceParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given drgn_type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph, ClangTypeParserOptions options)
+      : typeGraph_(typeGraph), options_(options) {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Class& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  TemplateParam enumerateTemplateParam(const clang::TemplateArgument&);
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index c792766..5213b3d 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -194,6 +194,14 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& p) {
+  RecursiveVisitor::visit(p);
+  p.regenerateName();
+  std::string inputName{p.pointeeType().inputName()};
+  inputName += '*';
+  p.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..e6437d0 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index e7d70eb..5913977 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -147,6 +147,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& p) {
+  if (prefix(p))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = p.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(p.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index a66581e..d7d60f3 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& p) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;
diff --git a/oi/type_graph/RemoveTopLevelPointer.cpp b/oi/type_graph/RemoveTopLevelPointer.cpp
index 9ecaaaa..e1909a7 100644
--- a/oi/type_graph/RemoveTopLevelPointer.cpp
+++ b/oi/type_graph/RemoveTopLevelPointer.cpp
@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& p) {
+  topLevelType_ = &p.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/RemoveTopLevelPointer.h b/oi/type_graph/RemoveTopLevelPointer.h
index 611429f..6b70545 100644
--- a/oi/type_graph/RemoveTopLevelPointer.h
+++ b/oi/type_graph/RemoveTopLevelPointer.h
@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
 
  private:
   Type* topLevelType_ = nullptr;
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f503886..8aba760 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -715,6 +716,61 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    name_ = pointeeType_.get().name() + "&";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index 7e4395e..b2fb7a2 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -121,6 +121,9 @@ class RecursiveVisitor : public Visitor {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -182,6 +185,10 @@ class RecursiveMutator : public Mutator {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index d1e345d..87598c3 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -18,127 +18,71 @@
 #include <glog/logging.h>
 
 #include <cstdlib>
+#include <cxxopts.hpp>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/action/drop_while.hpp>
+#include <range/v3/algorithm/find.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/all.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
-
-constexpr static OIOpts opts{
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
-
-void usage() {
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
-  std::cerr << opts;
-
-  std::cerr << std::endl
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
+using namespace ranges;
 
 int main(int argc, char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
-  std::vector<fs::path> configFilePaths;
-  fs::path sourceFileDumpPath = "";
-  bool exitCode = false;
-  bool pic = false;
+  cxxopts::Options options("oilgen",
+                           "Generate OIL object code from an input file");
 
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
+  options.add_options()
+    ("h,help", "Print usage")
+    ("o,output", "Write output(s) to file(s) with this prefix", cxxopts::value<fs::path>()->default_value("a.o"))
+    ("c,config-file", "Path to OI configuration file(s)", cxxopts::value<std::vector<fs::path>>())
+    ("d,debug-level", "Verbose level for logging", cxxopts::value<int>())
+    ("j,dump-jit", "Write generated code to a file (for debugging)", cxxopts::value<fs::path>()->default_value("jit.cpp"))
+    ("e,exit-code", "Return a bad exit code if nothing is generated")
+    ("p,pic", "Generate position independent code")
+      ;
+  options.positional_help("CLANG_ARGS...");
+  options.allow_unrecognised_options();
+
+  auto args = options.parse(argc, argv);
+  if (args.count("help") > 0) {
+    std::cout << options.help() << std::endl;
+    return 0;
   }
 
-  if (optind >= argc) {
-    usage();
-    return EXIT_FAILURE;
-  }
-  fs::path primaryObject = argv[optind];
-
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
+  if (args.count("debug-level") > 0) {
+    int level = args["debug-level"].as<int>();
+    google::LogToStderr();
+    google::SetStderrLogging(google::INFO);
+    google::SetVLOGLevel("*", level);
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
+    gflags::SetCommandLineOption("minloglevel", "0");
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
-  oigen.setConfigFilePaths(std::move(configFilePaths));
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
-  oigen.setFailIfNothingGenerated(exitCode);
-  oigen.setUsePIC(pic);
+  oigen.setOutputPath(args["output"].as<fs::path>());
 
-  SymbolService symbols(primaryObject);
+  oigen.setUsePIC(args["pic"].as<bool>());
+  oigen.setFailIfNothingGenerated(args["exit-code"].as<bool>());
 
-  return oigen.generate(primaryObject, symbols);
+  if (args.count("config-file") > 0)
+    oigen.setConfigFilePaths(args["config-file"].as<std::vector<fs::path>>());
+  if (args.count("dump-jit") > 0)
+    oigen.setSourceFileDumpPath(args["dump-jit"].as<fs::path>());
+
+  oigen.setClangArgs(args.unmatched());
+
+  return oigen.generate();
 }

From d6e6394328f07fd0d70ef4f8c8a84d4c0fe8e3f6 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 5 Dec 2023 08:21:20 -0800
Subject: [PATCH 042/188] oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD
---
 CMakeLists.txt                          |  13 +-
 oi/CodeGen.cpp                          |  30 ++-
 oi/CodeGen.h                            |   8 +-
 oi/Config.cpp                           |   9 +-
 oi/ContainerInfo.cpp                    |   2 +-
 oi/OICodeGen.h                          |   2 +-
 oi/OIGenerator.cpp                      | 339 +++++++++++++-----------
 oi/OIGenerator.h                        |  27 +-
 oi/type_graph/CMakeLists.txt            |   2 +
 oi/type_graph/ClangTypeParser.cpp       | 284 ++++++++++++++++++++
 oi/type_graph/ClangTypeParser.h         | 102 +++++++
 oi/type_graph/NameGen.cpp               |   8 +
 oi/type_graph/NameGen.h                 |   1 +
 oi/type_graph/Printer.cpp               |  11 +
 oi/type_graph/Printer.h                 |   1 +
 oi/type_graph/RemoveTopLevelPointer.cpp |   4 +
 oi/type_graph/RemoveTopLevelPointer.h   |   1 +
 oi/type_graph/Types.h                   |  56 ++++
 oi/type_graph/Visitor.h                 |   7 +
 tools/OILGen.cpp                        | 144 ++++------
 20 files changed, 764 insertions(+), 287 deletions(-)
 create mode 100644 oi/type_graph/ClangTypeParser.cpp
 create mode 100644 oi/type_graph/ClangTypeParser.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f9ee7..eb18f4a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -104,6 +104,15 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(range-v3)
 
+### cxxopts
+FetchContent_Declare(
+  cxxopts
+  GIT_REPOSITORY https://github.com/jarro2783/cxxopts.git
+  GIT_TAG        eb787304d67ec22f7c3a184ee8b4c481d04357fd # v3.1.1
+  GIT_PROGRESS   TRUE
+)
+FetchContent_MakeAvailable(cxxopts)
+
 set_project_warnings()
 
 if (ASAN)
@@ -289,7 +298,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -363,8 +372,10 @@ add_executable(oilgen
   oi/OIGenerator.cpp
 )
 target_link_libraries(oilgen
+  cxxopts
   drgn_utils
   oicore
+  clangTooling
 )
 
 ### Object Introspection cache Printer (OIP)
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index bef102f..a7b59bb 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -539,7 +539,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1106,15 +1106,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
 }
 
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
-  try {
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
+  if (!registerContainers())
     return false;
-  }
 
   TypeGraph typeGraph;
   try {
@@ -1129,6 +1122,20 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
   return true;
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    size_t i = 0;
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1170,7 +1177,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(Flattener::createPass());
@@ -1286,7 +1293,8 @@ void CodeGen::generate(
   code += "\nusing __ROOT_TYPE__ = " + rootType.name() + ";\n";
   code += "} // namespace\n} // namespace OIInternal\n";
 
-  const auto typeName = SymbolService::getTypeName(drgnType);
+  // const auto typeName = SymbolService::getTypeName(drgnType);
+  const std::string typeName{"typeName"};
   if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
   } else {
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 186ec34..fb0ec20 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -40,8 +40,11 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config) : config_(config) {
+    assert(!config.features[Feature::PolymorphicInheritance]);
+  }
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   /*
@@ -53,6 +56,7 @@ class CodeGen {
                        std::string linkageName,
                        std::string& code);
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -66,7 +70,7 @@ class CodeGen {
 
  private:
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
diff --git a/oi/Config.cpp b/oi/Config.cpp
index 157b279..66798bb 100644
--- a/oi/Config.cpp
+++ b/oi/Config.cpp
@@ -22,6 +22,8 @@
 #include <boost/property_tree/ini_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 #include <filesystem>
+#include <range/v3/action/sort.hpp>
+#include <range/v3/action/unique.hpp>
 
 #include "oi/support/Toml.h"
 
@@ -54,6 +56,9 @@ std::optional<FeatureSet> processConfigFiles(
     enables |= *fs;
   }
 
+  ranges::actions::sort(generatorConfig.containerConfigPaths);
+  ranges::actions::unique(generatorConfig.containerConfigPaths);
+
   // Override anything from the config files with command line options
   for (auto [k, v] : featureMap) {
     enables[k] = v;
@@ -108,8 +113,8 @@ std::optional<FeatureSet> processConfigFile(
           if the right path is absolute, else append the right path to the
           left path.
           */
-          generatorConfig.containerConfigPaths.emplace(configDirectory /
-                                                       el.get());
+          generatorConfig.containerConfigPaths.emplace_back(configDirectory /
+                                                            el.get());
         }
       });
     }
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 99a7dc1..dbe10c6 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -202,7 +202,7 @@ std::regex getMatcher(const std::string& typeName) {
 ContainerInfo::ContainerInfo(const fs::path& path) {
   toml::table container;
   try {
-    container = toml::parse_file(std::string(path));
+    container = toml::parse_file(path.string());
   } catch (const toml::parse_error& err) {
     // Convert into a ContainerInfoError, just to avoid having to include
     // the huge TOML++ header in the caller's file. Use toml::parse_error's
diff --git a/oi/OICodeGen.h b/oi/OICodeGen.h
index a01548f..7226a4e 100644
--- a/oi/OICodeGen.h
+++ b/oi/OICodeGen.h
@@ -66,7 +66,7 @@ class OICodeGen {
     };
 
     FeatureSet features;
-    std::set<std::filesystem::path> containerConfigPaths;
+    std::vector<std::filesystem::path> containerConfigPaths;
     std::set<std::string> defaultHeaders;
     std::set<std::string> defaultNamespaces;
     std::vector<std::pair<std::string, std::string>> membersToStub;
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 1cb2a49..e3c196b 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -16,171 +16,98 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
 #include <glog/logging.h>
 
 #include <boost/core/demangle.hpp>
 #include <fstream>
 #include <iostream>
+#include <range/v3/action/transform.hpp>
+#include <range/v3/core.hpp>
+#include <range/v3/view/cache1.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/drop_while.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>  // TODO: remove
 #include <string_view>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
+class ConsumerContext;
 
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
-      templateArgsToSymbolsMap;
-
-  auto symbols = prog.find_all_symbols();
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
+class CreateTypeGraphConsumer;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
-    if (val.first.empty() || val.second.empty()) {
-      continue;
-    }
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  return strongToWeakSymbols;
-}
-
-std::unordered_map<std::string, drgn_qualified_type>
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> out;
-
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
-    std::string strongLinkageName;
-    {
-      const char* linkageNameCstr;
-      if (auto err = drgnplusplus::error(
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
-        // throw err;
-        continue;
-      }
-      strongLinkageName = linkageNameCstr;
-    }
-
-    std::string weakLinkageName;
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
   }
 
-  return out;
-}
+ private:
+  ConsumerContext& ctx;
+};
 
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
-                                      const OICompiler::Config& compilerConfig,
-                                      const drgn_qualified_type& type,
-                                      const std::string& linkageName,
-                                      SymbolService& symbols) {
-  CodeGen codegen{generatorConfig, symbols};
+class ConsumerContext {
+ public:
+  type_graph::TypeGraph typeGraph;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
 
-  std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
 
-  std::string sourcePath = sourceFileDumpPath;
-  if (sourceFileDumpPath.empty()) {
-    // This is the path Clang acts as if it has compiled from e.g. for debug
-    // information. It does not need to exist.
-    sourcePath = "oil_jit.cpp";
-  } else {
-    std::ofstream outputFile(sourcePath);
-    outputFile << code;
-  }
+}  // namespace
 
-  OICompiler compiler{{}, compilerConfig};
+int OIGenerator::generate(clang::tooling::CompilationDatabase& db,
+                          const std::vector<std::string>& sourcePaths) {
+  ConsumerContext ctx;
+  CreateTypeGraphActionFactory factory{ctx};
 
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
+  clang::tooling::ClangTool tool{db, sourcePaths};
+  if (auto ret = tool.run(&factory); ret != 0)
+    return ret;
 
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
-}
-
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
-  drgnplusplus::program prog;
-
-  {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(
-            drgn_program_load_debug_info(prog.get(),
-                                         std::data(objectPaths),
-                                         std::size(objectPaths),
-                                         false,
-                                         false))) {
-      LOG(ERROR) << "error loading debug info program: " << err;
-      throw err;
-    }
-  }
-
-  auto oilTypes = findOilTypesAndNames(prog);
+  if (ctx.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one type to generate for but we can't currently "
+        "handle this case");
 
   std::map<Feature, bool> featuresMap = {
       {Feature::TypeGraph, true},
@@ -203,27 +130,127 @@ int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
   generatorConfig.features = *features;
   compilerConfig.features = *features;
 
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
-    }
-  }
-
-  size_t successes = oilTypes.size() - failures;
-  LOG(INFO) << "object introspection generation complete. " << successes
-            << " successes and " << failures << " failures.";
-
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
+  CodeGen codegen{generatorConfig};
+  if (!codegen.registerContainers()) {
+    LOG(ERROR) << "Failed to register containers";
     return -1;
   }
-  return 0;
+  codegen.transform(ctx.typeGraph);
+
+  std::string code;
+  codegen.generate(ctx.typeGraph, code, nullptr);
+
+  std::string sourcePath = sourceFileDumpPath;
+  if (sourceFileDumpPath.empty()) {
+    // This is the path Clang acts as if it has compiled from e.g. for debug
+    // information. It does not need to exist.
+    sourcePath = "oil_jit.cpp";
+  } else {
+    std::ofstream outputFile(sourcePath);
+    outputFile << code;
+  }
+
+  OICompiler compiler{{}, compilerConfig};
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
 }
 
+namespace {
+
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx(ctx_) {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    LOG(ERROR) << "CreateTypeGraphConsumer::HandleTranslationUnit";
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
+    }
+
+    auto introspectImpl =
+        std::move(oi_namespaces) |
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
+        ranges::views::transform([](auto* p) {
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
+        }) |
+        ranges::views::filter([](auto* td) {
+          return td != nullptr && td->getName() == "introspectImpl";
+        }) |
+        ranges::views::take(1) | ranges::to<std::vector>();
+    if (introspectImpl.empty()) {
+      LOG(WARNING)
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
+             "you compile with `OIL_AOT_COMPILATION=1`?";
+      return;
+    }
+
+    auto nameToClangTypeMap =
+        ranges::views::single(introspectImpl[0]) |
+        ranges::views::for_each(
+            [](auto* td) { return td->specializations(); }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
+
+              assert(fd->getNumParams() == 1);
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{ctx.typeGraph, opts};
+
+    auto& Sema = *ctx.sema;
+    ctx.nameToTypeMap =
+        nameToClangTypeMap |
+        ranges::views::transform(
+            [&parser, &Context, &Sema](
+                auto& p) -> std::pair<std::string, type_graph::Type*> {
+              return {p.first, &parser.parse(Context, Sema, *p.second)};
+            }) |
+        ranges::to<std::unordered_map>();
+
+    for (const auto& [name, type] : ctx.nameToTypeMap)
+      ctx.typeGraph.addRoot(*type);
+  }
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
+  LOG(ERROR) << "left executeaction";
+}
+
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+}  // namespace
 }  // namespace oi::detail
diff --git a/oi/OIGenerator.h b/oi/OIGenerator.h
index dd38384..ca1ef89 100644
--- a/oi/OIGenerator.h
+++ b/oi/OIGenerator.h
@@ -23,11 +23,19 @@
 #include "oi/OICodeGen.h"
 #include "oi/OICompiler.h"
 
+namespace clang::tooling {
+class CompilationDatabase;
+}
+
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate(clang::tooling::CompilationDatabase&,
+               const std::vector<std::string>&);
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -44,6 +52,9 @@ class OIGenerator {
   void setUsePIC(bool pic_) {
     pic = pic_;
   }
+  void setClangArgs(std::vector<std::string> args_) {
+    clangArgs = std::move(args_);
+  }
 
  private:
   std::filesystem::path outputPath;
@@ -51,19 +62,7 @@ class OIGenerator {
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
   bool pic = false;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
+  std::vector<std::string> clangArgs;
 };
 
 }  // namespace oi::detail
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 940c031..46add03 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
   Flattener.cpp
@@ -26,3 +27,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
new file mode 100644
index 0000000..1bc60f1
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -0,0 +1,284 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Basic/DiagnosticSema.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <stdexcept>  // TODO: remove
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (requireCompleteType(*sema, ty))
+    return makeType<Incomplete>(ty, "incomplete (TODO naming)");
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(*ty.getAs<const clang::RecordType>());
+    case clang::Type::LValueReference:
+      return enumerateReference(*ty.getAs<const clang::LValueReferenceType>());
+    case clang::Type::Pointer:
+      return enumeratePointer(*ty.getAs<const clang::PointerType>());
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          *ty.getAs<const clang::SubstTemplateTypeParmType>());
+    case clang::Type::Builtin:
+      return enumeratePrimitive(*ty.getAs<const clang::BuiltinType>());
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(*ty.getAs<const clang::ElaboratedType>());
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          *ty.getAs<const clang::TemplateSpecializationType>());
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Class& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+  params.reserve(list.size());
+
+  for (const auto& arg : list.asArray()) {
+    params.emplace_back(enumerateTemplateParam(arg));
+  }
+}
+
+TemplateParam ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return {ttype, qualifiers};
+    }
+    default:
+      p.dump();
+      throw std::logic_error("unsupported template argument kind");
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function references
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      ty.dump();
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  // TODO: SEGVs if the type can't be completed
+  return sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      clang::diag::err_type_unsupported);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
new file mode 100644
index 0000000..13530eb
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.h
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ElaboratedType;
+class LValueReferenceType;
+class RecordType;
+class SubstTemplateTypeParmType;
+class Type;
+class BuiltinType;
+class TemplateArgument;
+class TemplateSpecializationType;
+class PointerType;
+class ASTContext;
+class Sema;
+}  // namespace clang
+
+namespace oi::detail::type_graph {
+
+class Class;
+class Member;
+class Primitive;
+class Reference;
+struct TemplateParam;
+class Type;
+class TypeGraph;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+};
+
+/*
+ * SourceParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given drgn_type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph, ClangTypeParserOptions options)
+      : typeGraph_(typeGraph), options_(options) {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Class& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  TemplateParam enumerateTemplateParam(const clang::TemplateArgument&);
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index c792766..5213b3d 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -194,6 +194,14 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& p) {
+  RecursiveVisitor::visit(p);
+  p.regenerateName();
+  std::string inputName{p.pointeeType().inputName()};
+  inputName += '*';
+  p.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..e6437d0 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index e7d70eb..5913977 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -147,6 +147,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& p) {
+  if (prefix(p))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = p.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(p.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index a66581e..d7d60f3 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& p) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;
diff --git a/oi/type_graph/RemoveTopLevelPointer.cpp b/oi/type_graph/RemoveTopLevelPointer.cpp
index 9ecaaaa..e1909a7 100644
--- a/oi/type_graph/RemoveTopLevelPointer.cpp
+++ b/oi/type_graph/RemoveTopLevelPointer.cpp
@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& p) {
+  topLevelType_ = &p.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/RemoveTopLevelPointer.h b/oi/type_graph/RemoveTopLevelPointer.h
index 611429f..6b70545 100644
--- a/oi/type_graph/RemoveTopLevelPointer.h
+++ b/oi/type_graph/RemoveTopLevelPointer.h
@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
 
  private:
   Type* topLevelType_ = nullptr;
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f503886..8aba760 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -715,6 +716,61 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    name_ = pointeeType_.get().name() + "&";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index 7e4395e..b2fb7a2 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -121,6 +121,9 @@ class RecursiveVisitor : public Visitor {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -182,6 +185,10 @@ class RecursiveMutator : public Mutator {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index d1e345d..5e5a000 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -14,131 +14,77 @@
  * limitations under the License.
  */
 
+#include <clang/Tooling/CommonOptionsParser.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include <cstdlib>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/action/drop_while.hpp>
+#include <range/v3/algorithm/find.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/all.hpp>
+#include <range/v3/view/transform.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
 
-constexpr static OIOpts opts{
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
+static llvm::cl::OptionCategory OilgenCategory("oilgen options");
 
-void usage() {
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
-  std::cerr << opts;
+static llvm::cl::list<std::string> ConfigFiles("config-file",
+                                               llvm::cl::desc(R"(TODO HELP)"),
+                                               llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<int> DebugLevel("debug-level",
+                                     llvm::cl::desc(R"(TODO HELP)"),
+                                     llvm::cl::init(-1),
+                                     llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> DumpJit("dump-jit",
+                                          llvm::cl::desc(R"(TODO HELP)"),
+                                          llvm::cl::init("jit.cpp"),
+                                          llvm::cl::cat(OilgenCategory));
 
-  std::cerr << std::endl
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
-
-int main(int argc, char* argv[]) {
+int main(int argc, const char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
-  std::vector<fs::path> configFilePaths;
-  fs::path sourceFileDumpPath = "";
-  bool exitCode = false;
-  bool pic = false;
-
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
+  auto expectedParser =
+      clang::tooling::CommonOptionsParser::create(argc, argv, OilgenCategory);
+  if (!expectedParser) {
+    llvm::errs() << expectedParser.takeError();
+    return -1;
   }
+  clang::tooling::CommonOptionsParser& options = expectedParser.get();
 
-  if (optind >= argc) {
-    usage();
-    return EXIT_FAILURE;
-  }
-  fs::path primaryObject = argv[optind];
-
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
+  if (DebugLevel.getNumOccurrences()) {
+    google::LogToStderr();
+    google::SetStderrLogging(google::INFO);
+    google::SetVLOGLevel("*", DebugLevel);
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
+    gflags::SetCommandLineOption("minloglevel", "0");
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
-  oigen.setConfigFilePaths(std::move(configFilePaths));
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
-  oigen.setFailIfNothingGenerated(exitCode);
-  oigen.setUsePIC(pic);
+  oigen.setConfigFilePaths(ConfigFiles |
+                           ranges::views::transform([](const auto& p) {
+                             return std::filesystem::path(p);
+                           }) |
+                           ranges::to<std::vector>());
+  if (DumpJit.getNumOccurrences())
+    oigen.setSourceFileDumpPath(DumpJit.getValue());
 
-  SymbolService symbols(primaryObject);
+  oigen.setOutputPath("a.o");
 
-  return oigen.generate(primaryObject, symbols);
+  // TODO: can we extract this from the original arguments?
+  // oigen.setUsePIC(args["pic"].as<bool>());
+
+  oigen.setFailIfNothingGenerated(true);
+  return oigen.generate(options.getCompilations(), options.getSourcePathList());
 }

From cf607afa54c59fa92786e2fb87ffee2795897113 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 5 Dec 2023 08:21:20 -0800
Subject: [PATCH 043/188] oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD
---
 CMakeLists.txt                          |  13 +-
 oi/CodeGen.cpp                          |  30 +-
 oi/CodeGen.h                            |   8 +-
 oi/Config.cpp                           |   9 +-
 oi/ContainerInfo.cpp                    |   2 +-
 oi/OICodeGen.h                          |   2 +-
 oi/OIGenerator.cpp                      | 357 +++++++++++++-----------
 oi/OIGenerator.h                        |  29 +-
 oi/type_graph/CMakeLists.txt            |   2 +
 oi/type_graph/ClangTypeParser.cpp       | 284 +++++++++++++++++++
 oi/type_graph/ClangTypeParser.h         | 102 +++++++
 oi/type_graph/NameGen.cpp               |   8 +
 oi/type_graph/NameGen.h                 |   1 +
 oi/type_graph/Printer.cpp               |  11 +
 oi/type_graph/Printer.h                 |   1 +
 oi/type_graph/RemoveTopLevelPointer.cpp |   4 +
 oi/type_graph/RemoveTopLevelPointer.h   |   1 +
 oi/type_graph/Types.h                   |  56 ++++
 oi/type_graph/Visitor.h                 |   7 +
 tools/OILGen.cpp                        | 144 +++-------
 20 files changed, 775 insertions(+), 296 deletions(-)
 create mode 100644 oi/type_graph/ClangTypeParser.cpp
 create mode 100644 oi/type_graph/ClangTypeParser.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f9ee7..eb18f4a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -104,6 +104,15 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(range-v3)
 
+### cxxopts
+FetchContent_Declare(
+  cxxopts
+  GIT_REPOSITORY https://github.com/jarro2783/cxxopts.git
+  GIT_TAG        eb787304d67ec22f7c3a184ee8b4c481d04357fd # v3.1.1
+  GIT_PROGRESS   TRUE
+)
+FetchContent_MakeAvailable(cxxopts)
+
 set_project_warnings()
 
 if (ASAN)
@@ -289,7 +298,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -363,8 +372,10 @@ add_executable(oilgen
   oi/OIGenerator.cpp
 )
 target_link_libraries(oilgen
+  cxxopts
   drgn_utils
   oicore
+  clangTooling
 )
 
 ### Object Introspection cache Printer (OIP)
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index bef102f..a7b59bb 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -539,7 +539,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1106,15 +1106,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
 }
 
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
-  try {
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
+  if (!registerContainers())
     return false;
-  }
 
   TypeGraph typeGraph;
   try {
@@ -1129,6 +1122,20 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
   return true;
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    size_t i = 0;
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1170,7 +1177,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(Flattener::createPass());
@@ -1286,7 +1293,8 @@ void CodeGen::generate(
   code += "\nusing __ROOT_TYPE__ = " + rootType.name() + ";\n";
   code += "} // namespace\n} // namespace OIInternal\n";
 
-  const auto typeName = SymbolService::getTypeName(drgnType);
+  // const auto typeName = SymbolService::getTypeName(drgnType);
+  const std::string typeName{"typeName"};
   if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
   } else {
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 186ec34..fb0ec20 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -40,8 +40,11 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config) : config_(config) {
+    assert(!config.features[Feature::PolymorphicInheritance]);
+  }
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   /*
@@ -53,6 +56,7 @@ class CodeGen {
                        std::string linkageName,
                        std::string& code);
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -66,7 +70,7 @@ class CodeGen {
 
  private:
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
diff --git a/oi/Config.cpp b/oi/Config.cpp
index 157b279..66798bb 100644
--- a/oi/Config.cpp
+++ b/oi/Config.cpp
@@ -22,6 +22,8 @@
 #include <boost/property_tree/ini_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 #include <filesystem>
+#include <range/v3/action/sort.hpp>
+#include <range/v3/action/unique.hpp>
 
 #include "oi/support/Toml.h"
 
@@ -54,6 +56,9 @@ std::optional<FeatureSet> processConfigFiles(
     enables |= *fs;
   }
 
+  ranges::actions::sort(generatorConfig.containerConfigPaths);
+  ranges::actions::unique(generatorConfig.containerConfigPaths);
+
   // Override anything from the config files with command line options
   for (auto [k, v] : featureMap) {
     enables[k] = v;
@@ -108,8 +113,8 @@ std::optional<FeatureSet> processConfigFile(
           if the right path is absolute, else append the right path to the
           left path.
           */
-          generatorConfig.containerConfigPaths.emplace(configDirectory /
-                                                       el.get());
+          generatorConfig.containerConfigPaths.emplace_back(configDirectory /
+                                                            el.get());
         }
       });
     }
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 99a7dc1..dbe10c6 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -202,7 +202,7 @@ std::regex getMatcher(const std::string& typeName) {
 ContainerInfo::ContainerInfo(const fs::path& path) {
   toml::table container;
   try {
-    container = toml::parse_file(std::string(path));
+    container = toml::parse_file(path.string());
   } catch (const toml::parse_error& err) {
     // Convert into a ContainerInfoError, just to avoid having to include
     // the huge TOML++ header in the caller's file. Use toml::parse_error's
diff --git a/oi/OICodeGen.h b/oi/OICodeGen.h
index a01548f..7226a4e 100644
--- a/oi/OICodeGen.h
+++ b/oi/OICodeGen.h
@@ -66,7 +66,7 @@ class OICodeGen {
     };
 
     FeatureSet features;
-    std::set<std::filesystem::path> containerConfigPaths;
+    std::vector<std::filesystem::path> containerConfigPaths;
     std::set<std::string> defaultHeaders;
     std::set<std::string> defaultNamespaces;
     std::vector<std::pair<std::string, std::string>> membersToStub;
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 1cb2a49..2affcef 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -16,130 +16,130 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
 #include <glog/logging.h>
 
 #include <boost/core/demangle.hpp>
 #include <fstream>
 #include <iostream>
+#include <range/v3/action/transform.hpp>
+#include <range/v3/core.hpp>
+#include <range/v3/view/cache1.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/drop_while.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>  // TODO: remove
 #include <string_view>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
+class ConsumerContext;
 
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
-      templateArgsToSymbolsMap;
-
-  auto symbols = prog.find_all_symbols();
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
+class CreateTypeGraphConsumer;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
-    if (val.first.empty() || val.second.empty()) {
-      continue;
-    }
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  return strongToWeakSymbols;
-}
-
-std::unordered_map<std::string, drgn_qualified_type>
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> out;
-
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
-    std::string strongLinkageName;
-    {
-      const char* linkageNameCstr;
-      if (auto err = drgnplusplus::error(
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
-        // throw err;
-        continue;
-      }
-      strongLinkageName = linkageNameCstr;
-    }
-
-    std::string weakLinkageName;
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
   }
 
-  return out;
-}
+ private:
+  ConsumerContext& ctx;
+};
 
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
-                                      const OICompiler::Config& compilerConfig,
-                                      const drgn_qualified_type& type,
-                                      const std::string& linkageName,
-                                      SymbolService& symbols) {
-  CodeGen codegen{generatorConfig, symbols};
+class ConsumerContext {
+ public:
+  type_graph::TypeGraph typeGraph;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
+  std::optional<bool> pic;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+}  // namespace
+
+int OIGenerator::generate(clang::tooling::CompilationDatabase& db,
+                          const std::vector<std::string>& sourcePaths) {
+  ConsumerContext ctx;
+  CreateTypeGraphActionFactory factory{ctx};
+
+  clang::tooling::ClangTool tool{db, sourcePaths};
+  if (auto ret = tool.run(&factory); ret != 0)
+    return ret;
+
+  if (ctx.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one type to generate for but we can't currently "
+        "handle this case");
+
+  std::map<Feature, bool> featuresMap = {
+      {Feature::TypeGraph, true},
+      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},
+      {Feature::PackStructs, true},
+      {Feature::PruneTypeGraph, true},
+  };
+
+  OICodeGen::Config generatorConfig{};
+  OICompiler::Config compilerConfig{};
+  compilerConfig.usePIC = ctx.pic.value();
+
+  auto features = config::processConfigFiles(
+      configFilePaths, featuresMap, compilerConfig, generatorConfig);
+  if (!features) {
+    LOG(ERROR) << "failed to process config file";
+    return -1;
+  }
+  generatorConfig.features = *features;
+  compilerConfig.features = *features;
+
+  CodeGen codegen{generatorConfig};
+  if (!codegen.registerContainers()) {
+    LOG(ERROR) << "Failed to register containers";
+    return -1;
+  }
+  codegen.transform(ctx.typeGraph);
 
   std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
+  codegen.generate(ctx.typeGraph, code, nullptr);
 
   std::string sourcePath = sourceFileDumpPath;
   if (sourceFileDumpPath.empty()) {
@@ -152,78 +152,111 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
   }
 
   OICompiler compiler{{}, compilerConfig};
-
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
-
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
 }
 
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
-  drgnplusplus::program prog;
+namespace {
 
-  {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(
-            drgn_program_load_debug_info(prog.get(),
-                                         std::data(objectPaths),
-                                         std::size(objectPaths),
-                                         false,
-                                         false))) {
-      LOG(ERROR) << "error loading debug info program: " << err;
-      throw err;
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx(ctx_) {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    LOG(ERROR) << "CreateTypeGraphConsumer::HandleTranslationUnit";
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
     }
-  }
 
-  auto oilTypes = findOilTypesAndNames(prog);
-
-  std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
-      {Feature::PruneTypeGraph, true},
-  };
-
-  OICodeGen::Config generatorConfig{};
-  OICompiler::Config compilerConfig{};
-  compilerConfig.usePIC = pic;
-
-  auto features = config::processConfigFiles(
-      configFilePaths, featuresMap, compilerConfig, generatorConfig);
-  if (!features) {
-    LOG(ERROR) << "failed to process config file";
-    return -1;
-  }
-  generatorConfig.features = *features;
-  compilerConfig.features = *features;
-
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
+    auto introspectImpl =
+        std::move(oi_namespaces) |
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
+        ranges::views::transform([](auto* p) {
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
+        }) |
+        ranges::views::filter([](auto* td) {
+          return td != nullptr && td->getName() == "introspectImpl";
+        }) |
+        ranges::views::take(1) | ranges::to<std::vector>();
+    if (introspectImpl.empty()) {
+      LOG(WARNING)
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
+             "you compile with `OIL_AOT_COMPILATION=1`?";
+      return;
     }
-  }
 
-  size_t successes = oilTypes.size() - failures;
-  LOG(INFO) << "object introspection generation complete. " << successes
-            << " successes and " << failures << " failures.";
+    auto nameToClangTypeMap =
+        ranges::views::single(introspectImpl[0]) |
+        ranges::views::for_each(
+            [](auto* td) { return td->specializations(); }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
 
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
-    return -1;
+              assert(fd->getNumParams() == 1);
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{ctx.typeGraph, opts};
+
+    auto& Sema = *ctx.sema;
+    ctx.nameToTypeMap =
+        nameToClangTypeMap |
+        ranges::views::transform(
+            [&parser, &Context, &Sema](
+                auto& p) -> std::pair<std::string, type_graph::Type*> {
+              return {p.first, &parser.parse(Context, Sema, *p.second)};
+            }) |
+        ranges::to<std::unordered_map>();
+
+    for (const auto& [name, type] : ctx.nameToTypeMap)
+      ctx.typeGraph.addRoot(*type);
   }
-  return 0;
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  // Compile the output as position independent if any input is position
+  // independent
+  bool pic = CI.getCodeGenOpts().RelocationModel == llvm::Reloc::PIC_;
+  ctx.pic = ctx.pic.value_or(false) || pic;
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
 }
 
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+}  // namespace
 }  // namespace oi::detail
diff --git a/oi/OIGenerator.h b/oi/OIGenerator.h
index dd38384..fac947a 100644
--- a/oi/OIGenerator.h
+++ b/oi/OIGenerator.h
@@ -23,11 +23,19 @@
 #include "oi/OICodeGen.h"
 #include "oi/OICompiler.h"
 
+namespace clang::tooling {
+class CompilationDatabase;
+}
+
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate(clang::tooling::CompilationDatabase&,
+               const std::vector<std::string>&);
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -41,8 +49,8 @@ class OIGenerator {
   void setFailIfNothingGenerated(bool fail) {
     failIfNothingGenerated = fail;
   }
-  void setUsePIC(bool pic_) {
-    pic = pic_;
+  void setClangArgs(std::vector<std::string> args_) {
+    clangArgs = std::move(args_);
   }
 
  private:
@@ -50,20 +58,7 @@ class OIGenerator {
   std::vector<std::filesystem::path> configFilePaths;
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
-  bool pic = false;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
+  std::vector<std::string> clangArgs;
 };
 
 }  // namespace oi::detail
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 940c031..46add03 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
   Flattener.cpp
@@ -26,3 +27,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
new file mode 100644
index 0000000..1bc60f1
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -0,0 +1,284 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Basic/DiagnosticSema.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <stdexcept>  // TODO: remove
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (requireCompleteType(*sema, ty))
+    return makeType<Incomplete>(ty, "incomplete (TODO naming)");
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(*ty.getAs<const clang::RecordType>());
+    case clang::Type::LValueReference:
+      return enumerateReference(*ty.getAs<const clang::LValueReferenceType>());
+    case clang::Type::Pointer:
+      return enumeratePointer(*ty.getAs<const clang::PointerType>());
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          *ty.getAs<const clang::SubstTemplateTypeParmType>());
+    case clang::Type::Builtin:
+      return enumeratePrimitive(*ty.getAs<const clang::BuiltinType>());
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(*ty.getAs<const clang::ElaboratedType>());
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          *ty.getAs<const clang::TemplateSpecializationType>());
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Class& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+  params.reserve(list.size());
+
+  for (const auto& arg : list.asArray()) {
+    params.emplace_back(enumerateTemplateParam(arg));
+  }
+}
+
+TemplateParam ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return {ttype, qualifiers};
+    }
+    default:
+      p.dump();
+      throw std::logic_error("unsupported template argument kind");
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function references
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      ty.dump();
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  // TODO: SEGVs if the type can't be completed
+  return sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      clang::diag::err_type_unsupported);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
new file mode 100644
index 0000000..13530eb
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.h
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ElaboratedType;
+class LValueReferenceType;
+class RecordType;
+class SubstTemplateTypeParmType;
+class Type;
+class BuiltinType;
+class TemplateArgument;
+class TemplateSpecializationType;
+class PointerType;
+class ASTContext;
+class Sema;
+}  // namespace clang
+
+namespace oi::detail::type_graph {
+
+class Class;
+class Member;
+class Primitive;
+class Reference;
+struct TemplateParam;
+class Type;
+class TypeGraph;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+};
+
+/*
+ * SourceParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given drgn_type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph, ClangTypeParserOptions options)
+      : typeGraph_(typeGraph), options_(options) {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Class& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  TemplateParam enumerateTemplateParam(const clang::TemplateArgument&);
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index c792766..5213b3d 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -194,6 +194,14 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& p) {
+  RecursiveVisitor::visit(p);
+  p.regenerateName();
+  std::string inputName{p.pointeeType().inputName()};
+  inputName += '*';
+  p.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..e6437d0 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index e7d70eb..5913977 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -147,6 +147,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& p) {
+  if (prefix(p))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = p.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(p.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index a66581e..d7d60f3 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& p) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;
diff --git a/oi/type_graph/RemoveTopLevelPointer.cpp b/oi/type_graph/RemoveTopLevelPointer.cpp
index 9ecaaaa..e1909a7 100644
--- a/oi/type_graph/RemoveTopLevelPointer.cpp
+++ b/oi/type_graph/RemoveTopLevelPointer.cpp
@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& p) {
+  topLevelType_ = &p.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/RemoveTopLevelPointer.h b/oi/type_graph/RemoveTopLevelPointer.h
index 611429f..6b70545 100644
--- a/oi/type_graph/RemoveTopLevelPointer.h
+++ b/oi/type_graph/RemoveTopLevelPointer.h
@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
 
  private:
   Type* topLevelType_ = nullptr;
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f503886..8aba760 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -715,6 +716,61 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    name_ = pointeeType_.get().name() + "&";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index 7e4395e..b2fb7a2 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -121,6 +121,9 @@ class RecursiveVisitor : public Visitor {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -182,6 +185,10 @@ class RecursiveMutator : public Mutator {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index d1e345d..e8b8859 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -14,131 +14,77 @@
  * limitations under the License.
  */
 
+#include <clang/Tooling/CommonOptionsParser.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include <cstdlib>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/action/drop_while.hpp>
+#include <range/v3/algorithm/find.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/all.hpp>
+#include <range/v3/view/transform.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
 
-constexpr static OIOpts opts{
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
+static llvm::cl::OptionCategory OilgenCategory("oilgen options");
 
-void usage() {
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
-  std::cerr << opts;
+static llvm::cl::list<std::string> ConfigFiles("config-file",
+                                               llvm::cl::desc(R"(TODO HELP)"),
+                                               llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<int> DebugLevel("debug-level",
+                                     llvm::cl::desc(R"(TODO HELP)"),
+                                     llvm::cl::init(-1),
+                                     llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> DumpJit("dump-jit",
+                                          llvm::cl::desc(R"(TODO HELP)"),
+                                          llvm::cl::init("jit.cpp"),
+                                          llvm::cl::cat(OilgenCategory));
 
-  std::cerr << std::endl
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
-
-int main(int argc, char* argv[]) {
+int main(int argc, const char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
-  std::vector<fs::path> configFilePaths;
-  fs::path sourceFileDumpPath = "";
-  bool exitCode = false;
-  bool pic = false;
-
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
+  auto expectedParser =
+      clang::tooling::CommonOptionsParser::create(argc, argv, OilgenCategory);
+  if (!expectedParser) {
+    llvm::errs() << expectedParser.takeError();
+    return -1;
   }
+  clang::tooling::CommonOptionsParser& options = expectedParser.get();
 
-  if (optind >= argc) {
-    usage();
-    return EXIT_FAILURE;
-  }
-  fs::path primaryObject = argv[optind];
-
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
+  if (DebugLevel.getNumOccurrences()) {
+    google::LogToStderr();
+    google::SetStderrLogging(google::INFO);
+    google::SetVLOGLevel("*", DebugLevel);
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
+    gflags::SetCommandLineOption("minloglevel", "0");
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
-  oigen.setConfigFilePaths(std::move(configFilePaths));
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
-  oigen.setFailIfNothingGenerated(exitCode);
-  oigen.setUsePIC(pic);
+  oigen.setConfigFilePaths(ConfigFiles |
+                           ranges::views::transform([](const auto& p) {
+                             return std::filesystem::path(p);
+                           }) |
+                           ranges::to<std::vector>());
+  if (DumpJit.getNumOccurrences())
+    oigen.setSourceFileDumpPath(DumpJit.getValue());
 
-  SymbolService symbols(primaryObject);
+  oigen.setOutputPath("a.o");  // TODO: make this an opt
 
-  return oigen.generate(primaryObject, symbols);
+  // TODO: can we extract this from the original arguments?
+  // oigen.setUsePIC(args["pic"].as<bool>());
+
+  oigen.setFailIfNothingGenerated(true);
+  return oigen.generate(options.getCompilations(), options.getSourcePathList());
 }

From c88dcd4bd50f32f8757f1983300860d2fb891397 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Mon, 11 Dec 2023 06:12:27 -0800
Subject: [PATCH 044/188] oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD
---
 CMakeLists.txt                          |  13 +-
 oi/CodeGen.cpp                          |  30 +-
 oi/CodeGen.h                            |   8 +-
 oi/Config.cpp                           |   9 +-
 oi/ContainerInfo.cpp                    |   2 +-
 oi/OICodeGen.h                          |   2 +-
 oi/OIGenerator.cpp                      | 357 +++++++++++++-----------
 oi/OIGenerator.h                        |  29 +-
 oi/type_graph/CMakeLists.txt            |   2 +
 oi/type_graph/ClangTypeParser.cpp       | 301 ++++++++++++++++++++
 oi/type_graph/ClangTypeParser.h         | 108 +++++++
 oi/type_graph/NameGen.cpp               |   8 +
 oi/type_graph/NameGen.h                 |   1 +
 oi/type_graph/Printer.cpp               |  11 +
 oi/type_graph/Printer.h                 |   1 +
 oi/type_graph/RemoveTopLevelPointer.cpp |   4 +
 oi/type_graph/RemoveTopLevelPointer.h   |   1 +
 oi/type_graph/Types.h                   |  56 ++++
 oi/type_graph/Visitor.h                 |   7 +
 tools/OILGen.cpp                        | 147 ++++------
 types/string_type.toml                  |  31 ++
 21 files changed, 832 insertions(+), 296 deletions(-)
 create mode 100644 oi/type_graph/ClangTypeParser.cpp
 create mode 100644 oi/type_graph/ClangTypeParser.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f9ee7..eb18f4a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -104,6 +104,15 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(range-v3)
 
+### cxxopts
+FetchContent_Declare(
+  cxxopts
+  GIT_REPOSITORY https://github.com/jarro2783/cxxopts.git
+  GIT_TAG        eb787304d67ec22f7c3a184ee8b4c481d04357fd # v3.1.1
+  GIT_PROGRESS   TRUE
+)
+FetchContent_MakeAvailable(cxxopts)
+
 set_project_warnings()
 
 if (ASAN)
@@ -289,7 +298,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -363,8 +372,10 @@ add_executable(oilgen
   oi/OIGenerator.cpp
 )
 target_link_libraries(oilgen
+  cxxopts
   drgn_utils
   oicore
+  clangTooling
 )
 
 ### Object Introspection cache Printer (OIP)
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index bef102f..a7b59bb 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -539,7 +539,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1106,15 +1106,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
 }
 
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
-  try {
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
+  if (!registerContainers())
     return false;
-  }
 
   TypeGraph typeGraph;
   try {
@@ -1129,6 +1122,20 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
   return true;
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    size_t i = 0;
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1170,7 +1177,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(Flattener::createPass());
@@ -1286,7 +1293,8 @@ void CodeGen::generate(
   code += "\nusing __ROOT_TYPE__ = " + rootType.name() + ";\n";
   code += "} // namespace\n} // namespace OIInternal\n";
 
-  const auto typeName = SymbolService::getTypeName(drgnType);
+  // const auto typeName = SymbolService::getTypeName(drgnType);
+  const std::string typeName{"typeName"};
   if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
   } else {
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 186ec34..fb0ec20 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -40,8 +40,11 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config) : config_(config) {
+    assert(!config.features[Feature::PolymorphicInheritance]);
+  }
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   /*
@@ -53,6 +56,7 @@ class CodeGen {
                        std::string linkageName,
                        std::string& code);
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -66,7 +70,7 @@ class CodeGen {
 
  private:
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
diff --git a/oi/Config.cpp b/oi/Config.cpp
index 157b279..66798bb 100644
--- a/oi/Config.cpp
+++ b/oi/Config.cpp
@@ -22,6 +22,8 @@
 #include <boost/property_tree/ini_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 #include <filesystem>
+#include <range/v3/action/sort.hpp>
+#include <range/v3/action/unique.hpp>
 
 #include "oi/support/Toml.h"
 
@@ -54,6 +56,9 @@ std::optional<FeatureSet> processConfigFiles(
     enables |= *fs;
   }
 
+  ranges::actions::sort(generatorConfig.containerConfigPaths);
+  ranges::actions::unique(generatorConfig.containerConfigPaths);
+
   // Override anything from the config files with command line options
   for (auto [k, v] : featureMap) {
     enables[k] = v;
@@ -108,8 +113,8 @@ std::optional<FeatureSet> processConfigFile(
           if the right path is absolute, else append the right path to the
           left path.
           */
-          generatorConfig.containerConfigPaths.emplace(configDirectory /
-                                                       el.get());
+          generatorConfig.containerConfigPaths.emplace_back(configDirectory /
+                                                            el.get());
         }
       });
     }
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 99a7dc1..dbe10c6 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -202,7 +202,7 @@ std::regex getMatcher(const std::string& typeName) {
 ContainerInfo::ContainerInfo(const fs::path& path) {
   toml::table container;
   try {
-    container = toml::parse_file(std::string(path));
+    container = toml::parse_file(path.string());
   } catch (const toml::parse_error& err) {
     // Convert into a ContainerInfoError, just to avoid having to include
     // the huge TOML++ header in the caller's file. Use toml::parse_error's
diff --git a/oi/OICodeGen.h b/oi/OICodeGen.h
index a01548f..7226a4e 100644
--- a/oi/OICodeGen.h
+++ b/oi/OICodeGen.h
@@ -66,7 +66,7 @@ class OICodeGen {
     };
 
     FeatureSet features;
-    std::set<std::filesystem::path> containerConfigPaths;
+    std::vector<std::filesystem::path> containerConfigPaths;
     std::set<std::string> defaultHeaders;
     std::set<std::string> defaultNamespaces;
     std::vector<std::pair<std::string, std::string>> membersToStub;
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 1cb2a49..2affcef 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -16,130 +16,130 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
 #include <glog/logging.h>
 
 #include <boost/core/demangle.hpp>
 #include <fstream>
 #include <iostream>
+#include <range/v3/action/transform.hpp>
+#include <range/v3/core.hpp>
+#include <range/v3/view/cache1.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/drop_while.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>  // TODO: remove
 #include <string_view>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
+class ConsumerContext;
 
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
-      templateArgsToSymbolsMap;
-
-  auto symbols = prog.find_all_symbols();
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
+class CreateTypeGraphConsumer;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
-    if (val.first.empty() || val.second.empty()) {
-      continue;
-    }
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  return strongToWeakSymbols;
-}
-
-std::unordered_map<std::string, drgn_qualified_type>
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> out;
-
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
-    std::string strongLinkageName;
-    {
-      const char* linkageNameCstr;
-      if (auto err = drgnplusplus::error(
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
-        // throw err;
-        continue;
-      }
-      strongLinkageName = linkageNameCstr;
-    }
-
-    std::string weakLinkageName;
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
   }
 
-  return out;
-}
+ private:
+  ConsumerContext& ctx;
+};
 
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
-                                      const OICompiler::Config& compilerConfig,
-                                      const drgn_qualified_type& type,
-                                      const std::string& linkageName,
-                                      SymbolService& symbols) {
-  CodeGen codegen{generatorConfig, symbols};
+class ConsumerContext {
+ public:
+  type_graph::TypeGraph typeGraph;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
+  std::optional<bool> pic;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+}  // namespace
+
+int OIGenerator::generate(clang::tooling::CompilationDatabase& db,
+                          const std::vector<std::string>& sourcePaths) {
+  ConsumerContext ctx;
+  CreateTypeGraphActionFactory factory{ctx};
+
+  clang::tooling::ClangTool tool{db, sourcePaths};
+  if (auto ret = tool.run(&factory); ret != 0)
+    return ret;
+
+  if (ctx.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one type to generate for but we can't currently "
+        "handle this case");
+
+  std::map<Feature, bool> featuresMap = {
+      {Feature::TypeGraph, true},
+      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},
+      {Feature::PackStructs, true},
+      {Feature::PruneTypeGraph, true},
+  };
+
+  OICodeGen::Config generatorConfig{};
+  OICompiler::Config compilerConfig{};
+  compilerConfig.usePIC = ctx.pic.value();
+
+  auto features = config::processConfigFiles(
+      configFilePaths, featuresMap, compilerConfig, generatorConfig);
+  if (!features) {
+    LOG(ERROR) << "failed to process config file";
+    return -1;
+  }
+  generatorConfig.features = *features;
+  compilerConfig.features = *features;
+
+  CodeGen codegen{generatorConfig};
+  if (!codegen.registerContainers()) {
+    LOG(ERROR) << "Failed to register containers";
+    return -1;
+  }
+  codegen.transform(ctx.typeGraph);
 
   std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
+  codegen.generate(ctx.typeGraph, code, nullptr);
 
   std::string sourcePath = sourceFileDumpPath;
   if (sourceFileDumpPath.empty()) {
@@ -152,78 +152,111 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
   }
 
   OICompiler compiler{{}, compilerConfig};
-
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
-
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
 }
 
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
-  drgnplusplus::program prog;
+namespace {
 
-  {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(
-            drgn_program_load_debug_info(prog.get(),
-                                         std::data(objectPaths),
-                                         std::size(objectPaths),
-                                         false,
-                                         false))) {
-      LOG(ERROR) << "error loading debug info program: " << err;
-      throw err;
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx(ctx_) {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    LOG(ERROR) << "CreateTypeGraphConsumer::HandleTranslationUnit";
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
     }
-  }
 
-  auto oilTypes = findOilTypesAndNames(prog);
-
-  std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
-      {Feature::PruneTypeGraph, true},
-  };
-
-  OICodeGen::Config generatorConfig{};
-  OICompiler::Config compilerConfig{};
-  compilerConfig.usePIC = pic;
-
-  auto features = config::processConfigFiles(
-      configFilePaths, featuresMap, compilerConfig, generatorConfig);
-  if (!features) {
-    LOG(ERROR) << "failed to process config file";
-    return -1;
-  }
-  generatorConfig.features = *features;
-  compilerConfig.features = *features;
-
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
+    auto introspectImpl =
+        std::move(oi_namespaces) |
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
+        ranges::views::transform([](auto* p) {
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
+        }) |
+        ranges::views::filter([](auto* td) {
+          return td != nullptr && td->getName() == "introspectImpl";
+        }) |
+        ranges::views::take(1) | ranges::to<std::vector>();
+    if (introspectImpl.empty()) {
+      LOG(WARNING)
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
+             "you compile with `OIL_AOT_COMPILATION=1`?";
+      return;
     }
-  }
 
-  size_t successes = oilTypes.size() - failures;
-  LOG(INFO) << "object introspection generation complete. " << successes
-            << " successes and " << failures << " failures.";
+    auto nameToClangTypeMap =
+        ranges::views::single(introspectImpl[0]) |
+        ranges::views::for_each(
+            [](auto* td) { return td->specializations(); }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
 
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
-    return -1;
+              assert(fd->getNumParams() == 1);
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{ctx.typeGraph, opts};
+
+    auto& Sema = *ctx.sema;
+    ctx.nameToTypeMap =
+        nameToClangTypeMap |
+        ranges::views::transform(
+            [&parser, &Context, &Sema](
+                auto& p) -> std::pair<std::string, type_graph::Type*> {
+              return {p.first, &parser.parse(Context, Sema, *p.second)};
+            }) |
+        ranges::to<std::unordered_map>();
+
+    for (const auto& [name, type] : ctx.nameToTypeMap)
+      ctx.typeGraph.addRoot(*type);
   }
-  return 0;
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  // Compile the output as position independent if any input is position
+  // independent
+  bool pic = CI.getCodeGenOpts().RelocationModel == llvm::Reloc::PIC_;
+  ctx.pic = ctx.pic.value_or(false) || pic;
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
 }
 
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+}  // namespace
 }  // namespace oi::detail
diff --git a/oi/OIGenerator.h b/oi/OIGenerator.h
index dd38384..fac947a 100644
--- a/oi/OIGenerator.h
+++ b/oi/OIGenerator.h
@@ -23,11 +23,19 @@
 #include "oi/OICodeGen.h"
 #include "oi/OICompiler.h"
 
+namespace clang::tooling {
+class CompilationDatabase;
+}
+
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate(clang::tooling::CompilationDatabase&,
+               const std::vector<std::string>&);
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -41,8 +49,8 @@ class OIGenerator {
   void setFailIfNothingGenerated(bool fail) {
     failIfNothingGenerated = fail;
   }
-  void setUsePIC(bool pic_) {
-    pic = pic_;
+  void setClangArgs(std::vector<std::string> args_) {
+    clangArgs = std::move(args_);
   }
 
  private:
@@ -50,20 +58,7 @@ class OIGenerator {
   std::vector<std::filesystem::path> configFilePaths;
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
-  bool pic = false;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
+  std::vector<std::string> clangArgs;
 };
 
 }  // namespace oi::detail
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 940c031..46add03 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
   Flattener.cpp
@@ -26,3 +27,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
new file mode 100644
index 0000000..40d5907
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -0,0 +1,301 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Basic/DiagnosticSema.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <stdexcept>  // TODO: remove
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (requireCompleteType(*sema, ty))
+    return makeType<Incomplete>(ty, "incomplete (TODO naming)");
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(*ty.getAs<const clang::RecordType>());
+    case clang::Type::LValueReference:
+      return enumerateReference(*ty.getAs<const clang::LValueReferenceType>());
+    case clang::Type::Pointer:
+      return enumeratePointer(*ty.getAs<const clang::PointerType>());
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          *ty.getAs<const clang::SubstTemplateTypeParmType>());
+    case clang::Type::Builtin:
+      return enumeratePrimitive(*ty.getAs<const clang::BuiltinType>());
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(*ty.getAs<const clang::ElaboratedType>());
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          *ty.getAs<const clang::TemplateSpecializationType>());
+    case clang::Type::Typedef:
+      return enumerateTypedef(*ty.getAs<const clang::TypedefType>());
+    case clang::Type::ConstantArray:
+      return enumerateArray(*llvm::cast<const clang::ConstantArrayType>(&ty));
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+
+  std::string name = ty.getDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
+  uint64_t len = ty.getSize().getLimitedValue();
+  auto& t = enumerateType(*ty.getElementType());
+  return makeType<Array>(ty, t, len);
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Class& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+  params.reserve(list.size());
+
+  for (const auto& arg : list.asArray()) {
+    params.emplace_back(enumerateTemplateParam(arg));
+  }
+}
+
+TemplateParam ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return {ttype, qualifiers};
+    }
+    default:
+      p.dump();
+      throw std::logic_error("unsupported template argument kind");
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function references
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      ty.dump();
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  // TODO: SEGVs if the type can't be completed
+  return sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      clang::diag::err_type_unsupported);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
new file mode 100644
index 0000000..4fa6374
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.h
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ASTContext;
+class BuiltinType;
+class ConstantArrayType;
+class ElaboratedType;
+class LValueReferenceType;
+class PointerType;
+class RecordType;
+class Sema;
+class SubstTemplateTypeParmType;
+class TemplateArgument;
+class TemplateSpecializationType;
+class Type;
+class TypedefType;
+}  // namespace clang
+
+namespace oi::detail::type_graph {
+
+class Array;
+class Class;
+class Member;
+class Primitive;
+class Reference;
+class Type;
+class TypeGraph;
+class Typedef;
+struct TemplateParam;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+};
+
+/*
+ * SourceParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given drgn_type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph, ClangTypeParserOptions options)
+      : typeGraph_(typeGraph), options_(options) {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Class& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+  Typedef& enumerateTypedef(const clang::TypedefType&);
+  Array& enumerateArray(const clang::ConstantArrayType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  TemplateParam enumerateTemplateParam(const clang::TemplateArgument&);
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index c792766..5213b3d 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -194,6 +194,14 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& p) {
+  RecursiveVisitor::visit(p);
+  p.regenerateName();
+  std::string inputName{p.pointeeType().inputName()};
+  inputName += '*';
+  p.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..e6437d0 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index e7d70eb..5913977 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -147,6 +147,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& p) {
+  if (prefix(p))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = p.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(p.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index a66581e..d7d60f3 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& p) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;
diff --git a/oi/type_graph/RemoveTopLevelPointer.cpp b/oi/type_graph/RemoveTopLevelPointer.cpp
index 9ecaaaa..e1909a7 100644
--- a/oi/type_graph/RemoveTopLevelPointer.cpp
+++ b/oi/type_graph/RemoveTopLevelPointer.cpp
@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& p) {
+  topLevelType_ = &p.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/RemoveTopLevelPointer.h b/oi/type_graph/RemoveTopLevelPointer.h
index 611429f..6b70545 100644
--- a/oi/type_graph/RemoveTopLevelPointer.h
+++ b/oi/type_graph/RemoveTopLevelPointer.h
@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
 
  private:
   Type* topLevelType_ = nullptr;
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f503886..8aba760 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -715,6 +716,61 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    name_ = pointeeType_.get().name() + "&";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index 7e4395e..b2fb7a2 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -121,6 +121,9 @@ class RecursiveVisitor : public Visitor {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -182,6 +185,10 @@ class RecursiveMutator : public Mutator {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index d1e345d..ec3810b 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -14,131 +14,80 @@
  * limitations under the License.
  */
 
+#include <clang/Tooling/CommonOptionsParser.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include <cstdlib>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/action/drop_while.hpp>
+#include <range/v3/algorithm/find.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/all.hpp>
+#include <range/v3/view/transform.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
 
-constexpr static OIOpts opts{
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
+static llvm::cl::OptionCategory OilgenCategory("oilgen options");
 
-void usage() {
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
-  std::cerr << opts;
+static llvm::cl::list<std::string> ConfigFiles("config-file",
+                                               llvm::cl::desc(R"(TODO HELP)"),
+                                               llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<int> DebugLevel("debug-level",
+                                     llvm::cl::desc(R"(TODO HELP)"),
+                                     llvm::cl::init(-1),
+                                     llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> DumpJit("dump-jit",
+                                          llvm::cl::desc(R"(TODO HELP)"),
+                                          llvm::cl::init("jit.cpp"),
+                                          llvm::cl::cat(OilgenCategory));
 
-  std::cerr << std::endl
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
-
-int main(int argc, char* argv[]) {
+int main(int argc, const char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
-  std::vector<fs::path> configFilePaths;
-  fs::path sourceFileDumpPath = "";
-  bool exitCode = false;
-  bool pic = false;
-
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
+  auto expectedParser =
+      clang::tooling::CommonOptionsParser::create(argc, argv, OilgenCategory);
+  if (!expectedParser) {
+    llvm::errs() << expectedParser.takeError();
+    return -1;
   }
+  clang::tooling::CommonOptionsParser& options = expectedParser.get();
+  auto& compilations = options.getCompilations();
 
-  if (optind >= argc) {
-    usage();
-    return EXIT_FAILURE;
-  }
-  fs::path primaryObject = argv[optind];
-
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
+  if (DebugLevel.getNumOccurrences()) {
+    google::LogToStderr();
+    google::SetStderrLogging(google::INFO);
+    google::SetVLOGLevel("*", DebugLevel);
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
+    gflags::SetCommandLineOption("minloglevel", "0");
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
-  oigen.setConfigFilePaths(std::move(configFilePaths));
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
-  oigen.setFailIfNothingGenerated(exitCode);
-  oigen.setUsePIC(pic);
+  oigen.setConfigFilePaths(ConfigFiles |
+                           ranges::views::transform([](const auto& p) {
+                             return std::filesystem::path(p);
+                           }) |
+                           ranges::to<std::vector>());
+  if (DumpJit.getNumOccurrences())
+    oigen.setSourceFileDumpPath(DumpJit.getValue());
 
-  SymbolService symbols(primaryObject);
+  oigen.setOutputPath("a.o");  // TODO: make this an opt
 
-  return oigen.generate(primaryObject, symbols);
+  // TODO: can we extract this from the original arguments?
+  // oigen.setUsePIC(args["pic"].as<bool>());
+
+  oigen.setFailIfNothingGenerated(true);
+  // compilations.getAllFiles();
+  // return oigen.generate(compilations, {"/data/users/jakehillion/fbsource/fbcode/object-introspection/oil/examples/compile-time/VectorOfStrings-Introspect.cpp"});
+  return oigen.generate(compilations, options.getSourcePathList());
 }
diff --git a/types/string_type.toml b/types/string_type.toml
index fa4a2ae..0d2ab0d 100644
--- a/types/string_type.toml
+++ b/types/string_type.toml
@@ -35,3 +35,34 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
     );
 }
 """
+
+traversal_func = """
+    bool sso = ((uintptr_t)container.data() <
+                (uintptr_t)(&container + sizeof(std::__cxx11::basic_string<T0>))) &&
+               ((uintptr_t)container.data() >= (uintptr_t)&container);
+
+    return returnArg.write(container.capacity())
+        .write(sso)
+        .write(container.size());
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+uint64_t capacity = std::get<ParsedData::VarInt>(d.val).value;
+el.container_stats.emplace(result::Element::ContainerStats { .capacity = capacity });
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+bool sso = std::get<ParsedData::VarInt>(d.val).value;
+if (!sso)
+  el.exclusive_size += el.container_stats->capacity * sizeof(T0);
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats->length = std::get<ParsedData::VarInt>(d.val).value;
+"""

From 65b739340ec5ec552e76fa74925583e795dc9fd8 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Mon, 11 Dec 2023 06:12:27 -0800
Subject: [PATCH 045/188] oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD
---
 CMakeLists.txt                          |  13 +-
 oi/CodeGen.cpp                          |  30 +-
 oi/CodeGen.h                            |   8 +-
 oi/Config.cpp                           |   9 +-
 oi/ContainerInfo.cpp                    |   2 +-
 oi/OICodeGen.h                          |   2 +-
 oi/OIGenerator.cpp                      | 357 ++++++++++++----------
 oi/OIGenerator.h                        |  29 +-
 oi/type_graph/CMakeLists.txt            |   2 +
 oi/type_graph/ClangTypeParser.cpp       | 378 ++++++++++++++++++++++++
 oi/type_graph/ClangTypeParser.h         | 118 ++++++++
 oi/type_graph/NameGen.cpp               |   8 +
 oi/type_graph/NameGen.h                 |   1 +
 oi/type_graph/Printer.cpp               |  11 +
 oi/type_graph/Printer.h                 |   1 +
 oi/type_graph/RemoveTopLevelPointer.cpp |   4 +
 oi/type_graph/RemoveTopLevelPointer.h   |   1 +
 oi/type_graph/Types.h                   |  56 ++++
 oi/type_graph/Visitor.h                 |   7 +
 tools/OILGen.cpp                        | 147 +++------
 types/string_type.toml                  |  31 ++
 21 files changed, 919 insertions(+), 296 deletions(-)
 create mode 100644 oi/type_graph/ClangTypeParser.cpp
 create mode 100644 oi/type_graph/ClangTypeParser.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f9ee7..eb18f4a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -104,6 +104,15 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(range-v3)
 
+### cxxopts
+FetchContent_Declare(
+  cxxopts
+  GIT_REPOSITORY https://github.com/jarro2783/cxxopts.git
+  GIT_TAG        eb787304d67ec22f7c3a184ee8b4c481d04357fd # v3.1.1
+  GIT_PROGRESS   TRUE
+)
+FetchContent_MakeAvailable(cxxopts)
+
 set_project_warnings()
 
 if (ASAN)
@@ -289,7 +298,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -363,8 +372,10 @@ add_executable(oilgen
   oi/OIGenerator.cpp
 )
 target_link_libraries(oilgen
+  cxxopts
   drgn_utils
   oicore
+  clangTooling
 )
 
 ### Object Introspection cache Printer (OIP)
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index bef102f..a7b59bb 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -539,7 +539,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1106,15 +1106,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
 }
 
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
-  try {
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
+  if (!registerContainers())
     return false;
-  }
 
   TypeGraph typeGraph;
   try {
@@ -1129,6 +1122,20 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
   return true;
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    size_t i = 0;
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1170,7 +1177,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(Flattener::createPass());
@@ -1286,7 +1293,8 @@ void CodeGen::generate(
   code += "\nusing __ROOT_TYPE__ = " + rootType.name() + ";\n";
   code += "} // namespace\n} // namespace OIInternal\n";
 
-  const auto typeName = SymbolService::getTypeName(drgnType);
+  // const auto typeName = SymbolService::getTypeName(drgnType);
+  const std::string typeName{"typeName"};
   if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
   } else {
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 186ec34..fb0ec20 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -40,8 +40,11 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config) : config_(config) {
+    assert(!config.features[Feature::PolymorphicInheritance]);
+  }
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   /*
@@ -53,6 +56,7 @@ class CodeGen {
                        std::string linkageName,
                        std::string& code);
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -66,7 +70,7 @@ class CodeGen {
 
  private:
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
diff --git a/oi/Config.cpp b/oi/Config.cpp
index 157b279..66798bb 100644
--- a/oi/Config.cpp
+++ b/oi/Config.cpp
@@ -22,6 +22,8 @@
 #include <boost/property_tree/ini_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 #include <filesystem>
+#include <range/v3/action/sort.hpp>
+#include <range/v3/action/unique.hpp>
 
 #include "oi/support/Toml.h"
 
@@ -54,6 +56,9 @@ std::optional<FeatureSet> processConfigFiles(
     enables |= *fs;
   }
 
+  ranges::actions::sort(generatorConfig.containerConfigPaths);
+  ranges::actions::unique(generatorConfig.containerConfigPaths);
+
   // Override anything from the config files with command line options
   for (auto [k, v] : featureMap) {
     enables[k] = v;
@@ -108,8 +113,8 @@ std::optional<FeatureSet> processConfigFile(
           if the right path is absolute, else append the right path to the
           left path.
           */
-          generatorConfig.containerConfigPaths.emplace(configDirectory /
-                                                       el.get());
+          generatorConfig.containerConfigPaths.emplace_back(configDirectory /
+                                                            el.get());
         }
       });
     }
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 99a7dc1..dbe10c6 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -202,7 +202,7 @@ std::regex getMatcher(const std::string& typeName) {
 ContainerInfo::ContainerInfo(const fs::path& path) {
   toml::table container;
   try {
-    container = toml::parse_file(std::string(path));
+    container = toml::parse_file(path.string());
   } catch (const toml::parse_error& err) {
     // Convert into a ContainerInfoError, just to avoid having to include
     // the huge TOML++ header in the caller's file. Use toml::parse_error's
diff --git a/oi/OICodeGen.h b/oi/OICodeGen.h
index a01548f..7226a4e 100644
--- a/oi/OICodeGen.h
+++ b/oi/OICodeGen.h
@@ -66,7 +66,7 @@ class OICodeGen {
     };
 
     FeatureSet features;
-    std::set<std::filesystem::path> containerConfigPaths;
+    std::vector<std::filesystem::path> containerConfigPaths;
     std::set<std::string> defaultHeaders;
     std::set<std::string> defaultNamespaces;
     std::vector<std::pair<std::string, std::string>> membersToStub;
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 1cb2a49..7a04310 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -16,130 +16,131 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
 #include <glog/logging.h>
 
 #include <boost/core/demangle.hpp>
 #include <fstream>
 #include <iostream>
+#include <range/v3/action/transform.hpp>
+#include <range/v3/core.hpp>
+#include <range/v3/view/cache1.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/drop_while.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>  // TODO: remove
 #include <string_view>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
+class ConsumerContext;
 
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
-      templateArgsToSymbolsMap;
-
-  auto symbols = prog.find_all_symbols();
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
+class CreateTypeGraphConsumer;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
-    if (val.first.empty() || val.second.empty()) {
-      continue;
-    }
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  return strongToWeakSymbols;
-}
-
-std::unordered_map<std::string, drgn_qualified_type>
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> out;
-
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
-    std::string strongLinkageName;
-    {
-      const char* linkageNameCstr;
-      if (auto err = drgnplusplus::error(
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
-        // throw err;
-        continue;
-      }
-      strongLinkageName = linkageNameCstr;
-    }
-
-    std::string weakLinkageName;
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
   }
 
-  return out;
-}
+ private:
+  ConsumerContext& ctx;
+};
 
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
-                                      const OICompiler::Config& compilerConfig,
-                                      const drgn_qualified_type& type,
-                                      const std::string& linkageName,
-                                      SymbolService& symbols) {
-  CodeGen codegen{generatorConfig, symbols};
+class ConsumerContext {
+ public:
+  type_graph::TypeGraph typeGraph;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
+  std::optional<bool> pic;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+}  // namespace
+
+int OIGenerator::generate(clang::tooling::CompilationDatabase& db,
+                          const std::vector<std::string>& sourcePaths) {
+  ConsumerContext ctx;
+  CreateTypeGraphActionFactory factory{ctx};
+
+  clang::tooling::ClangTool tool{db, sourcePaths};
+  if (auto ret = tool.run(&factory); ret != 0) {
+    return ret;
+  }
+
+  if (ctx.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one type to generate for but we can't currently "
+        "handle this case");
+
+  std::map<Feature, bool> featuresMap = {
+      {Feature::TypeGraph, true},
+      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},
+      {Feature::PackStructs, true},
+      {Feature::PruneTypeGraph, true},
+  };
+
+  OICodeGen::Config generatorConfig{};
+  OICompiler::Config compilerConfig{};
+  compilerConfig.usePIC = ctx.pic.value();
+
+  auto features = config::processConfigFiles(
+      configFilePaths, featuresMap, compilerConfig, generatorConfig);
+  if (!features) {
+    LOG(ERROR) << "failed to process config file";
+    return -1;
+  }
+  generatorConfig.features = *features;
+  compilerConfig.features = *features;
+
+  CodeGen codegen{generatorConfig};
+  if (!codegen.registerContainers()) {
+    LOG(ERROR) << "Failed to register containers";
+    return -1;
+  }
+  codegen.transform(ctx.typeGraph);
 
   std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
+  codegen.generate(ctx.typeGraph, code, nullptr);
 
   std::string sourcePath = sourceFileDumpPath;
   if (sourceFileDumpPath.empty()) {
@@ -152,78 +153,110 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
   }
 
   OICompiler compiler{{}, compilerConfig};
-
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
-
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
 }
 
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
-  drgnplusplus::program prog;
+namespace {
 
-  {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(
-            drgn_program_load_debug_info(prog.get(),
-                                         std::data(objectPaths),
-                                         std::size(objectPaths),
-                                         false,
-                                         false))) {
-      LOG(ERROR) << "error loading debug info program: " << err;
-      throw err;
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx(ctx_) {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
     }
-  }
 
-  auto oilTypes = findOilTypesAndNames(prog);
-
-  std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
-      {Feature::PruneTypeGraph, true},
-  };
-
-  OICodeGen::Config generatorConfig{};
-  OICompiler::Config compilerConfig{};
-  compilerConfig.usePIC = pic;
-
-  auto features = config::processConfigFiles(
-      configFilePaths, featuresMap, compilerConfig, generatorConfig);
-  if (!features) {
-    LOG(ERROR) << "failed to process config file";
-    return -1;
-  }
-  generatorConfig.features = *features;
-  compilerConfig.features = *features;
-
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
+    auto introspectImpl =
+        std::move(oi_namespaces) |
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
+        ranges::views::transform([](auto* p) {
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
+        }) |
+        ranges::views::filter([](auto* td) {
+          return td != nullptr && td->getName() == "introspectImpl";
+        }) |
+        ranges::views::take(1) | ranges::to<std::vector>();
+    if (introspectImpl.empty()) {
+      LOG(WARNING)
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
+             "you compile with `OIL_AOT_COMPILATION=1`?";
+      return;
     }
-  }
 
-  size_t successes = oilTypes.size() - failures;
-  LOG(INFO) << "object introspection generation complete. " << successes
-            << " successes and " << failures << " failures.";
+    auto nameToClangTypeMap =
+        ranges::views::single(introspectImpl[0]) |
+        ranges::views::for_each(
+            [](auto* td) { return td->specializations(); }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
 
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
-    return -1;
+              assert(fd->getNumParams() == 1);
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{ctx.typeGraph, opts};
+
+    auto& Sema = *ctx.sema;
+    ctx.nameToTypeMap =
+        nameToClangTypeMap |
+        ranges::views::transform(
+            [&parser, &Context, &Sema](
+                auto& p) -> std::pair<std::string, type_graph::Type*> {
+              return {p.first, &parser.parse(Context, Sema, *p.second)};
+            }) |
+        ranges::to<std::unordered_map>();
+
+    for (const auto& [name, type] : ctx.nameToTypeMap)
+      ctx.typeGraph.addRoot(*type);
   }
-  return 0;
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  // Compile the output as position independent if any input is position
+  // independent
+  bool pic = CI.getCodeGenOpts().RelocationModel == llvm::Reloc::PIC_;
+  ctx.pic = ctx.pic.value_or(false) || pic;
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
 }
 
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+}  // namespace
 }  // namespace oi::detail
diff --git a/oi/OIGenerator.h b/oi/OIGenerator.h
index dd38384..fac947a 100644
--- a/oi/OIGenerator.h
+++ b/oi/OIGenerator.h
@@ -23,11 +23,19 @@
 #include "oi/OICodeGen.h"
 #include "oi/OICompiler.h"
 
+namespace clang::tooling {
+class CompilationDatabase;
+}
+
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate(clang::tooling::CompilationDatabase&,
+               const std::vector<std::string>&);
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -41,8 +49,8 @@ class OIGenerator {
   void setFailIfNothingGenerated(bool fail) {
     failIfNothingGenerated = fail;
   }
-  void setUsePIC(bool pic_) {
-    pic = pic_;
+  void setClangArgs(std::vector<std::string> args_) {
+    clangArgs = std::move(args_);
   }
 
  private:
@@ -50,20 +58,7 @@ class OIGenerator {
   std::vector<std::filesystem::path> configFilePaths;
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
-  bool pic = false;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
+  std::vector<std::string> clangArgs;
 };
 
 }  // namespace oi::detail
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 940c031..46add03 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
   Flattener.cpp
@@ -26,3 +27,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
new file mode 100644
index 0000000..64d2265
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -0,0 +1,378 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Basic/DiagnosticSema.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <iostream>  // TODO: maybe remove
+#include <stdexcept>  // TODO: remove
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (!requireCompleteType(*sema, ty))
+    return makeType<Incomplete>(ty, "incomplete (TODO naming)");
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(*ty.getAs<const clang::RecordType>());
+    case clang::Type::LValueReference:
+      return enumerateReference(*ty.getAs<const clang::LValueReferenceType>());
+    case clang::Type::Pointer:
+      return enumeratePointer(*ty.getAs<const clang::PointerType>());
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          *ty.getAs<const clang::SubstTemplateTypeParmType>());
+    case clang::Type::Builtin:
+      return enumeratePrimitive(*ty.getAs<const clang::BuiltinType>());
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(*ty.getAs<const clang::ElaboratedType>());
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          *ty.getAs<const clang::TemplateSpecializationType>());
+
+    case clang::Type::Typedef:
+      return enumerateTypedef(*ty.getAs<const clang::TypedefType>());
+    case clang::Type::Using:
+      return enumerateUsing(*ty.getAs<const clang::UsingType>());
+
+    case clang::Type::ConstantArray:
+      return enumerateArray(*llvm::cast<const clang::ConstantArrayType>(&ty));
+    case clang::Type::Enum:
+      return enumerateEnum(*ty.getAs<const clang::EnumType>());
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Typedef& ClangTypeParser::enumerateUsing(const clang::UsingType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+  std::string name = ty.getFoundDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+
+  std::string name = ty.getDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string name = ty.getDecl()->getNameAsString();
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  std::map<int64_t, std::string> enumeratorMap;
+  if (options_.readEnumValues) {
+    for (const auto* enumerator : ty.getDecl()->enumerators()) {
+      enumeratorMap.emplace(
+          enumerator->getInitVal().getExtValue(),
+          enumerator->getNameAsString()
+      );
+    }
+  }
+
+  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+}
+
+Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
+  uint64_t len = ty.getSize().getLimitedValue();
+  auto& t = enumerateType(*ty.getElementType());
+  return makeType<Array>(ty, t, len);
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Class& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+
+  params.reserve(list.size());
+  for (const auto& arg : list.asArray()) {
+    params.emplace_back(enumerateTemplateParam(arg));
+  }
+}
+
+TemplateParam ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return {ttype, qualifiers};
+    }
+    case clang::TemplateArgument::Integral: {
+      auto& ty = enumerateType(*p.getIntegralType());
+      llvm::SmallString<32> val;
+      p.getAsIntegral().toString(val);
+      return {ty, std::string(val)};
+    }
+    case clang::TemplateArgument::Template: {
+      return enumerateTemplateTemplateParam(p.getAsTemplate());
+    }
+
+#define X(name) \
+case clang::TemplateArgument::name: \
+  throw std::logic_error("unsupported template argument kind: " #name);
+
+  X(Declaration)
+  X(NullPtr)
+  X(TemplateExpansion)
+  X(Expression)
+  X(Pack)
+#undef X
+  }
+}
+
+TemplateParam ClangTypeParser::enumerateTemplateTemplateParam(const clang::TemplateName& tn) {
+  switch (tn.getKind()) {
+    case clang::TemplateName::Template: {
+      auto* underlying = tn.getAsTemplateDecl();
+      return enumerateTemplateTemplateParam(underlying);
+    }
+
+#define X(name) \
+case clang::TemplateName::name: \
+  throw std::logic_error("unsupported template name kind: " #name);
+
+// X(Template)
+X(OverloadedTemplate)
+X(AssumedTemplate)
+X(QualifiedTemplate)
+X(DependentTemplate)
+X(SubstTemplateTemplateParm)
+X(SubstTemplateTemplateParmPack)
+X(UsingTemplate)
+#undef X
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function references
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Float:
+      return makeType<Primitive>(ty, Primitive::Kind::Float32);
+    case clang::BuiltinType::Double:
+    case clang::BuiltinType::LongDouble:
+      return makeType<Primitive>(ty, Primitive::Kind::Float64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      ty.dump();
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
+    return true; // treat as complete
+
+  return !sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      clang::diag::err_type_unsupported);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
new file mode 100644
index 0000000..7e36c42
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.h
@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ASTContext;
+class BuiltinType;
+class ConstantArrayType;
+class ElaboratedType;
+class EnumType;
+class LValueReferenceType;
+class PointerType;
+class RecordType;
+class Sema;
+class SubstTemplateTypeParmType;
+class TemplateArgument;
+class TemplateName;
+class TemplateSpecializationType;
+class Type;
+class TypedefType;
+class UsingType;
+}  // namespace clang
+
+namespace oi::detail::type_graph {
+
+class Array;
+class Class;
+class Enum;
+class Member;
+class Primitive;
+class Reference;
+class Type;
+class TypeGraph;
+class Typedef;
+struct TemplateParam;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+  bool readEnumValues = false;
+};
+
+/*
+ * ClangTypeParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given drgn_type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph, ClangTypeParserOptions options)
+      : typeGraph_(typeGraph), options_(options) {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Class& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+  Typedef& enumerateTypedef(const clang::TypedefType&);
+  Typedef& enumerateUsing(const clang::UsingType&);
+
+  Array& enumerateArray(const clang::ConstantArrayType&);
+  Enum& enumerateEnum(const clang::EnumType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  TemplateParam enumerateTemplateParam(const clang::TemplateArgument&);
+  TemplateParam enumerateTemplateTemplateParam(const clang::TemplateName&);
+
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index c792766..5213b3d 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -194,6 +194,14 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& p) {
+  RecursiveVisitor::visit(p);
+  p.regenerateName();
+  std::string inputName{p.pointeeType().inputName()};
+  inputName += '*';
+  p.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..e6437d0 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index e7d70eb..5913977 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -147,6 +147,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& p) {
+  if (prefix(p))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = p.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(p.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index a66581e..d7d60f3 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& p) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;
diff --git a/oi/type_graph/RemoveTopLevelPointer.cpp b/oi/type_graph/RemoveTopLevelPointer.cpp
index 9ecaaaa..e1909a7 100644
--- a/oi/type_graph/RemoveTopLevelPointer.cpp
+++ b/oi/type_graph/RemoveTopLevelPointer.cpp
@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& p) {
+  topLevelType_ = &p.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/RemoveTopLevelPointer.h b/oi/type_graph/RemoveTopLevelPointer.h
index 611429f..6b70545 100644
--- a/oi/type_graph/RemoveTopLevelPointer.h
+++ b/oi/type_graph/RemoveTopLevelPointer.h
@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
 
  private:
   Type* topLevelType_ = nullptr;
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f503886..8aba760 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -715,6 +716,61 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    name_ = pointeeType_.get().name() + "&";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index 7e4395e..b2fb7a2 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -121,6 +121,9 @@ class RecursiveVisitor : public Visitor {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -182,6 +185,10 @@ class RecursiveMutator : public Mutator {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index d1e345d..ec3810b 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -14,131 +14,80 @@
  * limitations under the License.
  */
 
+#include <clang/Tooling/CommonOptionsParser.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include <cstdlib>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/action/drop_while.hpp>
+#include <range/v3/algorithm/find.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/all.hpp>
+#include <range/v3/view/transform.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
 
-constexpr static OIOpts opts{
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
+static llvm::cl::OptionCategory OilgenCategory("oilgen options");
 
-void usage() {
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
-  std::cerr << opts;
+static llvm::cl::list<std::string> ConfigFiles("config-file",
+                                               llvm::cl::desc(R"(TODO HELP)"),
+                                               llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<int> DebugLevel("debug-level",
+                                     llvm::cl::desc(R"(TODO HELP)"),
+                                     llvm::cl::init(-1),
+                                     llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> DumpJit("dump-jit",
+                                          llvm::cl::desc(R"(TODO HELP)"),
+                                          llvm::cl::init("jit.cpp"),
+                                          llvm::cl::cat(OilgenCategory));
 
-  std::cerr << std::endl
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
-
-int main(int argc, char* argv[]) {
+int main(int argc, const char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
-  std::vector<fs::path> configFilePaths;
-  fs::path sourceFileDumpPath = "";
-  bool exitCode = false;
-  bool pic = false;
-
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
+  auto expectedParser =
+      clang::tooling::CommonOptionsParser::create(argc, argv, OilgenCategory);
+  if (!expectedParser) {
+    llvm::errs() << expectedParser.takeError();
+    return -1;
   }
+  clang::tooling::CommonOptionsParser& options = expectedParser.get();
+  auto& compilations = options.getCompilations();
 
-  if (optind >= argc) {
-    usage();
-    return EXIT_FAILURE;
-  }
-  fs::path primaryObject = argv[optind];
-
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
+  if (DebugLevel.getNumOccurrences()) {
+    google::LogToStderr();
+    google::SetStderrLogging(google::INFO);
+    google::SetVLOGLevel("*", DebugLevel);
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
+    gflags::SetCommandLineOption("minloglevel", "0");
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
-  oigen.setConfigFilePaths(std::move(configFilePaths));
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
-  oigen.setFailIfNothingGenerated(exitCode);
-  oigen.setUsePIC(pic);
+  oigen.setConfigFilePaths(ConfigFiles |
+                           ranges::views::transform([](const auto& p) {
+                             return std::filesystem::path(p);
+                           }) |
+                           ranges::to<std::vector>());
+  if (DumpJit.getNumOccurrences())
+    oigen.setSourceFileDumpPath(DumpJit.getValue());
 
-  SymbolService symbols(primaryObject);
+  oigen.setOutputPath("a.o");  // TODO: make this an opt
 
-  return oigen.generate(primaryObject, symbols);
+  // TODO: can we extract this from the original arguments?
+  // oigen.setUsePIC(args["pic"].as<bool>());
+
+  oigen.setFailIfNothingGenerated(true);
+  // compilations.getAllFiles();
+  // return oigen.generate(compilations, {"/data/users/jakehillion/fbsource/fbcode/object-introspection/oil/examples/compile-time/VectorOfStrings-Introspect.cpp"});
+  return oigen.generate(compilations, options.getSourcePathList());
 }
diff --git a/types/string_type.toml b/types/string_type.toml
index fa4a2ae..0d2ab0d 100644
--- a/types/string_type.toml
+++ b/types/string_type.toml
@@ -35,3 +35,34 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
     );
 }
 """
+
+traversal_func = """
+    bool sso = ((uintptr_t)container.data() <
+                (uintptr_t)(&container + sizeof(std::__cxx11::basic_string<T0>))) &&
+               ((uintptr_t)container.data() >= (uintptr_t)&container);
+
+    return returnArg.write(container.capacity())
+        .write(sso)
+        .write(container.size());
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+uint64_t capacity = std::get<ParsedData::VarInt>(d.val).value;
+el.container_stats.emplace(result::Element::ContainerStats { .capacity = capacity });
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+bool sso = std::get<ParsedData::VarInt>(d.val).value;
+if (!sso)
+  el.exclusive_size += el.container_stats->capacity * sizeof(T0);
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats->length = std::get<ParsedData::VarInt>(d.val).value;
+"""

From 606e3e3e498eab2dba3b45c0ebe3e3b20463eede Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 12 Dec 2023 11:01:11 -0800
Subject: [PATCH 046/188] oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD
---
 CMakeLists.txt                                |  13 +-
 oi/CodeGen.cpp                                |  30 +-
 oi/CodeGen.h                                  |   8 +-
 oi/Config.cpp                                 |   9 +-
 oi/ContainerInfo.cpp                          |   2 +-
 oi/OICodeGen.h                                |   2 +-
 oi/OIGenerator.cpp                            | 357 ++++++++--------
 oi/OIGenerator.h                              |  29 +-
 oi/type_graph/CMakeLists.txt                  |   2 +
 oi/type_graph/ClangTypeParser.cpp             | 393 ++++++++++++++++++
 oi/type_graph/ClangTypeParser.h               | 123 ++++++
 oi/type_graph/NameGen.cpp                     |   8 +
 oi/type_graph/NameGen.h                       |   1 +
 oi/type_graph/Printer.cpp                     |  11 +
 oi/type_graph/Printer.h                       |   1 +
 oi/type_graph/RemoveTopLevelPointer.cpp       |   4 +
 oi/type_graph/RemoveTopLevelPointer.h         |   1 +
 oi/type_graph/Types.h                         |  56 +++
 oi/type_graph/Visitor.h                       |   7 +
 test/integration/template_template_param.toml |  16 +
 tools/OILGen.cpp                              | 147 +++----
 types/string_type.toml                        |  31 ++
 22 files changed, 955 insertions(+), 296 deletions(-)
 create mode 100644 oi/type_graph/ClangTypeParser.cpp
 create mode 100644 oi/type_graph/ClangTypeParser.h
 create mode 100644 test/integration/template_template_param.toml

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f9ee7..eb18f4a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -104,6 +104,15 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(range-v3)
 
+### cxxopts
+FetchContent_Declare(
+  cxxopts
+  GIT_REPOSITORY https://github.com/jarro2783/cxxopts.git
+  GIT_TAG        eb787304d67ec22f7c3a184ee8b4c481d04357fd # v3.1.1
+  GIT_PROGRESS   TRUE
+)
+FetchContent_MakeAvailable(cxxopts)
+
 set_project_warnings()
 
 if (ASAN)
@@ -289,7 +298,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -363,8 +372,10 @@ add_executable(oilgen
   oi/OIGenerator.cpp
 )
 target_link_libraries(oilgen
+  cxxopts
   drgn_utils
   oicore
+  clangTooling
 )
 
 ### Object Introspection cache Printer (OIP)
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 5a4b9eb..e840104 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -537,7 +537,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1104,15 +1104,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
 }
 
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
-  try {
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
+  if (!registerContainers())
     return false;
-  }
 
   TypeGraph typeGraph;
   try {
@@ -1127,6 +1120,20 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
   return true;
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    size_t i = 0;
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1167,7 +1174,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, containerInfos_, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(Flattener::createPass());
@@ -1282,7 +1289,8 @@ void CodeGen::generate(
   code += "\nusing __ROOT_TYPE__ = " + rootType.name() + ";\n";
   code += "} // namespace\n} // namespace OIInternal\n";
 
-  const auto typeName = SymbolService::getTypeName(drgnType);
+  // const auto typeName = SymbolService::getTypeName(drgnType);
+  const std::string typeName{"typeName"};
   if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
   } else {
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 186ec34..fb0ec20 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -40,8 +40,11 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config) : config_(config) {
+    assert(!config.features[Feature::PolymorphicInheritance]);
+  }
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   /*
@@ -53,6 +56,7 @@ class CodeGen {
                        std::string linkageName,
                        std::string& code);
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -66,7 +70,7 @@ class CodeGen {
 
  private:
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
diff --git a/oi/Config.cpp b/oi/Config.cpp
index 157b279..66798bb 100644
--- a/oi/Config.cpp
+++ b/oi/Config.cpp
@@ -22,6 +22,8 @@
 #include <boost/property_tree/ini_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 #include <filesystem>
+#include <range/v3/action/sort.hpp>
+#include <range/v3/action/unique.hpp>
 
 #include "oi/support/Toml.h"
 
@@ -54,6 +56,9 @@ std::optional<FeatureSet> processConfigFiles(
     enables |= *fs;
   }
 
+  ranges::actions::sort(generatorConfig.containerConfigPaths);
+  ranges::actions::unique(generatorConfig.containerConfigPaths);
+
   // Override anything from the config files with command line options
   for (auto [k, v] : featureMap) {
     enables[k] = v;
@@ -108,8 +113,8 @@ std::optional<FeatureSet> processConfigFile(
           if the right path is absolute, else append the right path to the
           left path.
           */
-          generatorConfig.containerConfigPaths.emplace(configDirectory /
-                                                       el.get());
+          generatorConfig.containerConfigPaths.emplace_back(configDirectory /
+                                                            el.get());
         }
       });
     }
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 99a7dc1..dbe10c6 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -202,7 +202,7 @@ std::regex getMatcher(const std::string& typeName) {
 ContainerInfo::ContainerInfo(const fs::path& path) {
   toml::table container;
   try {
-    container = toml::parse_file(std::string(path));
+    container = toml::parse_file(path.string());
   } catch (const toml::parse_error& err) {
     // Convert into a ContainerInfoError, just to avoid having to include
     // the huge TOML++ header in the caller's file. Use toml::parse_error's
diff --git a/oi/OICodeGen.h b/oi/OICodeGen.h
index a01548f..7226a4e 100644
--- a/oi/OICodeGen.h
+++ b/oi/OICodeGen.h
@@ -66,7 +66,7 @@ class OICodeGen {
     };
 
     FeatureSet features;
-    std::set<std::filesystem::path> containerConfigPaths;
+    std::vector<std::filesystem::path> containerConfigPaths;
     std::set<std::string> defaultHeaders;
     std::set<std::string> defaultNamespaces;
     std::vector<std::pair<std::string, std::string>> membersToStub;
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 1cb2a49..7a04310 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -16,130 +16,131 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
 #include <glog/logging.h>
 
 #include <boost/core/demangle.hpp>
 #include <fstream>
 #include <iostream>
+#include <range/v3/action/transform.hpp>
+#include <range/v3/core.hpp>
+#include <range/v3/view/cache1.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/drop_while.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>  // TODO: remove
 #include <string_view>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
+class ConsumerContext;
 
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
-      templateArgsToSymbolsMap;
-
-  auto symbols = prog.find_all_symbols();
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
+class CreateTypeGraphConsumer;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
-    if (val.first.empty() || val.second.empty()) {
-      continue;
-    }
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  return strongToWeakSymbols;
-}
-
-std::unordered_map<std::string, drgn_qualified_type>
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> out;
-
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
-    std::string strongLinkageName;
-    {
-      const char* linkageNameCstr;
-      if (auto err = drgnplusplus::error(
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
-        // throw err;
-        continue;
-      }
-      strongLinkageName = linkageNameCstr;
-    }
-
-    std::string weakLinkageName;
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
   }
 
-  return out;
-}
+ private:
+  ConsumerContext& ctx;
+};
 
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
-                                      const OICompiler::Config& compilerConfig,
-                                      const drgn_qualified_type& type,
-                                      const std::string& linkageName,
-                                      SymbolService& symbols) {
-  CodeGen codegen{generatorConfig, symbols};
+class ConsumerContext {
+ public:
+  type_graph::TypeGraph typeGraph;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
+  std::optional<bool> pic;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+}  // namespace
+
+int OIGenerator::generate(clang::tooling::CompilationDatabase& db,
+                          const std::vector<std::string>& sourcePaths) {
+  ConsumerContext ctx;
+  CreateTypeGraphActionFactory factory{ctx};
+
+  clang::tooling::ClangTool tool{db, sourcePaths};
+  if (auto ret = tool.run(&factory); ret != 0) {
+    return ret;
+  }
+
+  if (ctx.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one type to generate for but we can't currently "
+        "handle this case");
+
+  std::map<Feature, bool> featuresMap = {
+      {Feature::TypeGraph, true},
+      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},
+      {Feature::PackStructs, true},
+      {Feature::PruneTypeGraph, true},
+  };
+
+  OICodeGen::Config generatorConfig{};
+  OICompiler::Config compilerConfig{};
+  compilerConfig.usePIC = ctx.pic.value();
+
+  auto features = config::processConfigFiles(
+      configFilePaths, featuresMap, compilerConfig, generatorConfig);
+  if (!features) {
+    LOG(ERROR) << "failed to process config file";
+    return -1;
+  }
+  generatorConfig.features = *features;
+  compilerConfig.features = *features;
+
+  CodeGen codegen{generatorConfig};
+  if (!codegen.registerContainers()) {
+    LOG(ERROR) << "Failed to register containers";
+    return -1;
+  }
+  codegen.transform(ctx.typeGraph);
 
   std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
+  codegen.generate(ctx.typeGraph, code, nullptr);
 
   std::string sourcePath = sourceFileDumpPath;
   if (sourceFileDumpPath.empty()) {
@@ -152,78 +153,110 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
   }
 
   OICompiler compiler{{}, compilerConfig};
-
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
-
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
 }
 
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
-  drgnplusplus::program prog;
+namespace {
 
-  {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(
-            drgn_program_load_debug_info(prog.get(),
-                                         std::data(objectPaths),
-                                         std::size(objectPaths),
-                                         false,
-                                         false))) {
-      LOG(ERROR) << "error loading debug info program: " << err;
-      throw err;
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx(ctx_) {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
     }
-  }
 
-  auto oilTypes = findOilTypesAndNames(prog);
-
-  std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
-      {Feature::PruneTypeGraph, true},
-  };
-
-  OICodeGen::Config generatorConfig{};
-  OICompiler::Config compilerConfig{};
-  compilerConfig.usePIC = pic;
-
-  auto features = config::processConfigFiles(
-      configFilePaths, featuresMap, compilerConfig, generatorConfig);
-  if (!features) {
-    LOG(ERROR) << "failed to process config file";
-    return -1;
-  }
-  generatorConfig.features = *features;
-  compilerConfig.features = *features;
-
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
+    auto introspectImpl =
+        std::move(oi_namespaces) |
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
+        ranges::views::transform([](auto* p) {
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
+        }) |
+        ranges::views::filter([](auto* td) {
+          return td != nullptr && td->getName() == "introspectImpl";
+        }) |
+        ranges::views::take(1) | ranges::to<std::vector>();
+    if (introspectImpl.empty()) {
+      LOG(WARNING)
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
+             "you compile with `OIL_AOT_COMPILATION=1`?";
+      return;
     }
-  }
 
-  size_t successes = oilTypes.size() - failures;
-  LOG(INFO) << "object introspection generation complete. " << successes
-            << " successes and " << failures << " failures.";
+    auto nameToClangTypeMap =
+        ranges::views::single(introspectImpl[0]) |
+        ranges::views::for_each(
+            [](auto* td) { return td->specializations(); }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
 
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
-    return -1;
+              assert(fd->getNumParams() == 1);
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{ctx.typeGraph, opts};
+
+    auto& Sema = *ctx.sema;
+    ctx.nameToTypeMap =
+        nameToClangTypeMap |
+        ranges::views::transform(
+            [&parser, &Context, &Sema](
+                auto& p) -> std::pair<std::string, type_graph::Type*> {
+              return {p.first, &parser.parse(Context, Sema, *p.second)};
+            }) |
+        ranges::to<std::unordered_map>();
+
+    for (const auto& [name, type] : ctx.nameToTypeMap)
+      ctx.typeGraph.addRoot(*type);
   }
-  return 0;
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  // Compile the output as position independent if any input is position
+  // independent
+  bool pic = CI.getCodeGenOpts().RelocationModel == llvm::Reloc::PIC_;
+  ctx.pic = ctx.pic.value_or(false) || pic;
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
 }
 
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+}  // namespace
 }  // namespace oi::detail
diff --git a/oi/OIGenerator.h b/oi/OIGenerator.h
index dd38384..fac947a 100644
--- a/oi/OIGenerator.h
+++ b/oi/OIGenerator.h
@@ -23,11 +23,19 @@
 #include "oi/OICodeGen.h"
 #include "oi/OICompiler.h"
 
+namespace clang::tooling {
+class CompilationDatabase;
+}
+
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate(clang::tooling::CompilationDatabase&,
+               const std::vector<std::string>&);
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -41,8 +49,8 @@ class OIGenerator {
   void setFailIfNothingGenerated(bool fail) {
     failIfNothingGenerated = fail;
   }
-  void setUsePIC(bool pic_) {
-    pic = pic_;
+  void setClangArgs(std::vector<std::string> args_) {
+    clangArgs = std::move(args_);
   }
 
  private:
@@ -50,20 +58,7 @@ class OIGenerator {
   std::vector<std::filesystem::path> configFilePaths;
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
-  bool pic = false;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
+  std::vector<std::string> clangArgs;
 };
 
 }  // namespace oi::detail
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 4e8f622..54aba34 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
   Flattener.cpp
@@ -25,3 +26,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
new file mode 100644
index 0000000..4bebc05
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -0,0 +1,393 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Basic/DiagnosticSema.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <iostream>  // TODO: maybe remove
+#include <stdexcept>  // TODO: remove
+#include <regex>
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (!requireCompleteType(*sema, ty))
+    return makeType<Incomplete>(ty, "incomplete (TODO naming)");
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(*ty.getAs<const clang::RecordType>());
+    case clang::Type::LValueReference:
+      return enumerateReference(*ty.getAs<const clang::LValueReferenceType>());
+    case clang::Type::Pointer:
+      return enumeratePointer(*ty.getAs<const clang::PointerType>());
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          *ty.getAs<const clang::SubstTemplateTypeParmType>());
+    case clang::Type::Builtin:
+      return enumeratePrimitive(*ty.getAs<const clang::BuiltinType>());
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(*ty.getAs<const clang::ElaboratedType>());
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          *ty.getAs<const clang::TemplateSpecializationType>());
+
+    case clang::Type::Typedef:
+      return enumerateTypedef(*ty.getAs<const clang::TypedefType>());
+    case clang::Type::Using:
+      return enumerateUsing(*ty.getAs<const clang::UsingType>());
+
+    case clang::Type::ConstantArray:
+      return enumerateArray(*llvm::cast<const clang::ConstantArrayType>(&ty));
+    case clang::Type::Enum:
+      return enumerateEnum(*ty.getAs<const clang::EnumType>());
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Typedef& ClangTypeParser::enumerateUsing(const clang::UsingType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+  std::string name = ty.getFoundDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+
+  std::string name = ty.getDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string name = ty.getDecl()->getNameAsString();
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  std::map<int64_t, std::string> enumeratorMap;
+  if (options_.readEnumValues) {
+    for (const auto* enumerator : ty.getDecl()->enumerators()) {
+      enumeratorMap.emplace(
+          enumerator->getInitVal().getExtValue(),
+          enumerator->getNameAsString()
+      );
+    }
+  }
+
+  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+}
+
+Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
+  uint64_t len = ty.getSize().getLimitedValue();
+  auto& t = enumerateType(*ty.getElementType());
+  return makeType<Array>(ty, t, len);
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  if (auto* info = getContainerInfo(fqName)) {
+    auto& c = makeType<Container>(ty, *info, size, nullptr);
+    enumerateClassTemplateParams(ty, c.templateParams);
+    c.setAlign();
+    return c;
+  }
+
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+
+  params.reserve(list.size());
+  for (const auto& arg : list.asArray()) {
+    params.emplace_back(enumerateTemplateParam(arg));
+  }
+}
+
+TemplateParam ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return {ttype, qualifiers};
+    }
+    case clang::TemplateArgument::Integral: {
+      auto& ty = enumerateType(*p.getIntegralType());
+      llvm::SmallString<32> val;
+      p.getAsIntegral().toString(val);
+      return {ty, std::string(val)};
+    }
+    case clang::TemplateArgument::Template: {
+      return enumerateTemplateTemplateParam(p.getAsTemplate());
+    }
+
+#define X(name) \
+case clang::TemplateArgument::name: \
+  throw std::logic_error("unsupported template argument kind: " #name);
+
+  X(Null)
+  X(Declaration)
+  X(NullPtr)
+  X(TemplateExpansion)
+  X(Expression)
+  X(Pack)
+#undef X
+  }
+}
+
+TemplateParam ClangTypeParser::enumerateTemplateTemplateParam(const clang::TemplateName& tn) {
+  switch (tn.getKind()) {
+
+#define X(name) \
+case clang::TemplateName::name: \
+  throw std::logic_error("unsupported template name kind: " #name);
+
+X(Template)
+X(OverloadedTemplate)
+X(AssumedTemplate)
+X(QualifiedTemplate)
+X(DependentTemplate)
+X(SubstTemplateTemplateParm)
+X(SubstTemplateTemplateParmPack)
+X(UsingTemplate)
+#undef X
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function references
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Float:
+      return makeType<Primitive>(ty, Primitive::Kind::Float32);
+    case clang::BuiltinType::Double:
+    case clang::BuiltinType::LongDouble:
+      return makeType<Primitive>(ty, Primitive::Kind::Float64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      ty.dump();
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+ContainerInfo* ClangTypeParser::getContainerInfo(std::string_view fqName) const {
+  for (const auto& containerInfo : containers_) {
+    if (std::regex_search(fqName, containerInfo->matcher)) {
+      return containerInfo.get();
+    }
+  }
+  return nullptr;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
+    return true; // treat as complete
+
+  return !sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      clang::diag::err_type_unsupported);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
new file mode 100644
index 0000000..297a65a
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.h
@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ASTContext;
+class BuiltinType;
+class ConstantArrayType;
+class ElaboratedType;
+class EnumType;
+class LValueReferenceType;
+class PointerType;
+class RecordType;
+class Sema;
+class SubstTemplateTypeParmType;
+class TemplateArgument;
+class TemplateName;
+class TemplateSpecializationType;
+class Type;
+class TypedefType;
+class UsingType;
+}  // namespace clang
+
+struct ContainerInfo;
+
+namespace oi::detail::type_graph {
+
+class Array;
+class Class;
+class Enum;
+class Member;
+class Primitive;
+class Reference;
+class Type;
+class TypeGraph;
+class Typedef;
+struct TemplateParam;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+  bool readEnumValues = false;
+};
+
+/*
+ * ClangTypeParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given drgn_type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph, const std::vector<std::unique_ptr<ContainerInfo>>& containers, ClangTypeParserOptions options)
+      : typeGraph_{typeGraph}, containers_{containers}, options_{options} {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containers_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Type& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+  Typedef& enumerateTypedef(const clang::TypedefType&);
+  Typedef& enumerateUsing(const clang::UsingType&);
+
+  Array& enumerateArray(const clang::ConstantArrayType&);
+  Enum& enumerateEnum(const clang::EnumType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  TemplateParam enumerateTemplateParam(const clang::TemplateArgument&);
+  TemplateParam enumerateTemplateTemplateParam(const clang::TemplateName&);
+
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  ContainerInfo* getContainerInfo(std::string_view fqName) const;
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index c792766..5213b3d 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -194,6 +194,14 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& p) {
+  RecursiveVisitor::visit(p);
+  p.regenerateName();
+  std::string inputName{p.pointeeType().inputName()};
+  inputName += '*';
+  p.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..e6437d0 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index 45e3920..f07ca48 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -150,6 +150,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& p) {
+  if (prefix(p))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = p.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(p.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index 9970b3d..e2e5068 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& p) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;
diff --git a/oi/type_graph/RemoveTopLevelPointer.cpp b/oi/type_graph/RemoveTopLevelPointer.cpp
index 9ecaaaa..e1909a7 100644
--- a/oi/type_graph/RemoveTopLevelPointer.cpp
+++ b/oi/type_graph/RemoveTopLevelPointer.cpp
@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& p) {
+  topLevelType_ = &p.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/RemoveTopLevelPointer.h b/oi/type_graph/RemoveTopLevelPointer.h
index 611429f..6b70545 100644
--- a/oi/type_graph/RemoveTopLevelPointer.h
+++ b/oi/type_graph/RemoveTopLevelPointer.h
@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
 
  private:
   Type* topLevelType_ = nullptr;
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index c16821c..21c03c9 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -733,6 +734,61 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    name_ = pointeeType_.get().name() + "&";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index 9e05821..7a8f12a 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -122,6 +122,9 @@ class RecursiveVisitor : public Visitor {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -189,6 +192,10 @@ class RecursiveMutator : public Mutator {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }
diff --git a/test/integration/template_template_param.toml b/test/integration/template_template_param.toml
new file mode 100644
index 0000000..c64590d
--- /dev/null
+++ b/test/integration/template_template_param.toml
@@ -0,0 +1,16 @@
+includes = ["vector", "utility", "string"]
+
+definitions = '''
+template <typename K, typename V, template <typename, typename> typename Pair>
+struct bad_map {
+  std::vector<Pair<K, K>> keys;
+  std::vector<Pair<V, V>> values;
+};
+'''
+
+[cases]
+  [cases.int_int_empty]
+    param_types = ["const bad_map<int, std::string, std::pair>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index d1e345d..ec3810b 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -14,131 +14,80 @@
  * limitations under the License.
  */
 
+#include <clang/Tooling/CommonOptionsParser.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include <cstdlib>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/action/drop_while.hpp>
+#include <range/v3/algorithm/find.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/all.hpp>
+#include <range/v3/view/transform.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
 
-constexpr static OIOpts opts{
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
+static llvm::cl::OptionCategory OilgenCategory("oilgen options");
 
-void usage() {
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
-  std::cerr << opts;
+static llvm::cl::list<std::string> ConfigFiles("config-file",
+                                               llvm::cl::desc(R"(TODO HELP)"),
+                                               llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<int> DebugLevel("debug-level",
+                                     llvm::cl::desc(R"(TODO HELP)"),
+                                     llvm::cl::init(-1),
+                                     llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> DumpJit("dump-jit",
+                                          llvm::cl::desc(R"(TODO HELP)"),
+                                          llvm::cl::init("jit.cpp"),
+                                          llvm::cl::cat(OilgenCategory));
 
-  std::cerr << std::endl
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
-
-int main(int argc, char* argv[]) {
+int main(int argc, const char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
-  std::vector<fs::path> configFilePaths;
-  fs::path sourceFileDumpPath = "";
-  bool exitCode = false;
-  bool pic = false;
-
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
+  auto expectedParser =
+      clang::tooling::CommonOptionsParser::create(argc, argv, OilgenCategory);
+  if (!expectedParser) {
+    llvm::errs() << expectedParser.takeError();
+    return -1;
   }
+  clang::tooling::CommonOptionsParser& options = expectedParser.get();
+  auto& compilations = options.getCompilations();
 
-  if (optind >= argc) {
-    usage();
-    return EXIT_FAILURE;
-  }
-  fs::path primaryObject = argv[optind];
-
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
+  if (DebugLevel.getNumOccurrences()) {
+    google::LogToStderr();
+    google::SetStderrLogging(google::INFO);
+    google::SetVLOGLevel("*", DebugLevel);
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
+    gflags::SetCommandLineOption("minloglevel", "0");
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
-  oigen.setConfigFilePaths(std::move(configFilePaths));
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
-  oigen.setFailIfNothingGenerated(exitCode);
-  oigen.setUsePIC(pic);
+  oigen.setConfigFilePaths(ConfigFiles |
+                           ranges::views::transform([](const auto& p) {
+                             return std::filesystem::path(p);
+                           }) |
+                           ranges::to<std::vector>());
+  if (DumpJit.getNumOccurrences())
+    oigen.setSourceFileDumpPath(DumpJit.getValue());
 
-  SymbolService symbols(primaryObject);
+  oigen.setOutputPath("a.o");  // TODO: make this an opt
 
-  return oigen.generate(primaryObject, symbols);
+  // TODO: can we extract this from the original arguments?
+  // oigen.setUsePIC(args["pic"].as<bool>());
+
+  oigen.setFailIfNothingGenerated(true);
+  // compilations.getAllFiles();
+  // return oigen.generate(compilations, {"/data/users/jakehillion/fbsource/fbcode/object-introspection/oil/examples/compile-time/VectorOfStrings-Introspect.cpp"});
+  return oigen.generate(compilations, options.getSourcePathList());
 }
diff --git a/types/string_type.toml b/types/string_type.toml
index fa4a2ae..0d2ab0d 100644
--- a/types/string_type.toml
+++ b/types/string_type.toml
@@ -35,3 +35,34 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
     );
 }
 """
+
+traversal_func = """
+    bool sso = ((uintptr_t)container.data() <
+                (uintptr_t)(&container + sizeof(std::__cxx11::basic_string<T0>))) &&
+               ((uintptr_t)container.data() >= (uintptr_t)&container);
+
+    return returnArg.write(container.capacity())
+        .write(sso)
+        .write(container.size());
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+uint64_t capacity = std::get<ParsedData::VarInt>(d.val).value;
+el.container_stats.emplace(result::Element::ContainerStats { .capacity = capacity });
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+bool sso = std::get<ParsedData::VarInt>(d.val).value;
+if (!sso)
+  el.exclusive_size += el.container_stats->capacity * sizeof(T0);
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats->length = std::get<ParsedData::VarInt>(d.val).value;
+"""

From f4af364497ff678501ff7663aba19926c237094b Mon Sep 17 00:00:00 2001
From: Alastair Robertson <ajor@meta.com>
Date: Tue, 12 Dec 2023 11:01:11 -0800
Subject: [PATCH 047/188] TypeGraph: Fix handling for classes which inherit
 from containers

We previously moved container identification later in CodeGen in order
to preserve information for AlignmentCalc.

However, Flattener needs to know if a class is a container in order to
apply its special handling for this case.

This new approach moves container identification back into DrgnParser,
but has Container own a type node, representing its layout. This
underlying type node can be used for calculating a container's
alignment.
---
 oi/CodeGen.cpp                       |   8 +-
 oi/type_graph/AlignmentCalc.cpp      |  18 +--
 oi/type_graph/AlignmentCalc.h        |   1 +
 oi/type_graph/CMakeLists.txt         |   1 -
 oi/type_graph/DrgnParser.cpp         |  23 +++-
 oi/type_graph/DrgnParser.h           |  12 +-
 oi/type_graph/Flattener.cpp          |  10 --
 oi/type_graph/Flattener.h            |   1 -
 oi/type_graph/IdentifyContainers.cpp |  73 ----------
 oi/type_graph/IdentifyContainers.h   |  58 --------
 oi/type_graph/Printer.cpp            |  13 +-
 oi/type_graph/Printer.h              |   2 +-
 oi/type_graph/Prune.cpp              |  19 +--
 oi/type_graph/Prune.h                |   1 +
 oi/type_graph/TypeIdentifier.cpp     |   3 +-
 oi/type_graph/Types.h                |  54 +++++---
 oi/type_graph/Visitor.h              |   7 +
 test/CMakeLists.txt                  |   1 -
 test/TypeGraphParser.cpp             |  26 +++-
 test/TypeGraphParser.h               |   3 +
 test/test_add_children.cpp           |  22 +--
 test/test_alignment_calc.cpp         |  38 +++++-
 test/test_codegen.cpp                | 100 ++++++++++++--
 test/test_drgn_parser.cpp            |  28 +++-
 test/test_flattener.cpp              |  24 +++-
 test/test_identify_containers.cpp    | 194 ---------------------------
 test/test_prune.cpp                  |  30 +++++
 test/test_type_identifier.cpp        |  12 ++
 test/type_graph_utils.cpp            |   9 +-
 29 files changed, 356 insertions(+), 435 deletions(-)
 delete mode 100644 oi/type_graph/IdentifyContainers.cpp
 delete mode 100644 oi/type_graph/IdentifyContainers.h
 delete mode 100644 test/test_identify_containers.cpp

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index bef102f..5a4b9eb 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -32,7 +32,6 @@
 #include "type_graph/DrgnParser.h"
 #include "type_graph/EnforceCompatibility.h"
 #include "type_graph/Flattener.h"
-#include "type_graph/IdentifyContainers.h"
 #include "type_graph/KeyCapture.h"
 #include "type_graph/NameGen.h"
 #include "type_graph/Prune.h"
@@ -56,7 +55,6 @@ using type_graph::DrgnParserOptions;
 using type_graph::EnforceCompatibility;
 using type_graph::Enum;
 using type_graph::Flattener;
-using type_graph::IdentifyContainers;
 using type_graph::KeyCapture;
 using type_graph::Member;
 using type_graph::NameGen;
@@ -1147,7 +1145,7 @@ void CodeGen::addDrgnRoot(struct drgn_type* drgnType, TypeGraph& typeGraph) {
   DrgnParserOptions options{
       .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
   };
-  DrgnParser drgnParser{typeGraph, options};
+  DrgnParser drgnParser{typeGraph, containerInfos_, options};
   Type& parsedRoot = drgnParser.parse(drgnType);
   typeGraph.addRoot(parsedRoot);
 }
@@ -1159,7 +1157,6 @@ void CodeGen::transform(TypeGraph& typeGraph) {
   pm.addPass(RemoveTopLevelPointer::createPass());
   pm.addPass(Flattener::createPass());
   pm.addPass(AlignmentCalc::createPass());
-  pm.addPass(IdentifyContainers::createPass(containerInfos_));
   pm.addPass(TypeIdentifier::createPass(config_.passThroughTypes));
   if (config_.features[Feature::PruneTypeGraph])
     pm.addPass(Prune::createPass());
@@ -1169,13 +1166,12 @@ void CodeGen::transform(TypeGraph& typeGraph) {
     DrgnParserOptions options{
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
-    DrgnParser drgnParser{typeGraph, options};
+    DrgnParser drgnParser{typeGraph, containerInfos_, options};
     pm.addPass(AddChildren::createPass(drgnParser, symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(Flattener::createPass());
     pm.addPass(AlignmentCalc::createPass());
-    pm.addPass(IdentifyContainers::createPass(containerInfos_));
     pm.addPass(TypeIdentifier::createPass(config_.passThroughTypes));
     if (config_.features[Feature::PruneTypeGraph])
       pm.addPass(Prune::createPass());
diff --git a/oi/type_graph/AlignmentCalc.cpp b/oi/type_graph/AlignmentCalc.cpp
index 31a7a30..cf23eff 100644
--- a/oi/type_graph/AlignmentCalc.cpp
+++ b/oi/type_graph/AlignmentCalc.cpp
@@ -48,15 +48,7 @@ void AlignmentCalc::accept(Type& type) {
 }
 
 void AlignmentCalc::visit(Class& c) {
-  for (const auto& param : c.templateParams) {
-    accept(param.type());
-  }
-  for (const auto& parent : c.parents) {
-    accept(parent.type());
-  }
-  for (const auto& child : c.children) {
-    accept(child);
-  }
+  RecursiveVisitor::visit(c);
 
   uint64_t alignment = 1;
   for (auto& member : c.members) {
@@ -82,4 +74,12 @@ void AlignmentCalc::visit(Class& c) {
   }
 }
 
+void AlignmentCalc::visit(Container& c) {
+  RecursiveVisitor::visit(c);
+
+  if (c.underlying()) {
+    c.setAlign(c.underlying()->align());
+  }
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/AlignmentCalc.h b/oi/type_graph/AlignmentCalc.h
index 4949341..b89f114 100644
--- a/oi/type_graph/AlignmentCalc.h
+++ b/oi/type_graph/AlignmentCalc.h
@@ -41,6 +41,7 @@ class AlignmentCalc final : public RecursiveVisitor {
 
   void accept(Type& type) override;
   void visit(Class& c) override;
+  void visit(Container& c) override;
 
  private:
   std::unordered_set<Type*> visited_;
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 940c031..4e8f622 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -5,7 +5,6 @@ add_library(type_graph
   DrgnParser.cpp
   EnforceCompatibility.cpp
   Flattener.cpp
-  IdentifyContainers.cpp
   KeyCapture.cpp
   NameGen.cpp
   PassManager.cpp
diff --git a/oi/type_graph/DrgnParser.cpp b/oi/type_graph/DrgnParser.cpp
index 0116edf..07202b0 100644
--- a/oi/type_graph/DrgnParser.cpp
+++ b/oi/type_graph/DrgnParser.cpp
@@ -25,6 +25,8 @@ extern "C" {
 #include <drgn.h>
 }
 
+#include <regex>
+
 namespace oi::detail::type_graph {
 namespace {
 
@@ -146,7 +148,7 @@ Type& DrgnParser::enumerateType(struct drgn_type* type) {
   return *t;
 }
 
-Class& DrgnParser::enumerateClass(struct drgn_type* type) {
+Type& DrgnParser::enumerateClass(struct drgn_type* type) {
   std::string fqName;
   char* nameStr = nullptr;
   size_t length = 0;
@@ -180,14 +182,20 @@ Class& DrgnParser::enumerateClass(struct drgn_type* type) {
                             std::to_string(drgn_type_kind(type))};
   }
 
-  auto& c = makeType<Class>(
-      type, kind, std::move(name), std::move(fqName), size, virtuality);
+  Class& c =
+      makeType<Class>(type, kind, std::move(name), fqName, size, virtuality);
 
   enumerateClassTemplateParams(type, c.templateParams);
   enumerateClassParents(type, c.parents);
   enumerateClassMembers(type, c.members);
   enumerateClassFunctions(type, c.functions);
 
+  if (auto* info = getContainerInfo(fqName)) {
+    auto& container = makeType<Container>(type, *info, size, &c);
+    container.templateParams = c.templateParams;
+    return container;
+  }
+
   return c;
 }
 
@@ -505,6 +513,15 @@ bool DrgnParser::chasePointer() const {
   return options_.chaseRawPointers;
 }
 
+ContainerInfo* DrgnParser::getContainerInfo(const std::string& fqName) const {
+  for (const auto& containerInfo : containers_) {
+    if (std::regex_search(fqName, containerInfo->matcher)) {
+      return containerInfo.get();
+    }
+  }
+  return nullptr;
+}
+
 DrgnParserError::DrgnParserError(const std::string& msg, struct drgn_error* err)
     : std::runtime_error{msg + ": " + std::to_string(err->code) + " " +
                          err->message},
diff --git a/oi/type_graph/DrgnParser.h b/oi/type_graph/DrgnParser.h
index 5b8fbe3..edb566d 100644
--- a/oi/type_graph/DrgnParser.h
+++ b/oi/type_graph/DrgnParser.h
@@ -51,14 +51,16 @@ struct DrgnParserOptions {
  */
 class DrgnParser {
  public:
-  DrgnParser(TypeGraph& typeGraph, DrgnParserOptions options)
-      : typeGraph_(typeGraph), options_(options) {
+  DrgnParser(TypeGraph& typeGraph,
+             const std::vector<std::unique_ptr<ContainerInfo>>& containers,
+             DrgnParserOptions options)
+      : typeGraph_(typeGraph), containers_(containers), options_(options) {
   }
   Type& parse(struct drgn_type* root);
 
  private:
   Type& enumerateType(struct drgn_type* type);
-  Class& enumerateClass(struct drgn_type* type);
+  Type& enumerateClass(struct drgn_type* type);
   Enum& enumerateEnum(struct drgn_type* type);
   Typedef& enumerateTypedef(struct drgn_type* type);
   Type& enumeratePointer(struct drgn_type* type);
@@ -81,10 +83,11 @@ class DrgnParser {
   template <typename T, typename... Args>
   T& makeType(struct drgn_type* drgnType, Args&&... args) {
     auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
-    drgn_types_.insert({drgnType, newType});
+    drgn_types_.insert_or_assign(drgnType, newType);
     return newType;
   }
   bool chasePointer() const;
+  ContainerInfo* getContainerInfo(const std::string& fqName) const;
 
   // Store a mapping of drgn types to type graph nodes for deduplication during
   // parsing. This stops us getting caught in cycles.
@@ -92,6 +95,7 @@ class DrgnParser {
       drgn_types_;
 
   TypeGraph& typeGraph_;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containers_;
   int depth_;
   DrgnParserOptions options_;
 };
diff --git a/oi/type_graph/Flattener.cpp b/oi/type_graph/Flattener.cpp
index 57e3748..28a9f32 100644
--- a/oi/type_graph/Flattener.cpp
+++ b/oi/type_graph/Flattener.cpp
@@ -134,8 +134,6 @@ void Flattener::visit(Class& c) {
   //   };
   //    TODO comment about virtual inheritance
 
-  // TODO alignment of parent classes
-
   // Flatten types referenced by template params, parents and members
   for (const auto& param : c.templateParams) {
     accept(param.type());
@@ -207,12 +205,4 @@ void Flattener::visit(Class& c) {
   }
 }
 
-void Flattener::visit(Container& c) {
-  // Containers themselves don't need to be flattened, but their template
-  // parameters might need to be
-  for (const auto& templateParam : c.templateParams) {
-    accept(templateParam.type());
-  }
-}
-
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/Flattener.h b/oi/type_graph/Flattener.h
index 1d46f02..95f78c6 100644
--- a/oi/type_graph/Flattener.h
+++ b/oi/type_graph/Flattener.h
@@ -42,7 +42,6 @@ class Flattener : public RecursiveVisitor {
 
   void accept(Type& type) override;
   void visit(Class& c) override;
-  void visit(Container& c) override;
 
   static const inline std::string ParentPrefix = "__oi_parent";
 
diff --git a/oi/type_graph/IdentifyContainers.cpp b/oi/type_graph/IdentifyContainers.cpp
deleted file mode 100644
index c6207c2..0000000
--- a/oi/type_graph/IdentifyContainers.cpp
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "IdentifyContainers.h"
-
-#include <regex>
-
-#include "TypeGraph.h"
-#include "oi/ContainerInfo.h"
-
-namespace oi::detail::type_graph {
-
-Pass IdentifyContainers::createPass(
-    const std::vector<std::unique_ptr<ContainerInfo>>& containers) {
-  auto fn = [&containers](TypeGraph& typeGraph, NodeTracker&) {
-    IdentifyContainers typeId{typeGraph, containers};
-    for (auto& type : typeGraph.rootTypes()) {
-      type = typeId.mutate(type);
-    }
-  };
-
-  return Pass("IdentifyContainers", fn);
-}
-
-IdentifyContainers::IdentifyContainers(
-    TypeGraph& typeGraph,
-    const std::vector<std::unique_ptr<ContainerInfo>>& containers)
-    : tracker_(typeGraph.size()),
-      typeGraph_(typeGraph),
-      containers_(containers) {
-}
-
-Type& IdentifyContainers::mutate(Type& type) {
-  if (Type* mutated = tracker_.get(type))
-    return *mutated;
-
-  Type& mutated = type.accept(*this);
-  tracker_.set(type, mutated);
-  return mutated;
-}
-
-Type& IdentifyContainers::visit(Class& c) {
-  for (const auto& containerInfo : containers_) {
-    if (!std::regex_search(c.fqName(), containerInfo->matcher)) {
-      continue;
-    }
-
-    auto& container = typeGraph_.makeType<Container>(*containerInfo, c.size());
-    container.templateParams = c.templateParams;
-
-    tracker_.set(c, container);
-    RecursiveMutator::visit(container);
-    return container;
-  }
-
-  tracker_.set(c, c);
-  RecursiveMutator::visit(c);
-  return c;
-}
-
-}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/IdentifyContainers.h b/oi/type_graph/IdentifyContainers.h
deleted file mode 100644
index b03c8bd..0000000
--- a/oi/type_graph/IdentifyContainers.h
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#pragma once
-
-#include <functional>
-#include <unordered_map>
-#include <vector>
-
-#include "NodeTracker.h"
-#include "PassManager.h"
-#include "Types.h"
-#include "Visitor.h"
-#include "oi/ContainerInfo.h"
-
-namespace oi::detail::type_graph {
-
-class TypeGraph;
-
-/*
- * IdentifyContainers
- *
- * Walks a flattened type graph and replaces type nodes based on container
- * definition TOML files.
- */
-class IdentifyContainers : public RecursiveMutator {
- public:
-  static Pass createPass(
-      const std::vector<std::unique_ptr<ContainerInfo>>& containers);
-
-  IdentifyContainers(
-      TypeGraph& typeGraph,
-      const std::vector<std::unique_ptr<ContainerInfo>>& containers);
-
-  using RecursiveMutator::mutate;
-
-  Type& mutate(Type& type) override;
-  Type& visit(Class& c) override;
-
- private:
-  MutationTracker tracker_;
-  TypeGraph& typeGraph_;
-  const std::vector<std::unique_ptr<ContainerInfo>>& containers_;
-};
-
-}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index e7d70eb..45e3920 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -85,7 +85,7 @@ void Printer::visit(const Class& c) {
     print_function(function);
   }
   for (auto& child : c.children) {
-    print_child(child);
+    print_type("Child", child);
   }
 }
 
@@ -93,11 +93,14 @@ void Printer::visit(const Container& c) {
   if (prefix(c))
     return;
 
-  out_ << "Container: " << c.name() << " (size: " << c.size() << ")"
-       << std::endl;
+  out_ << "Container: " << c.name() << " (size: " << c.size()
+       << align_str(c.align()) << ")" << std::endl;
   for (const auto& param : c.templateParams) {
     print_param(param);
   }
+  if (c.underlying()) {
+    print_type("Underlying", *c.underlying());
+  }
 }
 
 void Printer::visit(const Primitive& p) {
@@ -240,10 +243,10 @@ void Printer::print_function(const Function& function) {
   depth_--;
 }
 
-void Printer::print_child(const Type& child) {
+void Printer::print_type(std::string_view header, const Type& child) {
   depth_++;
   prefix();
-  out_ << "Child" << std::endl;
+  out_ << header << std::endl;
   print(child);
   depth_--;
 }
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index a66581e..9970b3d 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -51,7 +51,7 @@ class Printer : public ConstVisitor {
   void print_parent(const Parent& parent);
   void print_member(const Member& member);
   void print_function(const Function& function);
-  void print_child(const Type& child);
+  void print_type(std::string_view header, const Type& child);
   void print_value(const std::string& value);
   void print_qualifiers(const QualifierSet& qualifiers);
   void print_enumerator(int64_t val, const std::string& name);
diff --git a/oi/type_graph/Prune.cpp b/oi/type_graph/Prune.cpp
index 903a8c2..8effd3b 100644
--- a/oi/type_graph/Prune.cpp
+++ b/oi/type_graph/Prune.cpp
@@ -39,18 +39,7 @@ void Prune::accept(Type& type) {
 }
 
 void Prune::visit(Class& c) {
-  for (const auto& param : c.templateParams) {
-    accept(param.type());
-  }
-  for (const auto& parent : c.parents) {
-    accept(parent.type());
-  }
-  for (const auto& member : c.members) {
-    accept(member.type());
-  }
-  for (const auto& child : c.children) {
-    accept(child);
-  }
+  RecursiveVisitor::visit(c);
 
   c.templateParams.clear();
   c.parents.clear();
@@ -62,4 +51,10 @@ void Prune::visit(Class& c) {
   c.functions.shrink_to_fit();
 }
 
+void Prune::visit(Container& c) {
+  RecursiveVisitor::visit(c);
+
+  c.setUnderlying(nullptr);
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/Prune.h b/oi/type_graph/Prune.h
index d444ea3..c0701db 100644
--- a/oi/type_graph/Prune.h
+++ b/oi/type_graph/Prune.h
@@ -43,6 +43,7 @@ class Prune : public RecursiveVisitor {
 
   void accept(Type& type) override;
   void visit(Class& c) override;
+  void visit(Container& c) override;
 
  private:
   NodeTracker& tracker_;
diff --git a/oi/type_graph/TypeIdentifier.cpp b/oi/type_graph/TypeIdentifier.cpp
index 17fa43a..bba08f8 100644
--- a/oi/type_graph/TypeIdentifier.cpp
+++ b/oi/type_graph/TypeIdentifier.cpp
@@ -79,7 +79,8 @@ void TypeIdentifier::visit(Container& c) {
               it != passThroughTypeDummys_.end()) {
             dummy = &it->second.get();
           } else {
-            dummy = &typeGraph_.makeType<Container>(info, param.type().size());
+            dummy = &typeGraph_.makeType<Container>(
+                info, param.type().size(), paramClass);
             dummy->templateParams = paramClass->templateParams;
             passThroughTypeDummys_.insert(it,
                                           {paramClass->id(), std::ref(*dummy)});
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f503886..c16821c 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -150,7 +150,6 @@ struct Function {
   int virtuality;
 };
 
-class Class;
 struct Parent {
   Parent(Type& type, uint64_t bitOffset) : type_(type), bitOffset(bitOffset) {
   }
@@ -298,10 +297,6 @@ class Class : public Type {
 
   DECLARE_ACCEPT
 
-  Kind kind() const {
-    return kind_;
-  }
-
   virtual const std::string& name() const override {
     return name_;
   }
@@ -326,12 +321,16 @@ class Class : public Type {
     return align_;
   }
 
+  void setAlign(uint64_t alignment) {
+    align_ = alignment;
+  }
+
   virtual NodeId id() const override {
     return id_;
   }
 
-  void setAlign(uint64_t alignment) {
-    align_ = alignment;
+  Kind kind() const {
+    return kind_;
   }
 
   int virtuality() const {
@@ -371,10 +370,19 @@ class Class : public Type {
   bool packed_ = false;
 };
 
+/*
+ * Container
+ *
+ * A type of class for which we can do special processing.
+ */
 class Container : public Type {
  public:
-  Container(NodeId id, const ContainerInfo& containerInfo, size_t size)
+  Container(NodeId id,
+            const ContainerInfo& containerInfo,
+            size_t size,
+            Type* underlying)
       : containerInfo_(containerInfo),
+        underlying_(underlying),
         name_(containerInfo.typeName),
         inputName_(containerInfo.typeName),
         size_(size),
@@ -386,6 +394,7 @@ class Container : public Type {
             const ContainerInfo& containerInfo)
       : templateParams(other.templateParams),
         containerInfo_(containerInfo),
+        underlying_(other.underlying_),
         name_(other.name_),
         inputName_(other.inputName_),
         size_(other.size_),
@@ -396,22 +405,18 @@ class Container : public Type {
 
   DECLARE_ACCEPT
 
-  const std::string& containerName() const {
-    return containerInfo_.typeName;
-  }
-
   virtual const std::string& name() const override {
     return name_;
   }
 
-  void setName(std::string name) {
-    name_ = std::move(name);
-  }
-
   virtual std::string_view inputName() const override {
     return inputName_;
   }
 
+  void setName(std::string name) {
+    name_ = std::move(name);
+  }
+
   void setInputName(std::string name) {
     inputName_ = std::move(name);
   }
@@ -424,18 +429,31 @@ class Container : public Type {
     return align_;
   }
 
+  void setAlign(uint64_t alignment) {
+    align_ = alignment;
+  }
+
   virtual NodeId id() const override {
     return id_;
   }
 
-  void setAlign(uint64_t alignment) {
-    align_ = alignment;
+  const std::string& containerName() const {
+    return containerInfo_.typeName;
+  }
+
+  Type* underlying() const {
+    return underlying_;
+  }
+
+  void setUnderlying(Type* underlying) {
+    underlying_ = underlying;
   }
 
   std::vector<TemplateParam> templateParams;
   const ContainerInfo& containerInfo_;
 
  private:
+  Type* underlying_;
   std::string name_;
   std::string inputName_;
   size_t size_;
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index 7e4395e..9e05821 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -107,6 +107,7 @@ class RecursiveVisitor : public Visitor {
     for (const auto& param : c.templateParams) {
       accept(param.type());
     }
+    accept(c.underlying());
   }
   virtual void visit(Primitive&) {
   }
@@ -140,6 +141,11 @@ class RecursiveMutator : public Mutator {
  public:
   virtual ~RecursiveMutator() = default;
   virtual Type& mutate(Type&) = 0;
+  virtual Type* mutate(Type* type) {
+    if (type)
+      return &mutate(*type);
+    return nullptr;
+  }
   virtual Type& visit(Incomplete& i) {
     return i;
   }
@@ -162,6 +168,7 @@ class RecursiveMutator : public Mutator {
     for (auto& param : c.templateParams) {
       param.setType(mutate(param.type()));
     }
+    c.setUnderlying(mutate(c.underlying()));
     return c;
   }
   virtual Type& visit(Primitive& p) {
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 287ebcd..24b8f68 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -43,7 +43,6 @@ add_executable(test_type_graph
   test_drgn_parser.cpp
   test_enforce_compatibility.cpp
   test_flattener.cpp
-  test_identify_containers.cpp
   test_key_capture.cpp
   test_name_gen.cpp
   test_node_tracker.cpp
diff --git a/test/TypeGraphParser.cpp b/test/TypeGraphParser.cpp
index 7b6f425..f54fe08 100644
--- a/test/TypeGraphParser.cpp
+++ b/test/TypeGraphParser.cpp
@@ -265,10 +265,11 @@ Type& TypeGraphParser::parseType(std::string_view& input, size_t rootIndent) {
 
     auto size = parseNumericAttribute(line, nodeTypeName, "size: ");
 
-    Container& c = typeGraph_.makeType<Container>(id, info, size);
+    Container& c = typeGraph_.makeType<Container>(id, info, size, nullptr);
     nodesById_.insert({id, c});
 
     parseParams(c, input, indent + 2);
+    parseUnderlying(c, input, indent + 2);
 
     type = &c;
   } else if (nodeTypeName == "Primitive") {
@@ -452,3 +453,26 @@ void TypeGraphParser::parseChildren(Class& c,
   // No more children for us - put back the line we just read
   input = origInput;
 }
+
+void TypeGraphParser::parseUnderlying(Container& c,
+                                      std::string_view& input,
+                                      size_t rootIndent) {
+  std::string_view origInput = input;
+  std::string_view line;
+  getline(input, line);
+
+  size_t indent = stripIndent(line);
+  if (indent != rootIndent) {
+    input = origInput;
+    return;
+  }
+
+  // Format: "Underlying"
+  if (!tryRemovePrefix(line, "Underlying")) {
+    input = origInput;
+    return;
+  }
+
+  Type& type = parseType(input, rootIndent + 2);
+  c.setUnderlying(&type);
+}
diff --git a/test/TypeGraphParser.h b/test/TypeGraphParser.h
index 626c4a4..640b706 100644
--- a/test/TypeGraphParser.h
+++ b/test/TypeGraphParser.h
@@ -34,6 +34,9 @@ class TypeGraphParser {
   void parseMembers(Class& c, std::string_view& input, size_t rootIndent);
   void parseFunctions(Class& c, std::string_view& input, size_t rootIndent);
   void parseChildren(Class& c, std::string_view& input, size_t rootIndent);
+  void parseUnderlying(Container& c,
+                       std::string_view& input,
+                       size_t rootIndent);
 };
 
 class TypeGraphParserError : public std::runtime_error {
diff --git a/test/test_add_children.cpp b/test/test_add_children.cpp
index aca046e..fd8e66b 100644
--- a/test/test_add_children.cpp
+++ b/test/test_add_children.cpp
@@ -79,11 +79,11 @@ TEST_F(AddChildrenTest, InheritancePolymorphic) {
          Function: A
          Function: A
          Child
-[17]       Class: B (size: 40)
+[18]       Class: B (size: 40)
              Parent (offset: 0)
                [0]
              Member: vec_b (offset: 16)
-[4]            Class: vector<int, std::allocator<int> > (size: 24)
+[17]           Container: std::vector (size: 24)
                  Param
                    Primitive: int32_t
                  Param
@@ -106,15 +106,17 @@ TEST_F(AddChildrenTest, InheritancePolymorphic) {
                      Function: ~allocator
                      Function: allocate
                      Function: deallocate
-                 *
+                 Underlying
+[4]                Class: vector<int, std::allocator<int> > (size: 24)
+*
              Function: ~B (virtual)
              Function: myfunc (virtual)
              Function: B
              Function: B
              Child
-[19]           Class: C (size: 48)
+[20]           Class: C (size: 48)
                  Parent (offset: 0)
-                   [17]
+                   [18]
                  Member: int_c (offset: 40)
                    Primitive: int32_t
                  Function: ~C (virtual)
@@ -135,11 +137,11 @@ TEST_F(AddChildrenTest, InheritancePolymorphic) {
          Function: ~A (virtual)
          Function: myfunc (virtual)
          Child
-[13]       Class: B (size: 40)
+[14]       Class: B (size: 40)
              Parent (offset: 0)
                [0]
              Member: vec_b (offset: 16)
-[4]            Class: vector<int, std::allocator<int> > (size: 24)
+[13]           Container: std::vector (size: 24)
                  Param
                    Primitive: int32_t
                  Param
@@ -159,6 +161,8 @@ TEST_F(AddChildrenTest, InheritancePolymorphic) {
                      Function: ~allocator
                      Function: allocate
                      Function: deallocate
+                 Underlying
+[4]                Class: vector<int, std::allocator<int> > (size: 24)
                  *
              Function: operator=
              Function: B
@@ -166,9 +170,9 @@ TEST_F(AddChildrenTest, InheritancePolymorphic) {
              Function: ~B (virtual)
              Function: myfunc (virtual)
              Child
-[15]           Class: C (size: 48)
+[16]           Class: C (size: 48)
                  Parent (offset: 0)
-                   [13]
+                   [14]
                  Member: int_c (offset: 40)
                    Primitive: int32_t
                  Function: operator=
diff --git a/test/test_alignment_calc.cpp b/test/test_alignment_calc.cpp
index 5bc4573..2526cbe 100644
--- a/test/test_alignment_calc.cpp
+++ b/test/test_alignment_calc.cpp
@@ -58,16 +58,16 @@ TEST(AlignmentCalcTest, StructInContainer) {
           Member: n (offset: 0)
             Primitive: int8_t
           Member: n (offset: 8)
-            Primitive: int64_t
+            Primitive: int32_t
 )",
        R"(
 [0] Container: std::vector (size: 8)
       Param
-[1]     Class: MyClass (size: 16, align: 8)
+[1]     Class: MyClass (size: 16, align: 4)
           Member: n (offset: 0, align: 1)
             Primitive: int8_t
-          Member: n (offset: 8, align: 8)
-            Primitive: int64_t
+          Member: n (offset: 8, align: 4)
+            Primitive: int32_t
 )");
 }
 
@@ -264,3 +264,33 @@ TEST(AlignmentCalcTest, Typedef) {
               Primitive: int8_t
 )");
 }
+
+TEST(AlignmentCalcTest, Container) {
+  test(AlignmentCalc::createPass(),
+       R"(
+[0] Container: std::vector (size: 24)
+      Underlying
+[1]     Class: vector (size: 24)
+          Member: n (offset: 0)
+            Primitive: int8_t
+          Member: s (offset: 4)
+[2]         Struct: MyStruct (size: 8)
+              Member: n1 (offset: 0)
+                Primitive: int32_t
+              Member: n2 (offset: 4)
+                Primitive: int32_t
+)",
+       R"(
+[0] Container: std::vector (size: 24, align: 4)
+      Underlying
+[1]     Class: vector (size: 24, align: 4)
+          Member: n (offset: 0, align: 1)
+            Primitive: int8_t
+          Member: s (offset: 4, align: 4)
+[2]         Struct: MyStruct (size: 8, align: 4)
+              Member: n1 (offset: 0, align: 4)
+                Primitive: int32_t
+              Member: n2 (offset: 4, align: 4)
+                Primitive: int32_t
+)");
+}
diff --git a/test/test_codegen.cpp b/test/test_codegen.cpp
index 660c411..3e7855e 100644
--- a/test/test_codegen.cpp
+++ b/test/test_codegen.cpp
@@ -45,13 +45,15 @@ void testTransform(OICodeGen::Config& config,
 
 void testTransform(std::string_view input, std::string_view expectedAfter) {
   OICodeGen::Config config;
+  config.features[Feature::PruneTypeGraph] = true;
+  config.features[Feature::TreeBuilderV2] = true;
   testTransform(config, input, expectedAfter);
 }
 }  // namespace
 
 TEST(CodeGenTest, TransformContainerAllocator) {
   testTransform(R"(
-[0] Class: std::vector (size: 24)
+[0] Container: std::vector (size: 24)
       Param
         Primitive: int32_t
       Param
@@ -62,18 +64,18 @@ TEST(CodeGenTest, TransformContainerAllocator) {
           Function: deallocate
 )",
                 R"(
-[2] Container: std::vector<int32_t, DummyAllocator<int32_t, 8, 1, 3>> (size: 24)
+[0] Container: std::vector<int32_t, DummyAllocator<int32_t, 8, 1, 2>> (size: 24)
       Param
         Primitive: int32_t
       Param
-[3]     DummyAllocator [MyAlloc] (size: 8, align: 1)
+[2]     DummyAllocator [MyAlloc] (size: 8, align: 1)
           Primitive: int32_t
 )");
 }
 
 TEST(CodeGenTest, TransformContainerAllocatorParamInParent) {
   testTransform(R"(
-[0] Class: std::map (size: 24)
+[0] Container: std::map (size: 24)
       Param
         Primitive: int32_t
       Param
@@ -83,7 +85,7 @@ TEST(CodeGenTest, TransformContainerAllocatorParamInParent) {
           Parent (offset: 0)
 [2]         Struct: MyAllocBase<std::pair<const int, int>> (size: 1)
               Param
-[3]             Class: std::pair (size: 8)
+[3]             Container: std::pair (size: 8)
                   Param
                     Primitive: int32_t
                     Qualifiers: const
@@ -95,14 +97,14 @@ TEST(CodeGenTest, TransformContainerAllocatorParamInParent) {
           Function: deallocate
 )",
                 R"(
-[4] Container: std::map<int32_t, int32_t, DummyAllocator<std::pair<int32_t const, int32_t>, 0, 1, 6>> (size: 24)
+[0] Container: std::map<int32_t, int32_t, DummyAllocator<std::pair<int32_t const, int32_t>, 0, 1, 4>> (size: 24)
       Param
         Primitive: int32_t
       Param
         Primitive: int32_t
       Param
-[6]     DummyAllocator [MyAlloc<std::pair<const int, int>>] (size: 0, align: 1)
-[5]       Container: std::pair<int32_t const, int32_t> (size: 8)
+[4]     DummyAllocator [MyAlloc<std::pair<const int, int>>] (size: 0, align: 1)
+[3]       Container: std::pair<int32_t const, int32_t> (size: 8)
             Param
               Primitive: int32_t
               Qualifiers: const
@@ -157,7 +159,7 @@ TEST(CodeGenTest, UnionMembersAlignment) {
 
 TEST(CodeGenTest, ReplaceContainersAndDummies) {
   testTransform(R"(
-[0] Class: std::vector (size: 24)
+[0] Container: std::vector (size: 24)
       Param
         Primitive: uint32_t
       Param
@@ -168,11 +170,87 @@ TEST(CodeGenTest, ReplaceContainersAndDummies) {
           Function: deallocate
 )",
                 R"(
-[2] Container: std::vector<uint32_t, DummyAllocator<uint32_t, 0, 1, 3>> (size: 24)
+[0] Container: std::vector<uint32_t, DummyAllocator<uint32_t, 0, 1, 2>> (size: 24)
       Param
         Primitive: uint32_t
       Param
-[3]     DummyAllocator [allocator<int>] (size: 0, align: 1)
+[2]     DummyAllocator [allocator<int>] (size: 0, align: 1)
           Primitive: uint32_t
 )");
 }
+
+TEST(CodeGenTest, ContainerAlignment) {
+  testTransform(R"(
+[0] Class: MyClass (size: 24)
+      Member: container (offset: 0)
+[1]     Container: std::vector (size: 24)
+          Param
+            Primitive: int32_t
+          Underlying
+[2]         Class: vector (size: 24)
+              Member: __impl__ (offset: 0)
+                Primitive: StubbedPointer
+              Member: __impl__ (offset: 8)
+                Primitive: StubbedPointer
+              Member: __impl__ (offset: 16)
+                Primitive: StubbedPointer
+)",
+                R"(
+[0] Class: MyClass_0 [MyClass] (size: 24, align: 8)
+      Member: container_0 [container] (offset: 0, align: 8)
+[1]     Container: std::vector<int32_t> (size: 24, align: 8)
+          Param
+            Primitive: int32_t
+)");
+}
+
+TEST(CodeGenTest, InheritFromContainer) {
+  testTransform(R"(
+[0] Class: MyClass (size: 24)
+      Parent (offset: 0)
+[1]     Container: std::vector (size: 24)
+          Param
+            Primitive: int32_t
+          Underlying
+[2]         Class: vector (size: 24)
+              Member: __impl__ (offset: 0)
+                Primitive: StubbedPointer
+              Member: __impl__ (offset: 8)
+                Primitive: StubbedPointer
+              Member: __impl__ (offset: 16)
+                Primitive: StubbedPointer
+)",
+                R"(
+[0] Class: MyClass_0 [MyClass] (size: 24, align: 8)
+      Member: __oi_parent_0 [__oi_parent] (offset: 0, align: 8)
+[1]     Container: std::vector<int32_t> (size: 24, align: 8)
+          Param
+            Primitive: int32_t
+)");
+}
+
+TEST(CodeGenTest, InheritFromContainerCompat) {
+  OICodeGen::Config config;
+  testTransform(config,
+                R"(
+[0] Class: MyClass (size: 24)
+      Parent (offset: 0)
+[1]     Container: std::vector (size: 24)
+          Param
+            Primitive: int32_t
+          Underlying
+[2]         Class: vector (size: 24)
+              Member: __impl__ (offset: 0)
+                Primitive: StubbedPointer
+              Member: __impl__ (offset: 8)
+                Primitive: StubbedPointer
+              Member: __impl__ (offset: 16)
+                Primitive: StubbedPointer
+)",
+                R"(
+[0] Class: MyClass_0 [MyClass] (size: 24, align: 8)
+      Member: __oi_padding_0 (offset: 0)
+[3]     Array: [int8_t[24]] (length: 24)
+          Primitive: int8_t
+)");
+}
diff --git a/test/test_drgn_parser.cpp b/test/test_drgn_parser.cpp
index a5ebe30..4a51605 100644
--- a/test/test_drgn_parser.cpp
+++ b/test/test_drgn_parser.cpp
@@ -10,6 +10,7 @@
 #include "oi/type_graph/Printer.h"
 #include "oi/type_graph/TypeGraph.h"
 #include "oi/type_graph/Types.h"
+#include "test/type_graph_utils.h"
 
 using namespace type_graph;
 
@@ -28,7 +29,8 @@ void DrgnParserTest::TearDownTestSuite() {
 
 DrgnParser DrgnParserTest::getDrgnParser(TypeGraph& typeGraph,
                                          DrgnParserOptions options) {
-  DrgnParser drgnParser{typeGraph, options};
+  static auto infos = getContainerInfos();
+  DrgnParser drgnParser{typeGraph, infos, options};
   return drgnParser;
 }
 
@@ -255,8 +257,8 @@ TEST_F(DrgnParserTest, InheritanceMultiple) {
 TEST_F(DrgnParserTest, Container) {
   testMultiCompilerGlob("oid_test_case_std_vector_int_empty",
                         R"(
-[13] Pointer
-[0]    Class: vector<int, std::allocator<int> > (size: 24)
+[14] Pointer
+[13]   Container: std::vector (size: 24)
          Param
            Primitive: int32_t
          Param
@@ -279,12 +281,18 @@ TEST_F(DrgnParserTest, Container) {
              Function: ~allocator
              Function: allocate
              Function: deallocate
-         Parent (offset: 0)
+         Underlying
+[0]        Class: vector<int, std::allocator<int> > (size: 24)
+             Param
+               Primitive: int32_t
+             Param
+               [1]
+             Parent (offset: 0)
 *
 )",
                         R"(
-[9]  Pointer
-[0]    Class: vector<int, std::allocator<int> > (size: 24)
+[10] Pointer
+[9]    Container: std::vector (size: 24)
          Param
            Primitive: int32_t
          Param
@@ -304,7 +312,13 @@ TEST_F(DrgnParserTest, Container) {
              Function: ~allocator
              Function: allocate
              Function: deallocate
-         Parent (offset: 0)
+         Underlying
+[0]        Class: vector<int, std::allocator<int> > (size: 24)
+             Param
+               Primitive: int32_t
+             Param
+               [1]
+             Parent (offset: 0)
 *
 )");
 }
diff --git a/test/test_flattener.cpp b/test/test_flattener.cpp
index eab3cf8..0f73d03 100644
--- a/test/test_flattener.cpp
+++ b/test/test_flattener.cpp
@@ -799,6 +799,27 @@ TEST(FlattenerTest, ParentClassAndContainer) {
 )");
 }
 
+TEST(FlattenerTest, ContainerWithParent) {
+  // This is necessary to correctly calculate container alignment
+  test(Flattener::createPass(),
+       R"(
+[0] Container: std::vector (size: 24)
+      Underlying
+[1]     Class: vector (size: 24)
+          Parent (offset: 0)
+[2]         Class: Parent (size: 4)
+              Member: x (offset: 0)
+                Primitive: int32_t
+)",
+       R"(
+[0] Container: std::vector (size: 24)
+      Underlying
+[1]     Class: vector (size: 24)
+          Member: x (offset: 0)
+            Primitive: int32_t
+)");
+}
+
 TEST(FlattenerTest, AllocatorParamInParent) {
   test(Flattener::createPass(),
        R"(
@@ -857,8 +878,7 @@ TEST(FlattenerTest, AllocatorUnfixableNoParent) {
 )");
 }
 
-TEST(FlattenerTest, AllocatorUnfixableParentNotClass) {
-  // This could be supported if need-be, we just don't do it yet
+TEST(FlattenerTest, AllocatorParamInParentContainer) {
   test(Flattener::createPass(),
        R"(
 [0] Container: std::vector (size: 24)
diff --git a/test/test_identify_containers.cpp b/test/test_identify_containers.cpp
deleted file mode 100644
index 4c359eb..0000000
--- a/test/test_identify_containers.cpp
+++ /dev/null
@@ -1,194 +0,0 @@
-#include <gtest/gtest.h>
-
-#include "oi/ContainerInfo.h"
-#include "oi/type_graph/IdentifyContainers.h"
-#include "oi/type_graph/Types.h"
-#include "test/type_graph_utils.h"
-
-using namespace type_graph;
-
-namespace {
-void test(std::string_view input, std::string_view expectedAfter) {
-  ::test(IdentifyContainers::createPass(getContainerInfos()),
-         input,
-         expectedAfter);
-}
-};  // namespace
-
-TEST(IdentifyContainers, Container) {
-  test(R"(
-[0] Class: std::vector (size: 24)
-      Param
-        Primitive: int32_t
-      Member: a (offset: 0)
-        Primitive: int32_t
-)",
-       R"(
-[1] Container: std::vector (size: 24)
-      Param
-        Primitive: int32_t
-)");
-}
-
-TEST(IdentifyContainers, ContainerInClass) {
-  test(R"(
-[0] Class: MyClass (size: 0)
-      Param
-[1]     Class: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-      Parent (offset: 0)
-[2]     Class: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-      Member: a (offset: 0)
-[3]     Class: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-)",
-       R"(
-[0] Class: MyClass (size: 0)
-      Param
-[4]     Container: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-      Parent (offset: 0)
-[5]     Container: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-      Member: a (offset: 0)
-[6]     Container: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-)");
-}
-
-TEST(IdentifyContainers, ContainerInContainer) {
-  test(R"(
-[0] Class: std::vector (size: 24)
-      Param
-[1]     Class: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-)",
-       R"(
-[2] Container: std::vector (size: 24)
-      Param
-[3]     Container: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-)");
-}
-
-TEST(IdentifyContainers, ContainerInContainer2) {
-  test(R"(
-[0] Container: std::vector (size: 24)
-      Param
-[1]     Class: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-)",
-       R"(
-[0] Container: std::vector (size: 24)
-      Param
-[2]     Container: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-)");
-}
-
-TEST(IdentifyContainers, ContainerInArray) {
-  test(R"(
-[0] Array: (length: 2)
-[1]   Class: std::vector (size: 24)
-        Param
-          Primitive: int32_t
-)",
-       R"(
-[0] Array: (length: 2)
-[2]   Container: std::vector (size: 24)
-        Param
-          Primitive: int32_t
-)");
-}
-
-TEST(IdentifyContainers, ContainerInTypedef) {
-  test(R"(
-[0] Typedef: MyAlias
-[1]   Class: std::vector (size: 24)
-        Param
-          Primitive: int32_t
-)",
-       R"(
-[0] Typedef: MyAlias
-[2]   Container: std::vector (size: 24)
-        Param
-          Primitive: int32_t
-)");
-}
-
-TEST(IdentifyContainers, ContainerInPointer) {
-  test(R"(
-[0] Pointer
-[1]   Class: std::vector (size: 24)
-        Param
-          Primitive: int32_t
-)",
-       R"(
-[0] Pointer
-[2]   Container: std::vector (size: 24)
-        Param
-          Primitive: int32_t
-)");
-}
-
-TEST(IdentifyContainers, ContainerDuplicate) {
-  test(R"(
-[0] Class: std::vector (size: 24)
-      Param
-        Primitive: int32_t
-      Member: a (offset: 0)
-        Primitive: int32_t
-    [0]
-)",
-       R"(
-[1] Container: std::vector (size: 24)
-      Param
-        Primitive: int32_t
-    [1]
-)");
-}
-
-TEST(IdentifyContainers, CycleClass) {
-  test(R"(
-[0] Class: ClassA (size: 0)
-      Member: x (offset: 0)
-[1]     Class: ClassB (size: 0)
-          Param
-            [0]
-)",
-       R"(
-[0] Class: ClassA (size: 0)
-      Member: x (offset: 0)
-[1]     Class: ClassB (size: 0)
-          Param
-            [0]
-)");
-}
-
-TEST(IdentifyContainers, CycleContainer) {
-  test(R"(
-[0] Class: ClassA (size: 0)
-      Member: x (offset: 0)
-[1]     Class: std::vector (size: 0)
-          Param
-            [0]
-)",
-       R"(
-[0] Class: ClassA (size: 0)
-      Member: x (offset: 0)
-[2]     Container: std::vector (size: 0)
-          Param
-            [0]
-)");
-}
diff --git a/test/test_prune.cpp b/test/test_prune.cpp
index 3ae5d61..ad8ceee 100644
--- a/test/test_prune.cpp
+++ b/test/test_prune.cpp
@@ -63,3 +63,33 @@ TEST(PruneTest, RecurseClassChild) {
 [1]     Class: ClassA (size: 12)
 )");
 }
+
+TEST(PruneTest, PruneContainer) {
+  test(Prune::createPass(),
+       R"(
+[0] Container: std::vector (size: 24)
+      Param
+        Primitive: int32_t
+      Param
+        Value: "123"
+        Primitive: int32_t
+      Underlying
+[1]     Class: vector<int32_t> (size: 24)
+          Parent (offset: 0)
+[2]         Class: MyParent (size: 4)
+              Member: a (offset: 0)
+                Primitive: int32_t
+          Member: a (offset: 0)
+            Primitive: int32_t
+          Member: b (offset: 4)
+            Primitive: int32_t
+)",
+       R"(
+[0] Container: std::vector (size: 24)
+      Param
+        Primitive: int32_t
+      Param
+        Value: "123"
+        Primitive: int32_t
+)");
+}
diff --git a/test/test_type_identifier.cpp b/test/test_type_identifier.cpp
index b2b8c8c..ec9efd5 100644
--- a/test/test_type_identifier.cpp
+++ b/test/test_type_identifier.cpp
@@ -108,6 +108,12 @@ TEST(TypeIdentifierTest, PassThroughTypes) {
 [2]     Container: std::allocator (size: 1)
           Param
             Primitive: int32_t
+          Underlying
+[1]         Class: std::allocator (size: 1)
+              Param
+                Primitive: int32_t
+              Function: allocate
+              Function: deallocate
 )");
 }
 
@@ -137,6 +143,12 @@ TEST(TypeIdentifierTest, PassThroughSameType) {
 [2]     Container: std::allocator (size: 1)
           Param
             Primitive: int32_t
+          Underlying
+[1]         Class: std::allocator (size: 1)
+              Param
+                Primitive: int32_t
+              Function: allocate
+              Function: deallocate
       Param
         [2]
 )");
diff --git a/test/type_graph_utils.cpp b/test/type_graph_utils.cpp
index 815a46a..2d5709f 100644
--- a/test/type_graph_utils.cpp
+++ b/test/type_graph_utils.cpp
@@ -86,22 +86,23 @@ std::vector<std::unique_ptr<ContainerInfo>> getContainerInfos() {
 Container getVector(NodeId id) {
   static ContainerInfo info{"std::vector", SEQ_TYPE, "vector"};
   info.stubTemplateParams = {1};
-  return Container{id, info, 24};
+  return Container{id, info, 24, nullptr};
 }
 
 Container getMap(NodeId id) {
   static ContainerInfo info{"std::map", STD_MAP_TYPE, "map"};
   info.stubTemplateParams = {2, 3};
-  return Container{id, info, 48};
+  return Container{id, info, 48, nullptr};
 }
 
 Container getList(NodeId id) {
   static ContainerInfo info{"std::list", LIST_TYPE, "list"};
   info.stubTemplateParams = {1};
-  return Container{id, info, 24};
+  return Container{id, info, 24, nullptr};
 }
 
 Container getPair(NodeId id) {
   static ContainerInfo info{"std::pair", PAIR_TYPE, "utility"};
-  return Container{id, info, 8};  // Nonsense size, shouldn't matter for tests
+  return Container{
+      id, info, 8, nullptr};  // Nonsense size, shouldn't matter for tests
 }

From 72ffe19515b8911d2c78ca758d7bfc75ad46377c Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 12 Dec 2023 11:02:47 -0800
Subject: [PATCH 048/188] incomplete: name type in compiler errors

Summary:

We have a good type representation in the Type Graph of an incomplete type and
the underlying type that represents. However, this incomplete type still ends
up in the generated code as `void` which loses information. For example, a
container that can't contain void may fail to compile because it was
initialised with `void` but really its because the type it was supposed to be
initialised with (say, `Foo`) had incomplete debug information.

This change identifies that a type is incomplete in the output by generating it
as an incomplete type `struct Incomplete<struct Foo>`. This allows us to name
the type correctly in the TreeBuilder output and filter for incomplete types,
as well as getting appropriate compiler errors if it mustn't be incomplete.

Test Plan:
- CI
- Added a unit test to namegen.
- Enabled and added an extra pointers_incomplete test.

This change is tricky to test because it isn't really user visible. The types
still use their `inputName` which is unchanged in any successful output - this
change is used so the compiler fails with a more detailed error.
---
 oi/CodeGen.cpp                            |  9 ++++
 oi/OITraceCode.cpp                        | 15 +++++++
 oi/type_graph/NameGen.cpp                 |  5 +++
 oi/type_graph/NameGen.h                   |  1 +
 oi/type_graph/Printer.cpp                 |  4 +-
 oi/type_graph/TopoSorter.cpp              |  4 ++
 oi/type_graph/TopoSorter.h                |  1 +
 oi/type_graph/Types.cpp                   |  2 -
 oi/type_graph/Types.h                     | 53 +++++++++++++++++------
 test/TypeGraphParser.cpp                  |  5 ++-
 test/integration/pointers_incomplete.toml | 32 ++++++++++++--
 test/test_drgn_parser.cpp                 |  4 +-
 test/test_enforce_compatibility.cpp       |  4 +-
 test/test_flattener.cpp                   |  2 +-
 test/test_name_gen.cpp                    | 13 ++++++
 test/test_remove_members.cpp              |  2 +-
 types/shrd_ptr_type.toml                  |  6 +--
 types/uniq_ptr_type.toml                  |  6 +--
 18 files changed, 133 insertions(+), 35 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 5a4b9eb..3803375 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -55,6 +55,7 @@ using type_graph::DrgnParserOptions;
 using type_graph::EnforceCompatibility;
 using type_graph::Enum;
 using type_graph::Flattener;
+using type_graph::Incomplete;
 using type_graph::KeyCapture;
 using type_graph::Member;
 using type_graph::NameGen;
@@ -176,12 +177,20 @@ void genDeclsEnum(const Enum& e, std::string& code) {
   code += " {};\n";
 }
 
+void genDeclsIncomplete(const Incomplete& i, std::string& code) {
+  code += "template<> struct ";
+  code += i.name();
+  code += ";\n";
+}
+
 void genDecls(const TypeGraph& typeGraph, std::string& code) {
   for (const Type& t : typeGraph.finalTypes) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
       genDeclsClass(*c, code);
     } else if (const auto* e = dynamic_cast<const Enum*>(&t)) {
       genDeclsEnum(*e, code);
+    } else if (const auto* i = dynamic_cast<const Incomplete*>(&t)) {
+      genDeclsIncomplete(*i, code);
     }
   }
 }
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index 31a873b..416d81c 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -182,3 +182,18 @@ bool isStorageInline(const auto& c) {
   return (uintptr_t)std::data(c) < (uintptr_t)(&c + sizeof(c)) &&
          (uintptr_t)std::data(c) >= (uintptr_t)&c;
 }
+
+namespace OIInternal {
+namespace {
+
+template <typename T>
+struct Incomplete;
+
+template <typename T>
+constexpr bool oi_is_complete = true;
+
+template <typename T>
+constexpr bool oi_is_complete<Incomplete<T>> = false;
+
+}  // namespace
+}  // namespace OIInternal
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index c792766..115c06c 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -204,4 +204,9 @@ void NameGen::visit(CaptureKeys& c) {
   c.regenerateName();
 }
 
+void NameGen::visit(Incomplete& i) {
+  RecursiveVisitor::visit(i);
+  i.regenerateName();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..4857ded 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -48,6 +48,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
+  void visit(Incomplete& i) override;
 
   static const inline std::string AnonPrefix = "__oi_anon";
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index 45e3920..5876ff1 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -37,7 +37,9 @@ void Printer::print(const Type& type) {
 }
 
 void Printer::visit(const Incomplete& i) {
-  prefix();
+  if (prefix(i))
+    return;
+
   out_ << "Incomplete";
   if (auto underlyingType = i.underlyingType()) {
     out_ << std::endl;
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index 6ca10af..ff2715a 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -139,6 +139,10 @@ void TopoSorter::visit(CaptureKeys& c) {
   sortedTypes_.push_back(c);
 }
 
+void TopoSorter::visit(Incomplete& i) {
+  sortedTypes_.push_back(i);
+}
+
 /*
  * A type graph may contain cycles, so we need to slightly tweak the standard
  * topological sorting algorithm. Cycles can only be introduced by certain
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 374e943..44f2364 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -48,6 +48,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
+  void visit(Incomplete& i) override;
 
  private:
   std::unordered_set<Type*> visited_;
diff --git a/oi/type_graph/Types.cpp b/oi/type_graph/Types.cpp
index 8be091f..af39606 100644
--- a/oi/type_graph/Types.cpp
+++ b/oi/type_graph/Types.cpp
@@ -32,8 +32,6 @@ namespace oi::detail::type_graph {
 OI_TYPE_LIST
 #undef X
 
-const std::string Incomplete::kName = "void";
-
 std::string Primitive::getName(Kind kind) {
   switch (kind) {
     case Kind::Int8:
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index c16821c..0894166 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -29,6 +29,7 @@
  * debugging.
  */
 
+#include <algorithm>
 #include <cstddef>
 #include <map>
 #include <optional>
@@ -207,29 +208,33 @@ struct TemplateParam {
  */
 class Incomplete : public Type {
  public:
-  Incomplete(Type& underlyingType) : underlyingType_(underlyingType) {
+  Incomplete(NodeId id, Type& underlyingType)
+      : id_(id), underlyingType_(underlyingType) {
   }
 
-  Incomplete(std::string underlyingTypeName)
-      : underlyingType_(std::move(underlyingTypeName)) {
+  Incomplete(NodeId id, std::string underlyingTypeName)
+      : id_(id), underlyingType_(std::move(underlyingTypeName)) {
   }
 
-  static inline constexpr bool has_node_id = false;
+  static inline constexpr bool has_node_id = true;
 
   DECLARE_ACCEPT
 
   const std::string& name() const override {
-    return kName;
+    return name_;
   }
 
   std::string_view inputName() const override {
-    if (std::holds_alternative<std::string>(underlyingType_)) {
-      return std::get<std::string>(underlyingType_);
-    }
-
-    return std::get<std::reference_wrapper<Type>>(underlyingType_)
-        .get()
-        .inputName();
+    return std::visit(
+        [](const auto& el) -> std::string_view {
+          using T = std::decay_t<decltype(el)>;
+          if constexpr (std::is_same_v<T, std::string>) {
+            return el;
+          } else {
+            return el.get().inputName();
+          }
+        },
+        underlyingType_);
   }
 
   size_t size() const override {
@@ -241,7 +246,26 @@ class Incomplete : public Type {
   }
 
   NodeId id() const override {
-    return -1;
+    return id_;
+  }
+
+  void regenerateName() {
+    std::string_view inputName = this->inputName();
+
+    constexpr std::string_view kPrefix{"Incomplete<struct "};
+    name_.clear();
+    name_.reserve(kPrefix.size() + inputName.size() + 2);
+
+    name_ = kPrefix;
+    for (const auto c : inputName) {
+      if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') ||
+          (c >= 'A' && c <= 'Z')) {
+        name_ += c;
+      } else {
+        name_ += '_';
+      }
+    }
+    name_ += ">";
   }
 
   std::optional<std::reference_wrapper<Type>> underlyingType() const {
@@ -253,8 +277,9 @@ class Incomplete : public Type {
   }
 
  private:
+  NodeId id_ = -1;
   std::variant<std::string, std::reference_wrapper<Type>> underlyingType_;
-  static const std::string kName;
+  std::string name_;
 };
 
 /*
diff --git a/test/TypeGraphParser.cpp b/test/TypeGraphParser.cpp
index f54fe08..d70236c 100644
--- a/test/TypeGraphParser.cpp
+++ b/test/TypeGraphParser.cpp
@@ -215,10 +215,11 @@ Type& TypeGraphParser::parseType(std::string_view& input, size_t rootIndent) {
       auto nameEndPos = line.find(']', nameStartPos);
       auto underlyingTypeName =
           line.substr(nameStartPos, nameEndPos - nameStartPos);
-      type = &typeGraph_.makeType<Incomplete>(std::string(underlyingTypeName));
+      type =
+          &typeGraph_.makeType<Incomplete>(id, std::string(underlyingTypeName));
     } else {
       auto& underlyingType = parseType(input, indent + 2);
-      type = &typeGraph_.makeType<Incomplete>(underlyingType);
+      type = &typeGraph_.makeType<Incomplete>(id, underlyingType);
     }
   } else if (nodeTypeName == "Class" || nodeTypeName == "Struct" ||
              nodeTypeName == "Union") {
diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml
index 437041c..aeebb24 100644
--- a/test/integration/pointers_incomplete.toml
+++ b/test/integration/pointers_incomplete.toml
@@ -55,7 +55,6 @@ definitions = '''
     }]'''
 
   [cases.unique_ptr]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299
     param_types = ["const std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>&"]
     setup = '''
       auto raw_ptr = static_cast<IncompleteType*>(::operator new(5));
@@ -63,28 +62,29 @@ definitions = '''
         raw_ptr, &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
   [cases.unique_ptr_null]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299
     param_types = ["const std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>&"]
     setup = '''
       return std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>(
         nullptr, &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.shared_ptr]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300
     param_types = ["const std::shared_ptr<IncompleteType>&"]
     setup = '''
       auto raw_ptr = static_cast<IncompleteType*>(::operator new(5));
       return std::shared_ptr<IncompleteType>(raw_ptr , &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
   [cases.shared_ptr_null]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300
     param_types = ["const std::shared_ptr<IncompleteType>"]
     setup = "return nullptr;"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.containing_struct]
     oil_disable = "oil can't chase raw pointers safely"
@@ -117,3 +117,27 @@ definitions = '''
         }
       ]
     }]'''
+
+  [cases.containing_struct_no_follow]
+    param_types = ["const IncompleteTypeContainer&"]
+    setup = "return IncompleteTypeContainer{};"
+    expect_json = '''[{
+      "staticSize": 88,
+      "members": [
+        { "name": "ptrundef", "staticSize": 8 },
+        { "name": "__makePad1", "staticSize": 1 },
+        { "name": "shundef", "staticSize": 16 },
+        { "name": "__makePad2", "staticSize": 1 },
+        { "name": "shoptundef",
+          "staticSize": 24,
+          "length": 0,
+          "capacity": 1
+        },
+        { "name": "__makePad3", "staticSize": 1 },
+        { "name": "optundef",
+          "staticSize": 16,
+          "length": 0,
+          "capacity": 1
+        }
+      ]
+    }]'''
diff --git a/test/test_drgn_parser.cpp b/test/test_drgn_parser.cpp
index 4a51605..6cb4d31 100644
--- a/test/test_drgn_parser.cpp
+++ b/test/test_drgn_parser.cpp
@@ -440,8 +440,8 @@ TEST_F(DrgnParserTest, PointerNoFollow) {
 
 TEST_F(DrgnParserTest, PointerIncomplete) {
   test("oid_test_case_pointers_incomplete_raw", R"(
-[0] Pointer
-      Incomplete: [IncompleteType]
+[1] Pointer
+[0]   Incomplete: [IncompleteType]
 )");
 }
 
diff --git a/test/test_enforce_compatibility.cpp b/test/test_enforce_compatibility.cpp
index 2632823..d91f356 100644
--- a/test/test_enforce_compatibility.cpp
+++ b/test/test_enforce_compatibility.cpp
@@ -38,8 +38,8 @@ TEST(EnforceCompatibilityTest, VoidPointer) {
        R"(
 [0] Class: MyClass (size: 8)
       Member: p (offset: 0)
-[1]     Pointer
-          Incomplete
+[2]     Pointer
+[1]       Incomplete
             Primitive: void
 )",
        R"(
diff --git a/test/test_flattener.cpp b/test/test_flattener.cpp
index 0f73d03..0b6660d 100644
--- a/test/test_flattener.cpp
+++ b/test/test_flattener.cpp
@@ -1006,7 +1006,7 @@ TEST(FlattenerTest, IncompleteParent) {
        R"(
 [0] Class: MyClass (size: 4)
       Parent (offset: 0)
-        Incomplete: [IncompleteParent]
+[1]     Incomplete: [IncompleteParent]
 )",
        R"(
 [0] Class: MyClass (size: 4)
diff --git a/test/test_name_gen.cpp b/test/test_name_gen.cpp
index 72177d7..d721f7d 100644
--- a/test/test_name_gen.cpp
+++ b/test/test_name_gen.cpp
@@ -475,3 +475,16 @@ TEST(NameGenTest, AnonymousTypes) {
   EXPECT_EQ(myenum.inputName(), "__oi_anon_1");
   EXPECT_EQ(mytypedef.inputName(), "");
 }
+
+TEST(NameGenTest, IncompleteTypes) {
+  auto myincompletevector = Incomplete{0, "std::vector<int>"};
+
+  auto myint = Primitive{Primitive::Kind::Int32};
+  auto myincompleteint = Incomplete{1, myint};
+
+  NameGen nameGen;
+  nameGen.generateNames({myincompletevector, myincompleteint});
+
+  EXPECT_EQ(myincompletevector.name(), "Incomplete<struct std__vector_int_>");
+  EXPECT_EQ(myincompleteint.name(), "Incomplete<struct int32_t>");
+}
diff --git a/test/test_remove_members.cpp b/test/test_remove_members.cpp
index 0d4a54e..9eff4ed 100644
--- a/test/test_remove_members.cpp
+++ b/test/test_remove_members.cpp
@@ -191,7 +191,7 @@ TEST(RemoveMembersTest, Incomplete) {
       Member: a (offset: 0)
         Primitive: int32_t
       Member: b (offset: 4)
-        Incomplete: [MyIncompleteType]
+[1]     Incomplete: [MyIncompleteType]
       Member: c (offset: 8)
         Primitive: int32_t
 )",
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index bb3a622..de84b71 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (!std::is_void<T>::value && oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
         if (s_ptr && ctx.pointers.add((uintptr_t)(s_ptr.get()))) {
@@ -37,7 +37,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (std::is_void<T0>::value || !oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
@@ -85,7 +85,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index f723f80..09e351e 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T,Deleter>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (!std::is_void<T>::value && oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
         if (u_ptr && ctx.pointers.add((uintptr_t)(u_ptr.get()))) {
@@ -38,7 +38,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (std::is_void<T0>::value || !oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
@@ -71,7 +71,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);

From 6be7e7708bfd69c75faa46d42df81a79734eab46 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 12 Dec 2023 11:01:11 -0800
Subject: [PATCH 049/188] oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD
---
 CMakeLists.txt                                |  13 +-
 oi/CodeGen.cpp                                |  30 +-
 oi/CodeGen.h                                  |   8 +-
 oi/Config.cpp                                 |   9 +-
 oi/ContainerInfo.cpp                          |   2 +-
 oi/OICodeGen.h                                |   2 +-
 oi/OIGenerator.cpp                            | 362 +++++++++-------
 oi/OIGenerator.h                              |  29 +-
 oi/type_graph/CMakeLists.txt                  |   2 +
 oi/type_graph/ClangTypeParser.cpp             | 393 ++++++++++++++++++
 oi/type_graph/ClangTypeParser.h               | 123 ++++++
 oi/type_graph/NameGen.cpp                     |   8 +
 oi/type_graph/NameGen.h                       |   1 +
 oi/type_graph/Printer.cpp                     |  11 +
 oi/type_graph/Printer.h                       |   1 +
 oi/type_graph/RemoveTopLevelPointer.cpp       |   4 +
 oi/type_graph/RemoveTopLevelPointer.h         |   1 +
 oi/type_graph/Types.h                         |  56 +++
 oi/type_graph/Visitor.h                       |   7 +
 test/integration/template_template_param.toml |  16 +
 tools/OILGen.cpp                              | 147 +++----
 types/string_type.toml                        |  31 ++
 22 files changed, 966 insertions(+), 290 deletions(-)
 create mode 100644 oi/type_graph/ClangTypeParser.cpp
 create mode 100644 oi/type_graph/ClangTypeParser.h
 create mode 100644 test/integration/template_template_param.toml

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f9ee7..eb18f4a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -104,6 +104,15 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(range-v3)
 
+### cxxopts
+FetchContent_Declare(
+  cxxopts
+  GIT_REPOSITORY https://github.com/jarro2783/cxxopts.git
+  GIT_TAG        eb787304d67ec22f7c3a184ee8b4c481d04357fd # v3.1.1
+  GIT_PROGRESS   TRUE
+)
+FetchContent_MakeAvailable(cxxopts)
+
 set_project_warnings()
 
 if (ASAN)
@@ -289,7 +298,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -363,8 +372,10 @@ add_executable(oilgen
   oi/OIGenerator.cpp
 )
 target_link_libraries(oilgen
+  cxxopts
   drgn_utils
   oicore
+  clangTooling
 )
 
 ### Object Introspection cache Printer (OIP)
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 5a4b9eb..e840104 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -537,7 +537,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1104,15 +1104,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
 }
 
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
-  try {
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
+  if (!registerContainers())
     return false;
-  }
 
   TypeGraph typeGraph;
   try {
@@ -1127,6 +1120,20 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
   return true;
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    size_t i = 0;
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1167,7 +1174,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, containerInfos_, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(Flattener::createPass());
@@ -1282,7 +1289,8 @@ void CodeGen::generate(
   code += "\nusing __ROOT_TYPE__ = " + rootType.name() + ";\n";
   code += "} // namespace\n} // namespace OIInternal\n";
 
-  const auto typeName = SymbolService::getTypeName(drgnType);
+  // const auto typeName = SymbolService::getTypeName(drgnType);
+  const std::string typeName{"typeName"};
   if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
   } else {
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 186ec34..fb0ec20 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -40,8 +40,11 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config) : config_(config) {
+    assert(!config.features[Feature::PolymorphicInheritance]);
+  }
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   /*
@@ -53,6 +56,7 @@ class CodeGen {
                        std::string linkageName,
                        std::string& code);
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -66,7 +70,7 @@ class CodeGen {
 
  private:
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
diff --git a/oi/Config.cpp b/oi/Config.cpp
index 157b279..66798bb 100644
--- a/oi/Config.cpp
+++ b/oi/Config.cpp
@@ -22,6 +22,8 @@
 #include <boost/property_tree/ini_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 #include <filesystem>
+#include <range/v3/action/sort.hpp>
+#include <range/v3/action/unique.hpp>
 
 #include "oi/support/Toml.h"
 
@@ -54,6 +56,9 @@ std::optional<FeatureSet> processConfigFiles(
     enables |= *fs;
   }
 
+  ranges::actions::sort(generatorConfig.containerConfigPaths);
+  ranges::actions::unique(generatorConfig.containerConfigPaths);
+
   // Override anything from the config files with command line options
   for (auto [k, v] : featureMap) {
     enables[k] = v;
@@ -108,8 +113,8 @@ std::optional<FeatureSet> processConfigFile(
           if the right path is absolute, else append the right path to the
           left path.
           */
-          generatorConfig.containerConfigPaths.emplace(configDirectory /
-                                                       el.get());
+          generatorConfig.containerConfigPaths.emplace_back(configDirectory /
+                                                            el.get());
         }
       });
     }
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 99a7dc1..dbe10c6 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -202,7 +202,7 @@ std::regex getMatcher(const std::string& typeName) {
 ContainerInfo::ContainerInfo(const fs::path& path) {
   toml::table container;
   try {
-    container = toml::parse_file(std::string(path));
+    container = toml::parse_file(path.string());
   } catch (const toml::parse_error& err) {
     // Convert into a ContainerInfoError, just to avoid having to include
     // the huge TOML++ header in the caller's file. Use toml::parse_error's
diff --git a/oi/OICodeGen.h b/oi/OICodeGen.h
index a01548f..7226a4e 100644
--- a/oi/OICodeGen.h
+++ b/oi/OICodeGen.h
@@ -66,7 +66,7 @@ class OICodeGen {
     };
 
     FeatureSet features;
-    std::set<std::filesystem::path> containerConfigPaths;
+    std::vector<std::filesystem::path> containerConfigPaths;
     std::set<std::string> defaultHeaders;
     std::set<std::string> defaultNamespaces;
     std::vector<std::pair<std::string, std::string>> membersToStub;
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 1cb2a49..7d749c3 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -16,130 +16,148 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
 #include <glog/logging.h>
 
 #include <boost/core/demangle.hpp>
 #include <fstream>
 #include <iostream>
+#include <range/v3/action/transform.hpp>
+#include <range/v3/core.hpp>
+#include <range/v3/view/cache1.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/drop_while.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>  // TODO: remove
 #include <string_view>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
+class ConsumerContext;
 
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
-      templateArgsToSymbolsMap;
-
-  auto symbols = prog.find_all_symbols();
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
+class CreateTypeGraphConsumer;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
-    if (val.first.empty() || val.second.empty()) {
-      continue;
-    }
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  return strongToWeakSymbols;
-}
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
 
-std::unordered_map<std::string, drgn_qualified_type>
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
+ private:
+  ConsumerContext& ctx;
+};
 
-  std::unordered_map<std::string, drgn_qualified_type> out;
+class ConsumerContext {
+ public:
+  ConsumerContext(const std::vector<std::unique_ptr<ContainerInfo>>& cis) : containerInfos{cis} {}
 
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
-    std::string strongLinkageName;
-    {
-      const char* linkageNameCstr;
-      if (auto err = drgnplusplus::error(
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
-        // throw err;
+  type_graph::TypeGraph typeGraph;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
+  std::optional<bool> pic;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+}  // namespace
+
+int OIGenerator::generate(clang::tooling::CompilationDatabase& db,
+                          const std::vector<std::string>& sourcePaths) {
+  std::map<Feature, bool> featuresMap = {
+      {Feature::TypeGraph, true},
+      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},
+      {Feature::PackStructs, true},
+      {Feature::PruneTypeGraph, true},
+  };
+
+  OICodeGen::Config generatorConfig{};
+  OICompiler::Config compilerConfig{};
+
+  auto features = config::processConfigFiles(
+      configFilePaths, featuresMap, compilerConfig, generatorConfig);
+  if (!features) {
+    LOG(ERROR) << "failed to process config file";
+    return -1;
+  }
+  generatorConfig.features = *features;
+  compilerConfig.features = *features;
+
+  std::vector<std::unique_ptr<ContainerInfo>> containerInfos;
+  containerInfos.reserve(generatorConfig.containerConfigPaths.size());
+  try {
+    for (const auto& path : generatorConfig.containerConfigPaths) {
+      auto info = std::make_unique<ContainerInfo>(path);
+      if (info->requiredFeatures != (*features & info->requiredFeatures)) {
+        VLOG(1) << "Skipping container (feature conflict): " << info->typeName;
         continue;
       }
-      strongLinkageName = linkageNameCstr;
+      containerInfos.emplace_back(std::move(info));
     }
-
-    std::string weakLinkageName;
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return -1;
   }
 
-  return out;
-}
+  ConsumerContext ctx{containerInfos};
+  CreateTypeGraphActionFactory factory{ctx};
 
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
-                                      const OICompiler::Config& compilerConfig,
-                                      const drgn_qualified_type& type,
-                                      const std::string& linkageName,
-                                      SymbolService& symbols) {
-  CodeGen codegen{generatorConfig, symbols};
+  clang::tooling::ClangTool tool{db, sourcePaths};
+  if (auto ret = tool.run(&factory); ret != 0) {
+    return ret;
+  }
+
+  if (ctx.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one type to generate for but we can't currently "
+        "handle this case");
+
+  compilerConfig.usePIC = ctx.pic.value();
+  CodeGen codegen{generatorConfig};
+  for (auto&& ptr : containerInfos)
+    codegen.registerContainer(std::move(ptr));
+  codegen.transform(ctx.typeGraph);
 
   std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
+  codegen.generate(ctx.typeGraph, code, nullptr);
 
   std::string sourcePath = sourceFileDumpPath;
   if (sourceFileDumpPath.empty()) {
@@ -152,78 +170,110 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
   }
 
   OICompiler compiler{{}, compilerConfig};
-
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
-
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
 }
 
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
-  drgnplusplus::program prog;
+namespace {
 
-  {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(
-            drgn_program_load_debug_info(prog.get(),
-                                         std::data(objectPaths),
-                                         std::size(objectPaths),
-                                         false,
-                                         false))) {
-      LOG(ERROR) << "error loading debug info program: " << err;
-      throw err;
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx(ctx_) {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
     }
-  }
 
-  auto oilTypes = findOilTypesAndNames(prog);
-
-  std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
-      {Feature::PruneTypeGraph, true},
-  };
-
-  OICodeGen::Config generatorConfig{};
-  OICompiler::Config compilerConfig{};
-  compilerConfig.usePIC = pic;
-
-  auto features = config::processConfigFiles(
-      configFilePaths, featuresMap, compilerConfig, generatorConfig);
-  if (!features) {
-    LOG(ERROR) << "failed to process config file";
-    return -1;
-  }
-  generatorConfig.features = *features;
-  compilerConfig.features = *features;
-
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
+    auto introspectImpl =
+        std::move(oi_namespaces) |
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
+        ranges::views::transform([](auto* p) {
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
+        }) |
+        ranges::views::filter([](auto* td) {
+          return td != nullptr && td->getName() == "introspectImpl";
+        }) |
+        ranges::views::take(1) | ranges::to<std::vector>();
+    if (introspectImpl.empty()) {
+      LOG(WARNING)
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
+             "you compile with `OIL_AOT_COMPILATION=1`?";
+      return;
     }
-  }
 
-  size_t successes = oilTypes.size() - failures;
-  LOG(INFO) << "object introspection generation complete. " << successes
-            << " successes and " << failures << " failures.";
+    auto nameToClangTypeMap =
+        ranges::views::single(introspectImpl[0]) |
+        ranges::views::for_each(
+            [](auto* td) { return td->specializations(); }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
 
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
-    return -1;
+              assert(fd->getNumParams() == 1);
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, opts};
+
+    auto& Sema = *ctx.sema;
+    ctx.nameToTypeMap =
+        nameToClangTypeMap |
+        ranges::views::transform(
+            [&parser, &Context, &Sema](
+                auto& p) -> std::pair<std::string, type_graph::Type*> {
+              return {p.first, &parser.parse(Context, Sema, *p.second)};
+            }) |
+        ranges::to<std::unordered_map>();
+
+    for (const auto& [name, type] : ctx.nameToTypeMap)
+      ctx.typeGraph.addRoot(*type);
   }
-  return 0;
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  // Compile the output as position independent if any input is position
+  // independent
+  bool pic = CI.getCodeGenOpts().RelocationModel == llvm::Reloc::PIC_;
+  ctx.pic = ctx.pic.value_or(false) || pic;
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
 }
 
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+}  // namespace
 }  // namespace oi::detail
diff --git a/oi/OIGenerator.h b/oi/OIGenerator.h
index dd38384..fac947a 100644
--- a/oi/OIGenerator.h
+++ b/oi/OIGenerator.h
@@ -23,11 +23,19 @@
 #include "oi/OICodeGen.h"
 #include "oi/OICompiler.h"
 
+namespace clang::tooling {
+class CompilationDatabase;
+}
+
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate(clang::tooling::CompilationDatabase&,
+               const std::vector<std::string>&);
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -41,8 +49,8 @@ class OIGenerator {
   void setFailIfNothingGenerated(bool fail) {
     failIfNothingGenerated = fail;
   }
-  void setUsePIC(bool pic_) {
-    pic = pic_;
+  void setClangArgs(std::vector<std::string> args_) {
+    clangArgs = std::move(args_);
   }
 
  private:
@@ -50,20 +58,7 @@ class OIGenerator {
   std::vector<std::filesystem::path> configFilePaths;
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
-  bool pic = false;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
+  std::vector<std::string> clangArgs;
 };
 
 }  // namespace oi::detail
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 4e8f622..54aba34 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
   Flattener.cpp
@@ -25,3 +26,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
new file mode 100644
index 0000000..61149a6
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -0,0 +1,393 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Basic/DiagnosticSema.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <iostream>  // TODO: maybe remove
+#include <stdexcept>  // TODO: remove
+#include <regex>
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (!requireCompleteType(*sema, ty))
+    return makeType<Incomplete>(ty, "incomplete (TODO naming)");
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(*ty.getAs<const clang::RecordType>());
+    case clang::Type::LValueReference:
+      return enumerateReference(*ty.getAs<const clang::LValueReferenceType>());
+    case clang::Type::Pointer:
+      return enumeratePointer(*ty.getAs<const clang::PointerType>());
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          *ty.getAs<const clang::SubstTemplateTypeParmType>());
+    case clang::Type::Builtin:
+      return enumeratePrimitive(*ty.getAs<const clang::BuiltinType>());
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(*ty.getAs<const clang::ElaboratedType>());
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          *ty.getAs<const clang::TemplateSpecializationType>());
+
+    case clang::Type::Typedef:
+      return enumerateTypedef(*ty.getAs<const clang::TypedefType>());
+    case clang::Type::Using:
+      return enumerateUsing(*ty.getAs<const clang::UsingType>());
+
+    case clang::Type::ConstantArray:
+      return enumerateArray(*llvm::cast<const clang::ConstantArrayType>(&ty));
+    case clang::Type::Enum:
+      return enumerateEnum(*ty.getAs<const clang::EnumType>());
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Typedef& ClangTypeParser::enumerateUsing(const clang::UsingType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+  std::string name = ty.getFoundDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+
+  std::string name = ty.getDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string name = ty.getDecl()->getNameAsString();
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  std::map<int64_t, std::string> enumeratorMap;
+  if (options_.readEnumValues) {
+    for (const auto* enumerator : ty.getDecl()->enumerators()) {
+      enumeratorMap.emplace(
+          enumerator->getInitVal().getExtValue(),
+          enumerator->getNameAsString()
+      );
+    }
+  }
+
+  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+}
+
+Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
+  uint64_t len = ty.getSize().getLimitedValue();
+  auto& t = enumerateType(*ty.getElementType());
+  return makeType<Array>(ty, t, len);
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  if (auto* info = getContainerInfo(fqName)) {
+    auto& c = makeType<Container>(ty, *info, size, nullptr);
+    enumerateClassTemplateParams(ty, c.templateParams);
+    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)));
+    return c;
+  }
+
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+
+  params.reserve(list.size());
+  for (const auto& arg : list.asArray()) {
+    params.emplace_back(enumerateTemplateParam(arg));
+  }
+}
+
+TemplateParam ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return {ttype, qualifiers};
+    }
+    case clang::TemplateArgument::Integral: {
+      auto& ty = enumerateType(*p.getIntegralType());
+      llvm::SmallString<32> val;
+      p.getAsIntegral().toString(val);
+      return {ty, std::string(val)};
+    }
+    case clang::TemplateArgument::Template: {
+      return enumerateTemplateTemplateParam(p.getAsTemplate());
+    }
+
+#define X(name) \
+case clang::TemplateArgument::name: \
+  throw std::logic_error("unsupported template argument kind: " #name);
+
+  X(Null)
+  X(Declaration)
+  X(NullPtr)
+  X(TemplateExpansion)
+  X(Expression)
+  X(Pack)
+#undef X
+  }
+}
+
+TemplateParam ClangTypeParser::enumerateTemplateTemplateParam(const clang::TemplateName& tn) {
+  switch (tn.getKind()) {
+
+#define X(name) \
+case clang::TemplateName::name: \
+  throw std::logic_error("unsupported template name kind: " #name);
+
+X(Template)
+X(OverloadedTemplate)
+X(AssumedTemplate)
+X(QualifiedTemplate)
+X(DependentTemplate)
+X(SubstTemplateTemplateParm)
+X(SubstTemplateTemplateParmPack)
+X(UsingTemplate)
+#undef X
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function references
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Float:
+      return makeType<Primitive>(ty, Primitive::Kind::Float32);
+    case clang::BuiltinType::Double:
+    case clang::BuiltinType::LongDouble:
+      return makeType<Primitive>(ty, Primitive::Kind::Float64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      ty.dump();
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+ContainerInfo* ClangTypeParser::getContainerInfo(const std::string& fqName) const {
+  for (const auto& containerInfo : containers_) {
+    if (std::regex_search(fqName, containerInfo->matcher)) {
+      return containerInfo.get();
+    }
+  }
+  return nullptr;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
+    return true; // treat as complete
+
+  return !sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      clang::diag::err_type_unsupported);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
new file mode 100644
index 0000000..b6d81da
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.h
@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ASTContext;
+class BuiltinType;
+class ConstantArrayType;
+class ElaboratedType;
+class EnumType;
+class LValueReferenceType;
+class PointerType;
+class RecordType;
+class Sema;
+class SubstTemplateTypeParmType;
+class TemplateArgument;
+class TemplateName;
+class TemplateSpecializationType;
+class Type;
+class TypedefType;
+class UsingType;
+}  // namespace clang
+
+struct ContainerInfo;
+
+namespace oi::detail::type_graph {
+
+class Array;
+class Class;
+class Enum;
+class Member;
+class Primitive;
+class Reference;
+class Type;
+class TypeGraph;
+class Typedef;
+struct TemplateParam;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+  bool readEnumValues = false;
+};
+
+/*
+ * ClangTypeParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given drgn_type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph, const std::vector<std::unique_ptr<ContainerInfo>>& containers, ClangTypeParserOptions options)
+      : typeGraph_{typeGraph}, containers_{containers}, options_{options} {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containers_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Type& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+  Typedef& enumerateTypedef(const clang::TypedefType&);
+  Typedef& enumerateUsing(const clang::UsingType&);
+
+  Array& enumerateArray(const clang::ConstantArrayType&);
+  Enum& enumerateEnum(const clang::EnumType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  TemplateParam enumerateTemplateParam(const clang::TemplateArgument&);
+  TemplateParam enumerateTemplateTemplateParam(const clang::TemplateName&);
+
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  ContainerInfo* getContainerInfo(const std::string& fqName) const;
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index c792766..5213b3d 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -194,6 +194,14 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& p) {
+  RecursiveVisitor::visit(p);
+  p.regenerateName();
+  std::string inputName{p.pointeeType().inputName()};
+  inputName += '*';
+  p.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..e6437d0 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index 45e3920..f07ca48 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -150,6 +150,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& p) {
+  if (prefix(p))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = p.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(p.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index 9970b3d..e2e5068 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& p) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;
diff --git a/oi/type_graph/RemoveTopLevelPointer.cpp b/oi/type_graph/RemoveTopLevelPointer.cpp
index 9ecaaaa..e1909a7 100644
--- a/oi/type_graph/RemoveTopLevelPointer.cpp
+++ b/oi/type_graph/RemoveTopLevelPointer.cpp
@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& p) {
+  topLevelType_ = &p.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/RemoveTopLevelPointer.h b/oi/type_graph/RemoveTopLevelPointer.h
index 611429f..6b70545 100644
--- a/oi/type_graph/RemoveTopLevelPointer.h
+++ b/oi/type_graph/RemoveTopLevelPointer.h
@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
 
  private:
   Type* topLevelType_ = nullptr;
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index c16821c..21c03c9 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -733,6 +734,61 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    name_ = pointeeType_.get().name() + "&";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index 9e05821..7a8f12a 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -122,6 +122,9 @@ class RecursiveVisitor : public Visitor {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -189,6 +192,10 @@ class RecursiveMutator : public Mutator {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }
diff --git a/test/integration/template_template_param.toml b/test/integration/template_template_param.toml
new file mode 100644
index 0000000..c64590d
--- /dev/null
+++ b/test/integration/template_template_param.toml
@@ -0,0 +1,16 @@
+includes = ["vector", "utility", "string"]
+
+definitions = '''
+template <typename K, typename V, template <typename, typename> typename Pair>
+struct bad_map {
+  std::vector<Pair<K, K>> keys;
+  std::vector<Pair<V, V>> values;
+};
+'''
+
+[cases]
+  [cases.int_int_empty]
+    param_types = ["const bad_map<int, std::string, std::pair>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index d1e345d..ec3810b 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -14,131 +14,80 @@
  * limitations under the License.
  */
 
+#include <clang/Tooling/CommonOptionsParser.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include <cstdlib>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/action/drop_while.hpp>
+#include <range/v3/algorithm/find.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/all.hpp>
+#include <range/v3/view/transform.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
 
-constexpr static OIOpts opts{
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
+static llvm::cl::OptionCategory OilgenCategory("oilgen options");
 
-void usage() {
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
-  std::cerr << opts;
+static llvm::cl::list<std::string> ConfigFiles("config-file",
+                                               llvm::cl::desc(R"(TODO HELP)"),
+                                               llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<int> DebugLevel("debug-level",
+                                     llvm::cl::desc(R"(TODO HELP)"),
+                                     llvm::cl::init(-1),
+                                     llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> DumpJit("dump-jit",
+                                          llvm::cl::desc(R"(TODO HELP)"),
+                                          llvm::cl::init("jit.cpp"),
+                                          llvm::cl::cat(OilgenCategory));
 
-  std::cerr << std::endl
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
-
-int main(int argc, char* argv[]) {
+int main(int argc, const char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
-  std::vector<fs::path> configFilePaths;
-  fs::path sourceFileDumpPath = "";
-  bool exitCode = false;
-  bool pic = false;
-
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
+  auto expectedParser =
+      clang::tooling::CommonOptionsParser::create(argc, argv, OilgenCategory);
+  if (!expectedParser) {
+    llvm::errs() << expectedParser.takeError();
+    return -1;
   }
+  clang::tooling::CommonOptionsParser& options = expectedParser.get();
+  auto& compilations = options.getCompilations();
 
-  if (optind >= argc) {
-    usage();
-    return EXIT_FAILURE;
-  }
-  fs::path primaryObject = argv[optind];
-
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
+  if (DebugLevel.getNumOccurrences()) {
+    google::LogToStderr();
+    google::SetStderrLogging(google::INFO);
+    google::SetVLOGLevel("*", DebugLevel);
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
+    gflags::SetCommandLineOption("minloglevel", "0");
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
-  oigen.setConfigFilePaths(std::move(configFilePaths));
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
-  oigen.setFailIfNothingGenerated(exitCode);
-  oigen.setUsePIC(pic);
+  oigen.setConfigFilePaths(ConfigFiles |
+                           ranges::views::transform([](const auto& p) {
+                             return std::filesystem::path(p);
+                           }) |
+                           ranges::to<std::vector>());
+  if (DumpJit.getNumOccurrences())
+    oigen.setSourceFileDumpPath(DumpJit.getValue());
 
-  SymbolService symbols(primaryObject);
+  oigen.setOutputPath("a.o");  // TODO: make this an opt
 
-  return oigen.generate(primaryObject, symbols);
+  // TODO: can we extract this from the original arguments?
+  // oigen.setUsePIC(args["pic"].as<bool>());
+
+  oigen.setFailIfNothingGenerated(true);
+  // compilations.getAllFiles();
+  // return oigen.generate(compilations, {"/data/users/jakehillion/fbsource/fbcode/object-introspection/oil/examples/compile-time/VectorOfStrings-Introspect.cpp"});
+  return oigen.generate(compilations, options.getSourcePathList());
 }
diff --git a/types/string_type.toml b/types/string_type.toml
index fa4a2ae..0d2ab0d 100644
--- a/types/string_type.toml
+++ b/types/string_type.toml
@@ -35,3 +35,34 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
     );
 }
 """
+
+traversal_func = """
+    bool sso = ((uintptr_t)container.data() <
+                (uintptr_t)(&container + sizeof(std::__cxx11::basic_string<T0>))) &&
+               ((uintptr_t)container.data() >= (uintptr_t)&container);
+
+    return returnArg.write(container.capacity())
+        .write(sso)
+        .write(container.size());
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+uint64_t capacity = std::get<ParsedData::VarInt>(d.val).value;
+el.container_stats.emplace(result::Element::ContainerStats { .capacity = capacity });
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+bool sso = std::get<ParsedData::VarInt>(d.val).value;
+if (!sso)
+  el.exclusive_size += el.container_stats->capacity * sizeof(T0);
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats->length = std::get<ParsedData::VarInt>(d.val).value;
+"""

From 11489e2f7a593bff4c3e8c50256f24263df8c8cc Mon Sep 17 00:00:00 2001
From: Alastair Robertson <ajor@meta.com>
Date: Wed, 13 Dec 2023 16:06:59 +0000
Subject: [PATCH 050/188] TypeGraph: Fix handling for classes which inherit
 from containers

We previously moved container identification later in CodeGen in order
to preserve information for AlignmentCalc.

However, Flattener needs to know if a class is a container in order to
apply its special handling for this case.

This new approach moves container identification back into DrgnParser,
but has Container own a type node, representing its layout. This
underlying type node can be used for calculating a container's
alignment.
---
 oi/CodeGen.cpp                       |   8 +-
 oi/type_graph/AlignmentCalc.cpp      |  18 +--
 oi/type_graph/AlignmentCalc.h        |   1 +
 oi/type_graph/CMakeLists.txt         |   1 -
 oi/type_graph/DrgnParser.cpp         |  23 +++-
 oi/type_graph/DrgnParser.h           |  12 +-
 oi/type_graph/Flattener.cpp          |  10 --
 oi/type_graph/Flattener.h            |   1 -
 oi/type_graph/IdentifyContainers.cpp |  73 ----------
 oi/type_graph/IdentifyContainers.h   |  58 --------
 oi/type_graph/Printer.cpp            |  13 +-
 oi/type_graph/Printer.h              |   2 +-
 oi/type_graph/Prune.cpp              |  19 +--
 oi/type_graph/Prune.h                |   1 +
 oi/type_graph/TypeIdentifier.cpp     |   3 +-
 oi/type_graph/Types.h                |  54 +++++---
 oi/type_graph/Visitor.h              |   7 +
 test/CMakeLists.txt                  |   1 -
 test/TypeGraphParser.cpp             |  26 +++-
 test/TypeGraphParser.h               |   3 +
 test/test_add_children.cpp           |  22 +--
 test/test_alignment_calc.cpp         |  38 +++++-
 test/test_codegen.cpp                | 100 ++++++++++++--
 test/test_drgn_parser.cpp            |  28 +++-
 test/test_flattener.cpp              |  24 +++-
 test/test_identify_containers.cpp    | 194 ---------------------------
 test/test_prune.cpp                  |  30 +++++
 test/test_type_identifier.cpp        |  12 ++
 test/type_graph_utils.cpp            |   9 +-
 29 files changed, 356 insertions(+), 435 deletions(-)
 delete mode 100644 oi/type_graph/IdentifyContainers.cpp
 delete mode 100644 oi/type_graph/IdentifyContainers.h
 delete mode 100644 test/test_identify_containers.cpp

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index bef102f..5a4b9eb 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -32,7 +32,6 @@
 #include "type_graph/DrgnParser.h"
 #include "type_graph/EnforceCompatibility.h"
 #include "type_graph/Flattener.h"
-#include "type_graph/IdentifyContainers.h"
 #include "type_graph/KeyCapture.h"
 #include "type_graph/NameGen.h"
 #include "type_graph/Prune.h"
@@ -56,7 +55,6 @@ using type_graph::DrgnParserOptions;
 using type_graph::EnforceCompatibility;
 using type_graph::Enum;
 using type_graph::Flattener;
-using type_graph::IdentifyContainers;
 using type_graph::KeyCapture;
 using type_graph::Member;
 using type_graph::NameGen;
@@ -1147,7 +1145,7 @@ void CodeGen::addDrgnRoot(struct drgn_type* drgnType, TypeGraph& typeGraph) {
   DrgnParserOptions options{
       .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
   };
-  DrgnParser drgnParser{typeGraph, options};
+  DrgnParser drgnParser{typeGraph, containerInfos_, options};
   Type& parsedRoot = drgnParser.parse(drgnType);
   typeGraph.addRoot(parsedRoot);
 }
@@ -1159,7 +1157,6 @@ void CodeGen::transform(TypeGraph& typeGraph) {
   pm.addPass(RemoveTopLevelPointer::createPass());
   pm.addPass(Flattener::createPass());
   pm.addPass(AlignmentCalc::createPass());
-  pm.addPass(IdentifyContainers::createPass(containerInfos_));
   pm.addPass(TypeIdentifier::createPass(config_.passThroughTypes));
   if (config_.features[Feature::PruneTypeGraph])
     pm.addPass(Prune::createPass());
@@ -1169,13 +1166,12 @@ void CodeGen::transform(TypeGraph& typeGraph) {
     DrgnParserOptions options{
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
-    DrgnParser drgnParser{typeGraph, options};
+    DrgnParser drgnParser{typeGraph, containerInfos_, options};
     pm.addPass(AddChildren::createPass(drgnParser, symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(Flattener::createPass());
     pm.addPass(AlignmentCalc::createPass());
-    pm.addPass(IdentifyContainers::createPass(containerInfos_));
     pm.addPass(TypeIdentifier::createPass(config_.passThroughTypes));
     if (config_.features[Feature::PruneTypeGraph])
       pm.addPass(Prune::createPass());
diff --git a/oi/type_graph/AlignmentCalc.cpp b/oi/type_graph/AlignmentCalc.cpp
index 31a7a30..cf23eff 100644
--- a/oi/type_graph/AlignmentCalc.cpp
+++ b/oi/type_graph/AlignmentCalc.cpp
@@ -48,15 +48,7 @@ void AlignmentCalc::accept(Type& type) {
 }
 
 void AlignmentCalc::visit(Class& c) {
-  for (const auto& param : c.templateParams) {
-    accept(param.type());
-  }
-  for (const auto& parent : c.parents) {
-    accept(parent.type());
-  }
-  for (const auto& child : c.children) {
-    accept(child);
-  }
+  RecursiveVisitor::visit(c);
 
   uint64_t alignment = 1;
   for (auto& member : c.members) {
@@ -82,4 +74,12 @@ void AlignmentCalc::visit(Class& c) {
   }
 }
 
+void AlignmentCalc::visit(Container& c) {
+  RecursiveVisitor::visit(c);
+
+  if (c.underlying()) {
+    c.setAlign(c.underlying()->align());
+  }
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/AlignmentCalc.h b/oi/type_graph/AlignmentCalc.h
index 4949341..b89f114 100644
--- a/oi/type_graph/AlignmentCalc.h
+++ b/oi/type_graph/AlignmentCalc.h
@@ -41,6 +41,7 @@ class AlignmentCalc final : public RecursiveVisitor {
 
   void accept(Type& type) override;
   void visit(Class& c) override;
+  void visit(Container& c) override;
 
  private:
   std::unordered_set<Type*> visited_;
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 940c031..4e8f622 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -5,7 +5,6 @@ add_library(type_graph
   DrgnParser.cpp
   EnforceCompatibility.cpp
   Flattener.cpp
-  IdentifyContainers.cpp
   KeyCapture.cpp
   NameGen.cpp
   PassManager.cpp
diff --git a/oi/type_graph/DrgnParser.cpp b/oi/type_graph/DrgnParser.cpp
index 0116edf..07202b0 100644
--- a/oi/type_graph/DrgnParser.cpp
+++ b/oi/type_graph/DrgnParser.cpp
@@ -25,6 +25,8 @@ extern "C" {
 #include <drgn.h>
 }
 
+#include <regex>
+
 namespace oi::detail::type_graph {
 namespace {
 
@@ -146,7 +148,7 @@ Type& DrgnParser::enumerateType(struct drgn_type* type) {
   return *t;
 }
 
-Class& DrgnParser::enumerateClass(struct drgn_type* type) {
+Type& DrgnParser::enumerateClass(struct drgn_type* type) {
   std::string fqName;
   char* nameStr = nullptr;
   size_t length = 0;
@@ -180,14 +182,20 @@ Class& DrgnParser::enumerateClass(struct drgn_type* type) {
                             std::to_string(drgn_type_kind(type))};
   }
 
-  auto& c = makeType<Class>(
-      type, kind, std::move(name), std::move(fqName), size, virtuality);
+  Class& c =
+      makeType<Class>(type, kind, std::move(name), fqName, size, virtuality);
 
   enumerateClassTemplateParams(type, c.templateParams);
   enumerateClassParents(type, c.parents);
   enumerateClassMembers(type, c.members);
   enumerateClassFunctions(type, c.functions);
 
+  if (auto* info = getContainerInfo(fqName)) {
+    auto& container = makeType<Container>(type, *info, size, &c);
+    container.templateParams = c.templateParams;
+    return container;
+  }
+
   return c;
 }
 
@@ -505,6 +513,15 @@ bool DrgnParser::chasePointer() const {
   return options_.chaseRawPointers;
 }
 
+ContainerInfo* DrgnParser::getContainerInfo(const std::string& fqName) const {
+  for (const auto& containerInfo : containers_) {
+    if (std::regex_search(fqName, containerInfo->matcher)) {
+      return containerInfo.get();
+    }
+  }
+  return nullptr;
+}
+
 DrgnParserError::DrgnParserError(const std::string& msg, struct drgn_error* err)
     : std::runtime_error{msg + ": " + std::to_string(err->code) + " " +
                          err->message},
diff --git a/oi/type_graph/DrgnParser.h b/oi/type_graph/DrgnParser.h
index 5b8fbe3..edb566d 100644
--- a/oi/type_graph/DrgnParser.h
+++ b/oi/type_graph/DrgnParser.h
@@ -51,14 +51,16 @@ struct DrgnParserOptions {
  */
 class DrgnParser {
  public:
-  DrgnParser(TypeGraph& typeGraph, DrgnParserOptions options)
-      : typeGraph_(typeGraph), options_(options) {
+  DrgnParser(TypeGraph& typeGraph,
+             const std::vector<std::unique_ptr<ContainerInfo>>& containers,
+             DrgnParserOptions options)
+      : typeGraph_(typeGraph), containers_(containers), options_(options) {
   }
   Type& parse(struct drgn_type* root);
 
  private:
   Type& enumerateType(struct drgn_type* type);
-  Class& enumerateClass(struct drgn_type* type);
+  Type& enumerateClass(struct drgn_type* type);
   Enum& enumerateEnum(struct drgn_type* type);
   Typedef& enumerateTypedef(struct drgn_type* type);
   Type& enumeratePointer(struct drgn_type* type);
@@ -81,10 +83,11 @@ class DrgnParser {
   template <typename T, typename... Args>
   T& makeType(struct drgn_type* drgnType, Args&&... args) {
     auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
-    drgn_types_.insert({drgnType, newType});
+    drgn_types_.insert_or_assign(drgnType, newType);
     return newType;
   }
   bool chasePointer() const;
+  ContainerInfo* getContainerInfo(const std::string& fqName) const;
 
   // Store a mapping of drgn types to type graph nodes for deduplication during
   // parsing. This stops us getting caught in cycles.
@@ -92,6 +95,7 @@ class DrgnParser {
       drgn_types_;
 
   TypeGraph& typeGraph_;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containers_;
   int depth_;
   DrgnParserOptions options_;
 };
diff --git a/oi/type_graph/Flattener.cpp b/oi/type_graph/Flattener.cpp
index 57e3748..28a9f32 100644
--- a/oi/type_graph/Flattener.cpp
+++ b/oi/type_graph/Flattener.cpp
@@ -134,8 +134,6 @@ void Flattener::visit(Class& c) {
   //   };
   //    TODO comment about virtual inheritance
 
-  // TODO alignment of parent classes
-
   // Flatten types referenced by template params, parents and members
   for (const auto& param : c.templateParams) {
     accept(param.type());
@@ -207,12 +205,4 @@ void Flattener::visit(Class& c) {
   }
 }
 
-void Flattener::visit(Container& c) {
-  // Containers themselves don't need to be flattened, but their template
-  // parameters might need to be
-  for (const auto& templateParam : c.templateParams) {
-    accept(templateParam.type());
-  }
-}
-
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/Flattener.h b/oi/type_graph/Flattener.h
index 1d46f02..95f78c6 100644
--- a/oi/type_graph/Flattener.h
+++ b/oi/type_graph/Flattener.h
@@ -42,7 +42,6 @@ class Flattener : public RecursiveVisitor {
 
   void accept(Type& type) override;
   void visit(Class& c) override;
-  void visit(Container& c) override;
 
   static const inline std::string ParentPrefix = "__oi_parent";
 
diff --git a/oi/type_graph/IdentifyContainers.cpp b/oi/type_graph/IdentifyContainers.cpp
deleted file mode 100644
index c6207c2..0000000
--- a/oi/type_graph/IdentifyContainers.cpp
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "IdentifyContainers.h"
-
-#include <regex>
-
-#include "TypeGraph.h"
-#include "oi/ContainerInfo.h"
-
-namespace oi::detail::type_graph {
-
-Pass IdentifyContainers::createPass(
-    const std::vector<std::unique_ptr<ContainerInfo>>& containers) {
-  auto fn = [&containers](TypeGraph& typeGraph, NodeTracker&) {
-    IdentifyContainers typeId{typeGraph, containers};
-    for (auto& type : typeGraph.rootTypes()) {
-      type = typeId.mutate(type);
-    }
-  };
-
-  return Pass("IdentifyContainers", fn);
-}
-
-IdentifyContainers::IdentifyContainers(
-    TypeGraph& typeGraph,
-    const std::vector<std::unique_ptr<ContainerInfo>>& containers)
-    : tracker_(typeGraph.size()),
-      typeGraph_(typeGraph),
-      containers_(containers) {
-}
-
-Type& IdentifyContainers::mutate(Type& type) {
-  if (Type* mutated = tracker_.get(type))
-    return *mutated;
-
-  Type& mutated = type.accept(*this);
-  tracker_.set(type, mutated);
-  return mutated;
-}
-
-Type& IdentifyContainers::visit(Class& c) {
-  for (const auto& containerInfo : containers_) {
-    if (!std::regex_search(c.fqName(), containerInfo->matcher)) {
-      continue;
-    }
-
-    auto& container = typeGraph_.makeType<Container>(*containerInfo, c.size());
-    container.templateParams = c.templateParams;
-
-    tracker_.set(c, container);
-    RecursiveMutator::visit(container);
-    return container;
-  }
-
-  tracker_.set(c, c);
-  RecursiveMutator::visit(c);
-  return c;
-}
-
-}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/IdentifyContainers.h b/oi/type_graph/IdentifyContainers.h
deleted file mode 100644
index b03c8bd..0000000
--- a/oi/type_graph/IdentifyContainers.h
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#pragma once
-
-#include <functional>
-#include <unordered_map>
-#include <vector>
-
-#include "NodeTracker.h"
-#include "PassManager.h"
-#include "Types.h"
-#include "Visitor.h"
-#include "oi/ContainerInfo.h"
-
-namespace oi::detail::type_graph {
-
-class TypeGraph;
-
-/*
- * IdentifyContainers
- *
- * Walks a flattened type graph and replaces type nodes based on container
- * definition TOML files.
- */
-class IdentifyContainers : public RecursiveMutator {
- public:
-  static Pass createPass(
-      const std::vector<std::unique_ptr<ContainerInfo>>& containers);
-
-  IdentifyContainers(
-      TypeGraph& typeGraph,
-      const std::vector<std::unique_ptr<ContainerInfo>>& containers);
-
-  using RecursiveMutator::mutate;
-
-  Type& mutate(Type& type) override;
-  Type& visit(Class& c) override;
-
- private:
-  MutationTracker tracker_;
-  TypeGraph& typeGraph_;
-  const std::vector<std::unique_ptr<ContainerInfo>>& containers_;
-};
-
-}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index e7d70eb..45e3920 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -85,7 +85,7 @@ void Printer::visit(const Class& c) {
     print_function(function);
   }
   for (auto& child : c.children) {
-    print_child(child);
+    print_type("Child", child);
   }
 }
 
@@ -93,11 +93,14 @@ void Printer::visit(const Container& c) {
   if (prefix(c))
     return;
 
-  out_ << "Container: " << c.name() << " (size: " << c.size() << ")"
-       << std::endl;
+  out_ << "Container: " << c.name() << " (size: " << c.size()
+       << align_str(c.align()) << ")" << std::endl;
   for (const auto& param : c.templateParams) {
     print_param(param);
   }
+  if (c.underlying()) {
+    print_type("Underlying", *c.underlying());
+  }
 }
 
 void Printer::visit(const Primitive& p) {
@@ -240,10 +243,10 @@ void Printer::print_function(const Function& function) {
   depth_--;
 }
 
-void Printer::print_child(const Type& child) {
+void Printer::print_type(std::string_view header, const Type& child) {
   depth_++;
   prefix();
-  out_ << "Child" << std::endl;
+  out_ << header << std::endl;
   print(child);
   depth_--;
 }
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index a66581e..9970b3d 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -51,7 +51,7 @@ class Printer : public ConstVisitor {
   void print_parent(const Parent& parent);
   void print_member(const Member& member);
   void print_function(const Function& function);
-  void print_child(const Type& child);
+  void print_type(std::string_view header, const Type& child);
   void print_value(const std::string& value);
   void print_qualifiers(const QualifierSet& qualifiers);
   void print_enumerator(int64_t val, const std::string& name);
diff --git a/oi/type_graph/Prune.cpp b/oi/type_graph/Prune.cpp
index 903a8c2..8effd3b 100644
--- a/oi/type_graph/Prune.cpp
+++ b/oi/type_graph/Prune.cpp
@@ -39,18 +39,7 @@ void Prune::accept(Type& type) {
 }
 
 void Prune::visit(Class& c) {
-  for (const auto& param : c.templateParams) {
-    accept(param.type());
-  }
-  for (const auto& parent : c.parents) {
-    accept(parent.type());
-  }
-  for (const auto& member : c.members) {
-    accept(member.type());
-  }
-  for (const auto& child : c.children) {
-    accept(child);
-  }
+  RecursiveVisitor::visit(c);
 
   c.templateParams.clear();
   c.parents.clear();
@@ -62,4 +51,10 @@ void Prune::visit(Class& c) {
   c.functions.shrink_to_fit();
 }
 
+void Prune::visit(Container& c) {
+  RecursiveVisitor::visit(c);
+
+  c.setUnderlying(nullptr);
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/Prune.h b/oi/type_graph/Prune.h
index d444ea3..c0701db 100644
--- a/oi/type_graph/Prune.h
+++ b/oi/type_graph/Prune.h
@@ -43,6 +43,7 @@ class Prune : public RecursiveVisitor {
 
   void accept(Type& type) override;
   void visit(Class& c) override;
+  void visit(Container& c) override;
 
  private:
   NodeTracker& tracker_;
diff --git a/oi/type_graph/TypeIdentifier.cpp b/oi/type_graph/TypeIdentifier.cpp
index 17fa43a..bba08f8 100644
--- a/oi/type_graph/TypeIdentifier.cpp
+++ b/oi/type_graph/TypeIdentifier.cpp
@@ -79,7 +79,8 @@ void TypeIdentifier::visit(Container& c) {
               it != passThroughTypeDummys_.end()) {
             dummy = &it->second.get();
           } else {
-            dummy = &typeGraph_.makeType<Container>(info, param.type().size());
+            dummy = &typeGraph_.makeType<Container>(
+                info, param.type().size(), paramClass);
             dummy->templateParams = paramClass->templateParams;
             passThroughTypeDummys_.insert(it,
                                           {paramClass->id(), std::ref(*dummy)});
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f503886..c16821c 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -150,7 +150,6 @@ struct Function {
   int virtuality;
 };
 
-class Class;
 struct Parent {
   Parent(Type& type, uint64_t bitOffset) : type_(type), bitOffset(bitOffset) {
   }
@@ -298,10 +297,6 @@ class Class : public Type {
 
   DECLARE_ACCEPT
 
-  Kind kind() const {
-    return kind_;
-  }
-
   virtual const std::string& name() const override {
     return name_;
   }
@@ -326,12 +321,16 @@ class Class : public Type {
     return align_;
   }
 
+  void setAlign(uint64_t alignment) {
+    align_ = alignment;
+  }
+
   virtual NodeId id() const override {
     return id_;
   }
 
-  void setAlign(uint64_t alignment) {
-    align_ = alignment;
+  Kind kind() const {
+    return kind_;
   }
 
   int virtuality() const {
@@ -371,10 +370,19 @@ class Class : public Type {
   bool packed_ = false;
 };
 
+/*
+ * Container
+ *
+ * A type of class for which we can do special processing.
+ */
 class Container : public Type {
  public:
-  Container(NodeId id, const ContainerInfo& containerInfo, size_t size)
+  Container(NodeId id,
+            const ContainerInfo& containerInfo,
+            size_t size,
+            Type* underlying)
       : containerInfo_(containerInfo),
+        underlying_(underlying),
         name_(containerInfo.typeName),
         inputName_(containerInfo.typeName),
         size_(size),
@@ -386,6 +394,7 @@ class Container : public Type {
             const ContainerInfo& containerInfo)
       : templateParams(other.templateParams),
         containerInfo_(containerInfo),
+        underlying_(other.underlying_),
         name_(other.name_),
         inputName_(other.inputName_),
         size_(other.size_),
@@ -396,22 +405,18 @@ class Container : public Type {
 
   DECLARE_ACCEPT
 
-  const std::string& containerName() const {
-    return containerInfo_.typeName;
-  }
-
   virtual const std::string& name() const override {
     return name_;
   }
 
-  void setName(std::string name) {
-    name_ = std::move(name);
-  }
-
   virtual std::string_view inputName() const override {
     return inputName_;
   }
 
+  void setName(std::string name) {
+    name_ = std::move(name);
+  }
+
   void setInputName(std::string name) {
     inputName_ = std::move(name);
   }
@@ -424,18 +429,31 @@ class Container : public Type {
     return align_;
   }
 
+  void setAlign(uint64_t alignment) {
+    align_ = alignment;
+  }
+
   virtual NodeId id() const override {
     return id_;
   }
 
-  void setAlign(uint64_t alignment) {
-    align_ = alignment;
+  const std::string& containerName() const {
+    return containerInfo_.typeName;
+  }
+
+  Type* underlying() const {
+    return underlying_;
+  }
+
+  void setUnderlying(Type* underlying) {
+    underlying_ = underlying;
   }
 
   std::vector<TemplateParam> templateParams;
   const ContainerInfo& containerInfo_;
 
  private:
+  Type* underlying_;
   std::string name_;
   std::string inputName_;
   size_t size_;
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index 7e4395e..9e05821 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -107,6 +107,7 @@ class RecursiveVisitor : public Visitor {
     for (const auto& param : c.templateParams) {
       accept(param.type());
     }
+    accept(c.underlying());
   }
   virtual void visit(Primitive&) {
   }
@@ -140,6 +141,11 @@ class RecursiveMutator : public Mutator {
  public:
   virtual ~RecursiveMutator() = default;
   virtual Type& mutate(Type&) = 0;
+  virtual Type* mutate(Type* type) {
+    if (type)
+      return &mutate(*type);
+    return nullptr;
+  }
   virtual Type& visit(Incomplete& i) {
     return i;
   }
@@ -162,6 +168,7 @@ class RecursiveMutator : public Mutator {
     for (auto& param : c.templateParams) {
       param.setType(mutate(param.type()));
     }
+    c.setUnderlying(mutate(c.underlying()));
     return c;
   }
   virtual Type& visit(Primitive& p) {
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 287ebcd..24b8f68 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -43,7 +43,6 @@ add_executable(test_type_graph
   test_drgn_parser.cpp
   test_enforce_compatibility.cpp
   test_flattener.cpp
-  test_identify_containers.cpp
   test_key_capture.cpp
   test_name_gen.cpp
   test_node_tracker.cpp
diff --git a/test/TypeGraphParser.cpp b/test/TypeGraphParser.cpp
index 7b6f425..f54fe08 100644
--- a/test/TypeGraphParser.cpp
+++ b/test/TypeGraphParser.cpp
@@ -265,10 +265,11 @@ Type& TypeGraphParser::parseType(std::string_view& input, size_t rootIndent) {
 
     auto size = parseNumericAttribute(line, nodeTypeName, "size: ");
 
-    Container& c = typeGraph_.makeType<Container>(id, info, size);
+    Container& c = typeGraph_.makeType<Container>(id, info, size, nullptr);
     nodesById_.insert({id, c});
 
     parseParams(c, input, indent + 2);
+    parseUnderlying(c, input, indent + 2);
 
     type = &c;
   } else if (nodeTypeName == "Primitive") {
@@ -452,3 +453,26 @@ void TypeGraphParser::parseChildren(Class& c,
   // No more children for us - put back the line we just read
   input = origInput;
 }
+
+void TypeGraphParser::parseUnderlying(Container& c,
+                                      std::string_view& input,
+                                      size_t rootIndent) {
+  std::string_view origInput = input;
+  std::string_view line;
+  getline(input, line);
+
+  size_t indent = stripIndent(line);
+  if (indent != rootIndent) {
+    input = origInput;
+    return;
+  }
+
+  // Format: "Underlying"
+  if (!tryRemovePrefix(line, "Underlying")) {
+    input = origInput;
+    return;
+  }
+
+  Type& type = parseType(input, rootIndent + 2);
+  c.setUnderlying(&type);
+}
diff --git a/test/TypeGraphParser.h b/test/TypeGraphParser.h
index 626c4a4..640b706 100644
--- a/test/TypeGraphParser.h
+++ b/test/TypeGraphParser.h
@@ -34,6 +34,9 @@ class TypeGraphParser {
   void parseMembers(Class& c, std::string_view& input, size_t rootIndent);
   void parseFunctions(Class& c, std::string_view& input, size_t rootIndent);
   void parseChildren(Class& c, std::string_view& input, size_t rootIndent);
+  void parseUnderlying(Container& c,
+                       std::string_view& input,
+                       size_t rootIndent);
 };
 
 class TypeGraphParserError : public std::runtime_error {
diff --git a/test/test_add_children.cpp b/test/test_add_children.cpp
index aca046e..fd8e66b 100644
--- a/test/test_add_children.cpp
+++ b/test/test_add_children.cpp
@@ -79,11 +79,11 @@ TEST_F(AddChildrenTest, InheritancePolymorphic) {
          Function: A
          Function: A
          Child
-[17]       Class: B (size: 40)
+[18]       Class: B (size: 40)
              Parent (offset: 0)
                [0]
              Member: vec_b (offset: 16)
-[4]            Class: vector<int, std::allocator<int> > (size: 24)
+[17]           Container: std::vector (size: 24)
                  Param
                    Primitive: int32_t
                  Param
@@ -106,15 +106,17 @@ TEST_F(AddChildrenTest, InheritancePolymorphic) {
                      Function: ~allocator
                      Function: allocate
                      Function: deallocate
-                 *
+                 Underlying
+[4]                Class: vector<int, std::allocator<int> > (size: 24)
+*
              Function: ~B (virtual)
              Function: myfunc (virtual)
              Function: B
              Function: B
              Child
-[19]           Class: C (size: 48)
+[20]           Class: C (size: 48)
                  Parent (offset: 0)
-                   [17]
+                   [18]
                  Member: int_c (offset: 40)
                    Primitive: int32_t
                  Function: ~C (virtual)
@@ -135,11 +137,11 @@ TEST_F(AddChildrenTest, InheritancePolymorphic) {
          Function: ~A (virtual)
          Function: myfunc (virtual)
          Child
-[13]       Class: B (size: 40)
+[14]       Class: B (size: 40)
              Parent (offset: 0)
                [0]
              Member: vec_b (offset: 16)
-[4]            Class: vector<int, std::allocator<int> > (size: 24)
+[13]           Container: std::vector (size: 24)
                  Param
                    Primitive: int32_t
                  Param
@@ -159,6 +161,8 @@ TEST_F(AddChildrenTest, InheritancePolymorphic) {
                      Function: ~allocator
                      Function: allocate
                      Function: deallocate
+                 Underlying
+[4]                Class: vector<int, std::allocator<int> > (size: 24)
                  *
              Function: operator=
              Function: B
@@ -166,9 +170,9 @@ TEST_F(AddChildrenTest, InheritancePolymorphic) {
              Function: ~B (virtual)
              Function: myfunc (virtual)
              Child
-[15]           Class: C (size: 48)
+[16]           Class: C (size: 48)
                  Parent (offset: 0)
-                   [13]
+                   [14]
                  Member: int_c (offset: 40)
                    Primitive: int32_t
                  Function: operator=
diff --git a/test/test_alignment_calc.cpp b/test/test_alignment_calc.cpp
index 5bc4573..2526cbe 100644
--- a/test/test_alignment_calc.cpp
+++ b/test/test_alignment_calc.cpp
@@ -58,16 +58,16 @@ TEST(AlignmentCalcTest, StructInContainer) {
           Member: n (offset: 0)
             Primitive: int8_t
           Member: n (offset: 8)
-            Primitive: int64_t
+            Primitive: int32_t
 )",
        R"(
 [0] Container: std::vector (size: 8)
       Param
-[1]     Class: MyClass (size: 16, align: 8)
+[1]     Class: MyClass (size: 16, align: 4)
           Member: n (offset: 0, align: 1)
             Primitive: int8_t
-          Member: n (offset: 8, align: 8)
-            Primitive: int64_t
+          Member: n (offset: 8, align: 4)
+            Primitive: int32_t
 )");
 }
 
@@ -264,3 +264,33 @@ TEST(AlignmentCalcTest, Typedef) {
               Primitive: int8_t
 )");
 }
+
+TEST(AlignmentCalcTest, Container) {
+  test(AlignmentCalc::createPass(),
+       R"(
+[0] Container: std::vector (size: 24)
+      Underlying
+[1]     Class: vector (size: 24)
+          Member: n (offset: 0)
+            Primitive: int8_t
+          Member: s (offset: 4)
+[2]         Struct: MyStruct (size: 8)
+              Member: n1 (offset: 0)
+                Primitive: int32_t
+              Member: n2 (offset: 4)
+                Primitive: int32_t
+)",
+       R"(
+[0] Container: std::vector (size: 24, align: 4)
+      Underlying
+[1]     Class: vector (size: 24, align: 4)
+          Member: n (offset: 0, align: 1)
+            Primitive: int8_t
+          Member: s (offset: 4, align: 4)
+[2]         Struct: MyStruct (size: 8, align: 4)
+              Member: n1 (offset: 0, align: 4)
+                Primitive: int32_t
+              Member: n2 (offset: 4, align: 4)
+                Primitive: int32_t
+)");
+}
diff --git a/test/test_codegen.cpp b/test/test_codegen.cpp
index 660c411..3e7855e 100644
--- a/test/test_codegen.cpp
+++ b/test/test_codegen.cpp
@@ -45,13 +45,15 @@ void testTransform(OICodeGen::Config& config,
 
 void testTransform(std::string_view input, std::string_view expectedAfter) {
   OICodeGen::Config config;
+  config.features[Feature::PruneTypeGraph] = true;
+  config.features[Feature::TreeBuilderV2] = true;
   testTransform(config, input, expectedAfter);
 }
 }  // namespace
 
 TEST(CodeGenTest, TransformContainerAllocator) {
   testTransform(R"(
-[0] Class: std::vector (size: 24)
+[0] Container: std::vector (size: 24)
       Param
         Primitive: int32_t
       Param
@@ -62,18 +64,18 @@ TEST(CodeGenTest, TransformContainerAllocator) {
           Function: deallocate
 )",
                 R"(
-[2] Container: std::vector<int32_t, DummyAllocator<int32_t, 8, 1, 3>> (size: 24)
+[0] Container: std::vector<int32_t, DummyAllocator<int32_t, 8, 1, 2>> (size: 24)
       Param
         Primitive: int32_t
       Param
-[3]     DummyAllocator [MyAlloc] (size: 8, align: 1)
+[2]     DummyAllocator [MyAlloc] (size: 8, align: 1)
           Primitive: int32_t
 )");
 }
 
 TEST(CodeGenTest, TransformContainerAllocatorParamInParent) {
   testTransform(R"(
-[0] Class: std::map (size: 24)
+[0] Container: std::map (size: 24)
       Param
         Primitive: int32_t
       Param
@@ -83,7 +85,7 @@ TEST(CodeGenTest, TransformContainerAllocatorParamInParent) {
           Parent (offset: 0)
 [2]         Struct: MyAllocBase<std::pair<const int, int>> (size: 1)
               Param
-[3]             Class: std::pair (size: 8)
+[3]             Container: std::pair (size: 8)
                   Param
                     Primitive: int32_t
                     Qualifiers: const
@@ -95,14 +97,14 @@ TEST(CodeGenTest, TransformContainerAllocatorParamInParent) {
           Function: deallocate
 )",
                 R"(
-[4] Container: std::map<int32_t, int32_t, DummyAllocator<std::pair<int32_t const, int32_t>, 0, 1, 6>> (size: 24)
+[0] Container: std::map<int32_t, int32_t, DummyAllocator<std::pair<int32_t const, int32_t>, 0, 1, 4>> (size: 24)
       Param
         Primitive: int32_t
       Param
         Primitive: int32_t
       Param
-[6]     DummyAllocator [MyAlloc<std::pair<const int, int>>] (size: 0, align: 1)
-[5]       Container: std::pair<int32_t const, int32_t> (size: 8)
+[4]     DummyAllocator [MyAlloc<std::pair<const int, int>>] (size: 0, align: 1)
+[3]       Container: std::pair<int32_t const, int32_t> (size: 8)
             Param
               Primitive: int32_t
               Qualifiers: const
@@ -157,7 +159,7 @@ TEST(CodeGenTest, UnionMembersAlignment) {
 
 TEST(CodeGenTest, ReplaceContainersAndDummies) {
   testTransform(R"(
-[0] Class: std::vector (size: 24)
+[0] Container: std::vector (size: 24)
       Param
         Primitive: uint32_t
       Param
@@ -168,11 +170,87 @@ TEST(CodeGenTest, ReplaceContainersAndDummies) {
           Function: deallocate
 )",
                 R"(
-[2] Container: std::vector<uint32_t, DummyAllocator<uint32_t, 0, 1, 3>> (size: 24)
+[0] Container: std::vector<uint32_t, DummyAllocator<uint32_t, 0, 1, 2>> (size: 24)
       Param
         Primitive: uint32_t
       Param
-[3]     DummyAllocator [allocator<int>] (size: 0, align: 1)
+[2]     DummyAllocator [allocator<int>] (size: 0, align: 1)
           Primitive: uint32_t
 )");
 }
+
+TEST(CodeGenTest, ContainerAlignment) {
+  testTransform(R"(
+[0] Class: MyClass (size: 24)
+      Member: container (offset: 0)
+[1]     Container: std::vector (size: 24)
+          Param
+            Primitive: int32_t
+          Underlying
+[2]         Class: vector (size: 24)
+              Member: __impl__ (offset: 0)
+                Primitive: StubbedPointer
+              Member: __impl__ (offset: 8)
+                Primitive: StubbedPointer
+              Member: __impl__ (offset: 16)
+                Primitive: StubbedPointer
+)",
+                R"(
+[0] Class: MyClass_0 [MyClass] (size: 24, align: 8)
+      Member: container_0 [container] (offset: 0, align: 8)
+[1]     Container: std::vector<int32_t> (size: 24, align: 8)
+          Param
+            Primitive: int32_t
+)");
+}
+
+TEST(CodeGenTest, InheritFromContainer) {
+  testTransform(R"(
+[0] Class: MyClass (size: 24)
+      Parent (offset: 0)
+[1]     Container: std::vector (size: 24)
+          Param
+            Primitive: int32_t
+          Underlying
+[2]         Class: vector (size: 24)
+              Member: __impl__ (offset: 0)
+                Primitive: StubbedPointer
+              Member: __impl__ (offset: 8)
+                Primitive: StubbedPointer
+              Member: __impl__ (offset: 16)
+                Primitive: StubbedPointer
+)",
+                R"(
+[0] Class: MyClass_0 [MyClass] (size: 24, align: 8)
+      Member: __oi_parent_0 [__oi_parent] (offset: 0, align: 8)
+[1]     Container: std::vector<int32_t> (size: 24, align: 8)
+          Param
+            Primitive: int32_t
+)");
+}
+
+TEST(CodeGenTest, InheritFromContainerCompat) {
+  OICodeGen::Config config;
+  testTransform(config,
+                R"(
+[0] Class: MyClass (size: 24)
+      Parent (offset: 0)
+[1]     Container: std::vector (size: 24)
+          Param
+            Primitive: int32_t
+          Underlying
+[2]         Class: vector (size: 24)
+              Member: __impl__ (offset: 0)
+                Primitive: StubbedPointer
+              Member: __impl__ (offset: 8)
+                Primitive: StubbedPointer
+              Member: __impl__ (offset: 16)
+                Primitive: StubbedPointer
+)",
+                R"(
+[0] Class: MyClass_0 [MyClass] (size: 24, align: 8)
+      Member: __oi_padding_0 (offset: 0)
+[3]     Array: [int8_t[24]] (length: 24)
+          Primitive: int8_t
+)");
+}
diff --git a/test/test_drgn_parser.cpp b/test/test_drgn_parser.cpp
index a5ebe30..4a51605 100644
--- a/test/test_drgn_parser.cpp
+++ b/test/test_drgn_parser.cpp
@@ -10,6 +10,7 @@
 #include "oi/type_graph/Printer.h"
 #include "oi/type_graph/TypeGraph.h"
 #include "oi/type_graph/Types.h"
+#include "test/type_graph_utils.h"
 
 using namespace type_graph;
 
@@ -28,7 +29,8 @@ void DrgnParserTest::TearDownTestSuite() {
 
 DrgnParser DrgnParserTest::getDrgnParser(TypeGraph& typeGraph,
                                          DrgnParserOptions options) {
-  DrgnParser drgnParser{typeGraph, options};
+  static auto infos = getContainerInfos();
+  DrgnParser drgnParser{typeGraph, infos, options};
   return drgnParser;
 }
 
@@ -255,8 +257,8 @@ TEST_F(DrgnParserTest, InheritanceMultiple) {
 TEST_F(DrgnParserTest, Container) {
   testMultiCompilerGlob("oid_test_case_std_vector_int_empty",
                         R"(
-[13] Pointer
-[0]    Class: vector<int, std::allocator<int> > (size: 24)
+[14] Pointer
+[13]   Container: std::vector (size: 24)
          Param
            Primitive: int32_t
          Param
@@ -279,12 +281,18 @@ TEST_F(DrgnParserTest, Container) {
              Function: ~allocator
              Function: allocate
              Function: deallocate
-         Parent (offset: 0)
+         Underlying
+[0]        Class: vector<int, std::allocator<int> > (size: 24)
+             Param
+               Primitive: int32_t
+             Param
+               [1]
+             Parent (offset: 0)
 *
 )",
                         R"(
-[9]  Pointer
-[0]    Class: vector<int, std::allocator<int> > (size: 24)
+[10] Pointer
+[9]    Container: std::vector (size: 24)
          Param
            Primitive: int32_t
          Param
@@ -304,7 +312,13 @@ TEST_F(DrgnParserTest, Container) {
              Function: ~allocator
              Function: allocate
              Function: deallocate
-         Parent (offset: 0)
+         Underlying
+[0]        Class: vector<int, std::allocator<int> > (size: 24)
+             Param
+               Primitive: int32_t
+             Param
+               [1]
+             Parent (offset: 0)
 *
 )");
 }
diff --git a/test/test_flattener.cpp b/test/test_flattener.cpp
index eab3cf8..0f73d03 100644
--- a/test/test_flattener.cpp
+++ b/test/test_flattener.cpp
@@ -799,6 +799,27 @@ TEST(FlattenerTest, ParentClassAndContainer) {
 )");
 }
 
+TEST(FlattenerTest, ContainerWithParent) {
+  // This is necessary to correctly calculate container alignment
+  test(Flattener::createPass(),
+       R"(
+[0] Container: std::vector (size: 24)
+      Underlying
+[1]     Class: vector (size: 24)
+          Parent (offset: 0)
+[2]         Class: Parent (size: 4)
+              Member: x (offset: 0)
+                Primitive: int32_t
+)",
+       R"(
+[0] Container: std::vector (size: 24)
+      Underlying
+[1]     Class: vector (size: 24)
+          Member: x (offset: 0)
+            Primitive: int32_t
+)");
+}
+
 TEST(FlattenerTest, AllocatorParamInParent) {
   test(Flattener::createPass(),
        R"(
@@ -857,8 +878,7 @@ TEST(FlattenerTest, AllocatorUnfixableNoParent) {
 )");
 }
 
-TEST(FlattenerTest, AllocatorUnfixableParentNotClass) {
-  // This could be supported if need-be, we just don't do it yet
+TEST(FlattenerTest, AllocatorParamInParentContainer) {
   test(Flattener::createPass(),
        R"(
 [0] Container: std::vector (size: 24)
diff --git a/test/test_identify_containers.cpp b/test/test_identify_containers.cpp
deleted file mode 100644
index 4c359eb..0000000
--- a/test/test_identify_containers.cpp
+++ /dev/null
@@ -1,194 +0,0 @@
-#include <gtest/gtest.h>
-
-#include "oi/ContainerInfo.h"
-#include "oi/type_graph/IdentifyContainers.h"
-#include "oi/type_graph/Types.h"
-#include "test/type_graph_utils.h"
-
-using namespace type_graph;
-
-namespace {
-void test(std::string_view input, std::string_view expectedAfter) {
-  ::test(IdentifyContainers::createPass(getContainerInfos()),
-         input,
-         expectedAfter);
-}
-};  // namespace
-
-TEST(IdentifyContainers, Container) {
-  test(R"(
-[0] Class: std::vector (size: 24)
-      Param
-        Primitive: int32_t
-      Member: a (offset: 0)
-        Primitive: int32_t
-)",
-       R"(
-[1] Container: std::vector (size: 24)
-      Param
-        Primitive: int32_t
-)");
-}
-
-TEST(IdentifyContainers, ContainerInClass) {
-  test(R"(
-[0] Class: MyClass (size: 0)
-      Param
-[1]     Class: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-      Parent (offset: 0)
-[2]     Class: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-      Member: a (offset: 0)
-[3]     Class: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-)",
-       R"(
-[0] Class: MyClass (size: 0)
-      Param
-[4]     Container: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-      Parent (offset: 0)
-[5]     Container: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-      Member: a (offset: 0)
-[6]     Container: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-)");
-}
-
-TEST(IdentifyContainers, ContainerInContainer) {
-  test(R"(
-[0] Class: std::vector (size: 24)
-      Param
-[1]     Class: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-)",
-       R"(
-[2] Container: std::vector (size: 24)
-      Param
-[3]     Container: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-)");
-}
-
-TEST(IdentifyContainers, ContainerInContainer2) {
-  test(R"(
-[0] Container: std::vector (size: 24)
-      Param
-[1]     Class: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-)",
-       R"(
-[0] Container: std::vector (size: 24)
-      Param
-[2]     Container: std::vector (size: 24)
-          Param
-            Primitive: int32_t
-)");
-}
-
-TEST(IdentifyContainers, ContainerInArray) {
-  test(R"(
-[0] Array: (length: 2)
-[1]   Class: std::vector (size: 24)
-        Param
-          Primitive: int32_t
-)",
-       R"(
-[0] Array: (length: 2)
-[2]   Container: std::vector (size: 24)
-        Param
-          Primitive: int32_t
-)");
-}
-
-TEST(IdentifyContainers, ContainerInTypedef) {
-  test(R"(
-[0] Typedef: MyAlias
-[1]   Class: std::vector (size: 24)
-        Param
-          Primitive: int32_t
-)",
-       R"(
-[0] Typedef: MyAlias
-[2]   Container: std::vector (size: 24)
-        Param
-          Primitive: int32_t
-)");
-}
-
-TEST(IdentifyContainers, ContainerInPointer) {
-  test(R"(
-[0] Pointer
-[1]   Class: std::vector (size: 24)
-        Param
-          Primitive: int32_t
-)",
-       R"(
-[0] Pointer
-[2]   Container: std::vector (size: 24)
-        Param
-          Primitive: int32_t
-)");
-}
-
-TEST(IdentifyContainers, ContainerDuplicate) {
-  test(R"(
-[0] Class: std::vector (size: 24)
-      Param
-        Primitive: int32_t
-      Member: a (offset: 0)
-        Primitive: int32_t
-    [0]
-)",
-       R"(
-[1] Container: std::vector (size: 24)
-      Param
-        Primitive: int32_t
-    [1]
-)");
-}
-
-TEST(IdentifyContainers, CycleClass) {
-  test(R"(
-[0] Class: ClassA (size: 0)
-      Member: x (offset: 0)
-[1]     Class: ClassB (size: 0)
-          Param
-            [0]
-)",
-       R"(
-[0] Class: ClassA (size: 0)
-      Member: x (offset: 0)
-[1]     Class: ClassB (size: 0)
-          Param
-            [0]
-)");
-}
-
-TEST(IdentifyContainers, CycleContainer) {
-  test(R"(
-[0] Class: ClassA (size: 0)
-      Member: x (offset: 0)
-[1]     Class: std::vector (size: 0)
-          Param
-            [0]
-)",
-       R"(
-[0] Class: ClassA (size: 0)
-      Member: x (offset: 0)
-[2]     Container: std::vector (size: 0)
-          Param
-            [0]
-)");
-}
diff --git a/test/test_prune.cpp b/test/test_prune.cpp
index 3ae5d61..ad8ceee 100644
--- a/test/test_prune.cpp
+++ b/test/test_prune.cpp
@@ -63,3 +63,33 @@ TEST(PruneTest, RecurseClassChild) {
 [1]     Class: ClassA (size: 12)
 )");
 }
+
+TEST(PruneTest, PruneContainer) {
+  test(Prune::createPass(),
+       R"(
+[0] Container: std::vector (size: 24)
+      Param
+        Primitive: int32_t
+      Param
+        Value: "123"
+        Primitive: int32_t
+      Underlying
+[1]     Class: vector<int32_t> (size: 24)
+          Parent (offset: 0)
+[2]         Class: MyParent (size: 4)
+              Member: a (offset: 0)
+                Primitive: int32_t
+          Member: a (offset: 0)
+            Primitive: int32_t
+          Member: b (offset: 4)
+            Primitive: int32_t
+)",
+       R"(
+[0] Container: std::vector (size: 24)
+      Param
+        Primitive: int32_t
+      Param
+        Value: "123"
+        Primitive: int32_t
+)");
+}
diff --git a/test/test_type_identifier.cpp b/test/test_type_identifier.cpp
index b2b8c8c..ec9efd5 100644
--- a/test/test_type_identifier.cpp
+++ b/test/test_type_identifier.cpp
@@ -108,6 +108,12 @@ TEST(TypeIdentifierTest, PassThroughTypes) {
 [2]     Container: std::allocator (size: 1)
           Param
             Primitive: int32_t
+          Underlying
+[1]         Class: std::allocator (size: 1)
+              Param
+                Primitive: int32_t
+              Function: allocate
+              Function: deallocate
 )");
 }
 
@@ -137,6 +143,12 @@ TEST(TypeIdentifierTest, PassThroughSameType) {
 [2]     Container: std::allocator (size: 1)
           Param
             Primitive: int32_t
+          Underlying
+[1]         Class: std::allocator (size: 1)
+              Param
+                Primitive: int32_t
+              Function: allocate
+              Function: deallocate
       Param
         [2]
 )");
diff --git a/test/type_graph_utils.cpp b/test/type_graph_utils.cpp
index 815a46a..2d5709f 100644
--- a/test/type_graph_utils.cpp
+++ b/test/type_graph_utils.cpp
@@ -86,22 +86,23 @@ std::vector<std::unique_ptr<ContainerInfo>> getContainerInfos() {
 Container getVector(NodeId id) {
   static ContainerInfo info{"std::vector", SEQ_TYPE, "vector"};
   info.stubTemplateParams = {1};
-  return Container{id, info, 24};
+  return Container{id, info, 24, nullptr};
 }
 
 Container getMap(NodeId id) {
   static ContainerInfo info{"std::map", STD_MAP_TYPE, "map"};
   info.stubTemplateParams = {2, 3};
-  return Container{id, info, 48};
+  return Container{id, info, 48, nullptr};
 }
 
 Container getList(NodeId id) {
   static ContainerInfo info{"std::list", LIST_TYPE, "list"};
   info.stubTemplateParams = {1};
-  return Container{id, info, 24};
+  return Container{id, info, 24, nullptr};
 }
 
 Container getPair(NodeId id) {
   static ContainerInfo info{"std::pair", PAIR_TYPE, "utility"};
-  return Container{id, info, 8};  // Nonsense size, shouldn't matter for tests
+  return Container{
+      id, info, 8, nullptr};  // Nonsense size, shouldn't matter for tests
 }

From a8fbefdcb8413473d6991faf63608a1feac9ba95 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 13 Dec 2023 16:09:58 +0000
Subject: [PATCH 051/188] type_graph: Add reference type

Our existing drgn parser emits pointers, regardless of whether the type in the
DWARF is really a reference. Newer parsers like the LLDB/Clang parsers can
differentiate between a pointer and a reference. We should do this as
references can be safely followed according to the language rules with
`ChaseRawPointers` disabled, while pointers cannot.

Test Plan:
- TBD
---
 oi/type_graph/NameGen.cpp               |  8 ++++
 oi/type_graph/NameGen.h                 |  1 +
 oi/type_graph/Printer.cpp               | 11 +++++
 oi/type_graph/Printer.h                 |  1 +
 oi/type_graph/RemoveTopLevelPointer.cpp |  4 ++
 oi/type_graph/RemoveTopLevelPointer.h   |  1 +
 oi/type_graph/Types.h                   | 56 +++++++++++++++++++++++++
 oi/type_graph/Visitor.h                 |  7 ++++
 8 files changed, 89 insertions(+)

diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index c792766..5213b3d 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -194,6 +194,14 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& p) {
+  RecursiveVisitor::visit(p);
+  p.regenerateName();
+  std::string inputName{p.pointeeType().inputName()};
+  inputName += '*';
+  p.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..e6437d0 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index 45e3920..f07ca48 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -150,6 +150,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& p) {
+  if (prefix(p))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = p.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(p.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index 9970b3d..e2e5068 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& p) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;
diff --git a/oi/type_graph/RemoveTopLevelPointer.cpp b/oi/type_graph/RemoveTopLevelPointer.cpp
index 9ecaaaa..e1909a7 100644
--- a/oi/type_graph/RemoveTopLevelPointer.cpp
+++ b/oi/type_graph/RemoveTopLevelPointer.cpp
@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& p) {
+  topLevelType_ = &p.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/RemoveTopLevelPointer.h b/oi/type_graph/RemoveTopLevelPointer.h
index 611429f..6b70545 100644
--- a/oi/type_graph/RemoveTopLevelPointer.h
+++ b/oi/type_graph/RemoveTopLevelPointer.h
@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
 
  private:
   Type* topLevelType_ = nullptr;
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index c16821c..21c03c9 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -733,6 +734,61 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    name_ = pointeeType_.get().name() + "&";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index 9e05821..7a8f12a 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -122,6 +122,9 @@ class RecursiveVisitor : public Visitor {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -189,6 +192,10 @@ class RecursiveMutator : public Mutator {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }

From 336a13cc2faa53aea28756b494ef837e883a5b4a Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 13 Dec 2023 16:11:53 +0000
Subject: [PATCH 052/188] oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD
---
 CMakeLists.txt                                |  13 +-
 oi/CodeGen.cpp                                |  30 +-
 oi/CodeGen.h                                  |   8 +-
 oi/Config.cpp                                 |   9 +-
 oi/ContainerInfo.cpp                          |   2 +-
 oi/OICodeGen.h                                |   2 +-
 oi/OIGenerator.cpp                            | 362 +++++++++-------
 oi/OIGenerator.h                              |  29 +-
 oi/type_graph/CMakeLists.txt                  |   2 +
 oi/type_graph/ClangTypeParser.cpp             | 401 ++++++++++++++++++
 oi/type_graph/ClangTypeParser.h               | 125 ++++++
 test/integration/template_template_param.toml |  16 +
 tools/OILGen.cpp                              | 147 +++----
 types/string_type.toml                        |  31 ++
 14 files changed, 887 insertions(+), 290 deletions(-)
 create mode 100644 oi/type_graph/ClangTypeParser.cpp
 create mode 100644 oi/type_graph/ClangTypeParser.h
 create mode 100644 test/integration/template_template_param.toml

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f9ee7..eb18f4a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -104,6 +104,15 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(range-v3)
 
+### cxxopts
+FetchContent_Declare(
+  cxxopts
+  GIT_REPOSITORY https://github.com/jarro2783/cxxopts.git
+  GIT_TAG        eb787304d67ec22f7c3a184ee8b4c481d04357fd # v3.1.1
+  GIT_PROGRESS   TRUE
+)
+FetchContent_MakeAvailable(cxxopts)
+
 set_project_warnings()
 
 if (ASAN)
@@ -289,7 +298,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -363,8 +372,10 @@ add_executable(oilgen
   oi/OIGenerator.cpp
 )
 target_link_libraries(oilgen
+  cxxopts
   drgn_utils
   oicore
+  clangTooling
 )
 
 ### Object Introspection cache Printer (OIP)
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 5a4b9eb..e840104 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -537,7 +537,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1104,15 +1104,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
 }
 
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
-  try {
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
+  if (!registerContainers())
     return false;
-  }
 
   TypeGraph typeGraph;
   try {
@@ -1127,6 +1120,20 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
   return true;
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    size_t i = 0;
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1167,7 +1174,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, containerInfos_, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(Flattener::createPass());
@@ -1282,7 +1289,8 @@ void CodeGen::generate(
   code += "\nusing __ROOT_TYPE__ = " + rootType.name() + ";\n";
   code += "} // namespace\n} // namespace OIInternal\n";
 
-  const auto typeName = SymbolService::getTypeName(drgnType);
+  // const auto typeName = SymbolService::getTypeName(drgnType);
+  const std::string typeName{"typeName"};
   if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
   } else {
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 186ec34..fb0ec20 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -40,8 +40,11 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config) : config_(config) {
+    assert(!config.features[Feature::PolymorphicInheritance]);
+  }
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   /*
@@ -53,6 +56,7 @@ class CodeGen {
                        std::string linkageName,
                        std::string& code);
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -66,7 +70,7 @@ class CodeGen {
 
  private:
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
diff --git a/oi/Config.cpp b/oi/Config.cpp
index 157b279..66798bb 100644
--- a/oi/Config.cpp
+++ b/oi/Config.cpp
@@ -22,6 +22,8 @@
 #include <boost/property_tree/ini_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 #include <filesystem>
+#include <range/v3/action/sort.hpp>
+#include <range/v3/action/unique.hpp>
 
 #include "oi/support/Toml.h"
 
@@ -54,6 +56,9 @@ std::optional<FeatureSet> processConfigFiles(
     enables |= *fs;
   }
 
+  ranges::actions::sort(generatorConfig.containerConfigPaths);
+  ranges::actions::unique(generatorConfig.containerConfigPaths);
+
   // Override anything from the config files with command line options
   for (auto [k, v] : featureMap) {
     enables[k] = v;
@@ -108,8 +113,8 @@ std::optional<FeatureSet> processConfigFile(
           if the right path is absolute, else append the right path to the
           left path.
           */
-          generatorConfig.containerConfigPaths.emplace(configDirectory /
-                                                       el.get());
+          generatorConfig.containerConfigPaths.emplace_back(configDirectory /
+                                                            el.get());
         }
       });
     }
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 99a7dc1..dbe10c6 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -202,7 +202,7 @@ std::regex getMatcher(const std::string& typeName) {
 ContainerInfo::ContainerInfo(const fs::path& path) {
   toml::table container;
   try {
-    container = toml::parse_file(std::string(path));
+    container = toml::parse_file(path.string());
   } catch (const toml::parse_error& err) {
     // Convert into a ContainerInfoError, just to avoid having to include
     // the huge TOML++ header in the caller's file. Use toml::parse_error's
diff --git a/oi/OICodeGen.h b/oi/OICodeGen.h
index a01548f..7226a4e 100644
--- a/oi/OICodeGen.h
+++ b/oi/OICodeGen.h
@@ -66,7 +66,7 @@ class OICodeGen {
     };
 
     FeatureSet features;
-    std::set<std::filesystem::path> containerConfigPaths;
+    std::vector<std::filesystem::path> containerConfigPaths;
     std::set<std::string> defaultHeaders;
     std::set<std::string> defaultNamespaces;
     std::vector<std::pair<std::string, std::string>> membersToStub;
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 1cb2a49..7d749c3 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -16,130 +16,148 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
 #include <glog/logging.h>
 
 #include <boost/core/demangle.hpp>
 #include <fstream>
 #include <iostream>
+#include <range/v3/action/transform.hpp>
+#include <range/v3/core.hpp>
+#include <range/v3/view/cache1.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/drop_while.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>  // TODO: remove
 #include <string_view>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
+class ConsumerContext;
 
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
-      templateArgsToSymbolsMap;
-
-  auto symbols = prog.find_all_symbols();
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
+class CreateTypeGraphConsumer;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
-    if (val.first.empty() || val.second.empty()) {
-      continue;
-    }
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  return strongToWeakSymbols;
-}
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
 
-std::unordered_map<std::string, drgn_qualified_type>
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
+ private:
+  ConsumerContext& ctx;
+};
 
-  std::unordered_map<std::string, drgn_qualified_type> out;
+class ConsumerContext {
+ public:
+  ConsumerContext(const std::vector<std::unique_ptr<ContainerInfo>>& cis) : containerInfos{cis} {}
 
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
-    std::string strongLinkageName;
-    {
-      const char* linkageNameCstr;
-      if (auto err = drgnplusplus::error(
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
-        // throw err;
+  type_graph::TypeGraph typeGraph;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
+  std::optional<bool> pic;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+}  // namespace
+
+int OIGenerator::generate(clang::tooling::CompilationDatabase& db,
+                          const std::vector<std::string>& sourcePaths) {
+  std::map<Feature, bool> featuresMap = {
+      {Feature::TypeGraph, true},
+      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},
+      {Feature::PackStructs, true},
+      {Feature::PruneTypeGraph, true},
+  };
+
+  OICodeGen::Config generatorConfig{};
+  OICompiler::Config compilerConfig{};
+
+  auto features = config::processConfigFiles(
+      configFilePaths, featuresMap, compilerConfig, generatorConfig);
+  if (!features) {
+    LOG(ERROR) << "failed to process config file";
+    return -1;
+  }
+  generatorConfig.features = *features;
+  compilerConfig.features = *features;
+
+  std::vector<std::unique_ptr<ContainerInfo>> containerInfos;
+  containerInfos.reserve(generatorConfig.containerConfigPaths.size());
+  try {
+    for (const auto& path : generatorConfig.containerConfigPaths) {
+      auto info = std::make_unique<ContainerInfo>(path);
+      if (info->requiredFeatures != (*features & info->requiredFeatures)) {
+        VLOG(1) << "Skipping container (feature conflict): " << info->typeName;
         continue;
       }
-      strongLinkageName = linkageNameCstr;
+      containerInfos.emplace_back(std::move(info));
     }
-
-    std::string weakLinkageName;
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return -1;
   }
 
-  return out;
-}
+  ConsumerContext ctx{containerInfos};
+  CreateTypeGraphActionFactory factory{ctx};
 
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
-                                      const OICompiler::Config& compilerConfig,
-                                      const drgn_qualified_type& type,
-                                      const std::string& linkageName,
-                                      SymbolService& symbols) {
-  CodeGen codegen{generatorConfig, symbols};
+  clang::tooling::ClangTool tool{db, sourcePaths};
+  if (auto ret = tool.run(&factory); ret != 0) {
+    return ret;
+  }
+
+  if (ctx.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one type to generate for but we can't currently "
+        "handle this case");
+
+  compilerConfig.usePIC = ctx.pic.value();
+  CodeGen codegen{generatorConfig};
+  for (auto&& ptr : containerInfos)
+    codegen.registerContainer(std::move(ptr));
+  codegen.transform(ctx.typeGraph);
 
   std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
+  codegen.generate(ctx.typeGraph, code, nullptr);
 
   std::string sourcePath = sourceFileDumpPath;
   if (sourceFileDumpPath.empty()) {
@@ -152,78 +170,110 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
   }
 
   OICompiler compiler{{}, compilerConfig};
-
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
-
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
 }
 
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
-  drgnplusplus::program prog;
+namespace {
 
-  {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(
-            drgn_program_load_debug_info(prog.get(),
-                                         std::data(objectPaths),
-                                         std::size(objectPaths),
-                                         false,
-                                         false))) {
-      LOG(ERROR) << "error loading debug info program: " << err;
-      throw err;
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx(ctx_) {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
     }
-  }
 
-  auto oilTypes = findOilTypesAndNames(prog);
-
-  std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
-      {Feature::PruneTypeGraph, true},
-  };
-
-  OICodeGen::Config generatorConfig{};
-  OICompiler::Config compilerConfig{};
-  compilerConfig.usePIC = pic;
-
-  auto features = config::processConfigFiles(
-      configFilePaths, featuresMap, compilerConfig, generatorConfig);
-  if (!features) {
-    LOG(ERROR) << "failed to process config file";
-    return -1;
-  }
-  generatorConfig.features = *features;
-  compilerConfig.features = *features;
-
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
+    auto introspectImpl =
+        std::move(oi_namespaces) |
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
+        ranges::views::transform([](auto* p) {
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
+        }) |
+        ranges::views::filter([](auto* td) {
+          return td != nullptr && td->getName() == "introspectImpl";
+        }) |
+        ranges::views::take(1) | ranges::to<std::vector>();
+    if (introspectImpl.empty()) {
+      LOG(WARNING)
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
+             "you compile with `OIL_AOT_COMPILATION=1`?";
+      return;
     }
-  }
 
-  size_t successes = oilTypes.size() - failures;
-  LOG(INFO) << "object introspection generation complete. " << successes
-            << " successes and " << failures << " failures.";
+    auto nameToClangTypeMap =
+        ranges::views::single(introspectImpl[0]) |
+        ranges::views::for_each(
+            [](auto* td) { return td->specializations(); }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
 
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
-    return -1;
+              assert(fd->getNumParams() == 1);
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, opts};
+
+    auto& Sema = *ctx.sema;
+    ctx.nameToTypeMap =
+        nameToClangTypeMap |
+        ranges::views::transform(
+            [&parser, &Context, &Sema](
+                auto& p) -> std::pair<std::string, type_graph::Type*> {
+              return {p.first, &parser.parse(Context, Sema, *p.second)};
+            }) |
+        ranges::to<std::unordered_map>();
+
+    for (const auto& [name, type] : ctx.nameToTypeMap)
+      ctx.typeGraph.addRoot(*type);
   }
-  return 0;
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  // Compile the output as position independent if any input is position
+  // independent
+  bool pic = CI.getCodeGenOpts().RelocationModel == llvm::Reloc::PIC_;
+  ctx.pic = ctx.pic.value_or(false) || pic;
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
 }
 
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+}  // namespace
 }  // namespace oi::detail
diff --git a/oi/OIGenerator.h b/oi/OIGenerator.h
index dd38384..fac947a 100644
--- a/oi/OIGenerator.h
+++ b/oi/OIGenerator.h
@@ -23,11 +23,19 @@
 #include "oi/OICodeGen.h"
 #include "oi/OICompiler.h"
 
+namespace clang::tooling {
+class CompilationDatabase;
+}
+
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate(clang::tooling::CompilationDatabase&,
+               const std::vector<std::string>&);
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -41,8 +49,8 @@ class OIGenerator {
   void setFailIfNothingGenerated(bool fail) {
     failIfNothingGenerated = fail;
   }
-  void setUsePIC(bool pic_) {
-    pic = pic_;
+  void setClangArgs(std::vector<std::string> args_) {
+    clangArgs = std::move(args_);
   }
 
  private:
@@ -50,20 +58,7 @@ class OIGenerator {
   std::vector<std::filesystem::path> configFilePaths;
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
-  bool pic = false;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
+  std::vector<std::string> clangArgs;
 };
 
 }  // namespace oi::detail
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 4e8f622..54aba34 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
   Flattener.cpp
@@ -25,3 +26,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
new file mode 100644
index 0000000..ee8f690
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -0,0 +1,401 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Basic/DiagnosticSema.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <iostream>  // TODO: maybe remove
+#include <stdexcept>  // TODO: remove
+#include <regex>
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (!requireCompleteType(*sema, ty))
+    return makeType<Incomplete>(ty, "incomplete (TODO naming)");
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(*ty.getAs<const clang::RecordType>());
+    case clang::Type::LValueReference:
+      return enumerateReference(*ty.getAs<const clang::LValueReferenceType>());
+    case clang::Type::Pointer:
+      return enumeratePointer(*ty.getAs<const clang::PointerType>());
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          *ty.getAs<const clang::SubstTemplateTypeParmType>());
+    case clang::Type::Builtin:
+      return enumeratePrimitive(*ty.getAs<const clang::BuiltinType>());
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(*ty.getAs<const clang::ElaboratedType>());
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          *ty.getAs<const clang::TemplateSpecializationType>());
+    case clang::Type::UnaryTransform:
+      return enumerateUnaryTransformType(llvm::cast<const clang::UnaryTransformType>(ty));
+
+    case clang::Type::Typedef:
+      return enumerateTypedef(*ty.getAs<const clang::TypedefType>());
+    case clang::Type::Using:
+      return enumerateUsing(*ty.getAs<const clang::UsingType>());
+
+    case clang::Type::ConstantArray:
+      return enumerateArray(*llvm::cast<const clang::ConstantArrayType>(&ty));
+    case clang::Type::Enum:
+      return enumerateEnum(*ty.getAs<const clang::EnumType>());
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Type& ClangTypeParser::enumerateUnaryTransformType(const clang::UnaryTransformType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Typedef& ClangTypeParser::enumerateUsing(const clang::UsingType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+  std::string name = ty.getFoundDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+
+  std::string name = ty.getDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string name = ty.getDecl()->getNameAsString();
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  std::map<int64_t, std::string> enumeratorMap;
+  if (options_.readEnumValues) {
+    for (const auto* enumerator : ty.getDecl()->enumerators()) {
+      enumeratorMap.emplace(
+          enumerator->getInitVal().getExtValue(),
+          enumerator->getNameAsString()
+      );
+    }
+  }
+
+  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+}
+
+Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
+  uint64_t len = ty.getSize().getLimitedValue();
+  auto& t = enumerateType(*ty.getElementType());
+  return makeType<Array>(ty, t, len);
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  if (auto* info = getContainerInfo(fqName)) {
+    auto& c = makeType<Container>(ty, *info, size, nullptr);
+    enumerateClassTemplateParams(ty, c.templateParams);
+    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)));
+    return c;
+  }
+
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+
+  params.reserve(list.size());
+  for (const auto& arg : list.asArray()) {
+    if (auto p = enumerateTemplateParam(arg))
+      params.emplace_back(std::move(p.value()));
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return TemplateParam{ttype, qualifiers};
+    }
+    case clang::TemplateArgument::Integral: {
+      auto& ty = enumerateType(*p.getIntegralType());
+      llvm::SmallString<32> val;
+      p.getAsIntegral().toString(val);
+      return TemplateParam{ty, std::string(val)};
+    }
+    case clang::TemplateArgument::Template: {
+      return enumerateTemplateTemplateParam(p.getAsTemplate());
+    }
+
+#define X(name) \
+case clang::TemplateArgument::name: \
+  throw std::logic_error("unsupported template argument kind: " #name);
+
+  X(Null)
+  X(Declaration)
+  X(NullPtr)
+  X(TemplateExpansion)
+  X(Expression)
+  X(Pack)
+#undef X
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateTemplateParam(const clang::TemplateName& tn) {
+  switch (tn.getKind()) {
+    case clang::TemplateName::Template:
+      return std::nullopt;
+
+#define X(name) \
+case clang::TemplateName::name: \
+  throw std::logic_error("unsupported template name kind: " #name);
+
+X(OverloadedTemplate)
+X(AssumedTemplate)
+X(QualifiedTemplate)
+X(DependentTemplate)
+X(SubstTemplateTemplateParm)
+X(SubstTemplateTemplateParmPack)
+X(UsingTemplate)
+#undef X
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function references
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Float:
+      return makeType<Primitive>(ty, Primitive::Kind::Float32);
+    case clang::BuiltinType::Double:
+    case clang::BuiltinType::LongDouble:
+      return makeType<Primitive>(ty, Primitive::Kind::Float64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      ty.dump();
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+ContainerInfo* ClangTypeParser::getContainerInfo(const std::string& fqName) const {
+  for (const auto& containerInfo : containers_) {
+    if (std::regex_search(fqName, containerInfo->matcher)) {
+      return containerInfo.get();
+    }
+  }
+  return nullptr;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
+    return true; // treat as complete
+
+  return !sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      clang::diag::err_type_unsupported);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
new file mode 100644
index 0000000..9ab36ed
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.h
@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ASTContext;
+class BuiltinType;
+class ConstantArrayType;
+class ElaboratedType;
+class EnumType;
+class LValueReferenceType;
+class PointerType;
+class RecordType;
+class Sema;
+class SubstTemplateTypeParmType;
+class TemplateArgument;
+class TemplateName;
+class TemplateSpecializationType;
+class Type;
+class TypedefType;
+class UnaryTransformType;
+class UsingType;
+}  // namespace clang
+
+struct ContainerInfo;
+
+namespace oi::detail::type_graph {
+
+class Array;
+class Class;
+class Enum;
+class Member;
+class Primitive;
+class Reference;
+class Type;
+class TypeGraph;
+class Typedef;
+struct TemplateParam;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+  bool readEnumValues = false;
+};
+
+/*
+ * ClangTypeParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given drgn_type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph, const std::vector<std::unique_ptr<ContainerInfo>>& containers, ClangTypeParserOptions options)
+      : typeGraph_{typeGraph}, containers_{containers}, options_{options} {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containers_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Type& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+  Typedef& enumerateTypedef(const clang::TypedefType&);
+  Typedef& enumerateUsing(const clang::UsingType&);
+  Type& enumerateUnaryTransformType(const clang::UnaryTransformType&);
+
+  Array& enumerateArray(const clang::ConstantArrayType&);
+  Enum& enumerateEnum(const clang::EnumType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  std::optional<TemplateParam> enumerateTemplateParam(const clang::TemplateArgument&);
+  std::optional<TemplateParam> enumerateTemplateTemplateParam(const clang::TemplateName&);
+
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  ContainerInfo* getContainerInfo(const std::string& fqName) const;
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph
diff --git a/test/integration/template_template_param.toml b/test/integration/template_template_param.toml
new file mode 100644
index 0000000..c64590d
--- /dev/null
+++ b/test/integration/template_template_param.toml
@@ -0,0 +1,16 @@
+includes = ["vector", "utility", "string"]
+
+definitions = '''
+template <typename K, typename V, template <typename, typename> typename Pair>
+struct bad_map {
+  std::vector<Pair<K, K>> keys;
+  std::vector<Pair<V, V>> values;
+};
+'''
+
+[cases]
+  [cases.int_int_empty]
+    param_types = ["const bad_map<int, std::string, std::pair>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index d1e345d..ec3810b 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -14,131 +14,80 @@
  * limitations under the License.
  */
 
+#include <clang/Tooling/CommonOptionsParser.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include <cstdlib>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/action/drop_while.hpp>
+#include <range/v3/algorithm/find.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/all.hpp>
+#include <range/v3/view/transform.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
 
-constexpr static OIOpts opts{
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
+static llvm::cl::OptionCategory OilgenCategory("oilgen options");
 
-void usage() {
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
-  std::cerr << opts;
+static llvm::cl::list<std::string> ConfigFiles("config-file",
+                                               llvm::cl::desc(R"(TODO HELP)"),
+                                               llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<int> DebugLevel("debug-level",
+                                     llvm::cl::desc(R"(TODO HELP)"),
+                                     llvm::cl::init(-1),
+                                     llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> DumpJit("dump-jit",
+                                          llvm::cl::desc(R"(TODO HELP)"),
+                                          llvm::cl::init("jit.cpp"),
+                                          llvm::cl::cat(OilgenCategory));
 
-  std::cerr << std::endl
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
-
-int main(int argc, char* argv[]) {
+int main(int argc, const char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
-  std::vector<fs::path> configFilePaths;
-  fs::path sourceFileDumpPath = "";
-  bool exitCode = false;
-  bool pic = false;
-
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
+  auto expectedParser =
+      clang::tooling::CommonOptionsParser::create(argc, argv, OilgenCategory);
+  if (!expectedParser) {
+    llvm::errs() << expectedParser.takeError();
+    return -1;
   }
+  clang::tooling::CommonOptionsParser& options = expectedParser.get();
+  auto& compilations = options.getCompilations();
 
-  if (optind >= argc) {
-    usage();
-    return EXIT_FAILURE;
-  }
-  fs::path primaryObject = argv[optind];
-
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
+  if (DebugLevel.getNumOccurrences()) {
+    google::LogToStderr();
+    google::SetStderrLogging(google::INFO);
+    google::SetVLOGLevel("*", DebugLevel);
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
+    gflags::SetCommandLineOption("minloglevel", "0");
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
-  oigen.setConfigFilePaths(std::move(configFilePaths));
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
-  oigen.setFailIfNothingGenerated(exitCode);
-  oigen.setUsePIC(pic);
+  oigen.setConfigFilePaths(ConfigFiles |
+                           ranges::views::transform([](const auto& p) {
+                             return std::filesystem::path(p);
+                           }) |
+                           ranges::to<std::vector>());
+  if (DumpJit.getNumOccurrences())
+    oigen.setSourceFileDumpPath(DumpJit.getValue());
 
-  SymbolService symbols(primaryObject);
+  oigen.setOutputPath("a.o");  // TODO: make this an opt
 
-  return oigen.generate(primaryObject, symbols);
+  // TODO: can we extract this from the original arguments?
+  // oigen.setUsePIC(args["pic"].as<bool>());
+
+  oigen.setFailIfNothingGenerated(true);
+  // compilations.getAllFiles();
+  // return oigen.generate(compilations, {"/data/users/jakehillion/fbsource/fbcode/object-introspection/oil/examples/compile-time/VectorOfStrings-Introspect.cpp"});
+  return oigen.generate(compilations, options.getSourcePathList());
 }
diff --git a/types/string_type.toml b/types/string_type.toml
index fa4a2ae..0d2ab0d 100644
--- a/types/string_type.toml
+++ b/types/string_type.toml
@@ -35,3 +35,34 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
     );
 }
 """
+
+traversal_func = """
+    bool sso = ((uintptr_t)container.data() <
+                (uintptr_t)(&container + sizeof(std::__cxx11::basic_string<T0>))) &&
+               ((uintptr_t)container.data() >= (uintptr_t)&container);
+
+    return returnArg.write(container.capacity())
+        .write(sso)
+        .write(container.size());
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+uint64_t capacity = std::get<ParsedData::VarInt>(d.val).value;
+el.container_stats.emplace(result::Element::ContainerStats { .capacity = capacity });
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+bool sso = std::get<ParsedData::VarInt>(d.val).value;
+if (!sso)
+  el.exclusive_size += el.container_stats->capacity * sizeof(T0);
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats->length = std::get<ParsedData::VarInt>(d.val).value;
+"""

From 9daca25cf6ba212835268c4decda154a1a444637 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 13 Dec 2023 16:11:53 +0000
Subject: [PATCH 053/188] oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD
---
 CMakeLists.txt                                |  13 +-
 oi/CodeGen.cpp                                |  30 +-
 oi/CodeGen.h                                  |   8 +-
 oi/Config.cpp                                 |   9 +-
 oi/ContainerInfo.cpp                          |   2 +-
 oi/OICodeGen.h                                |   2 +-
 oi/OIGenerator.cpp                            | 364 ++++++++-------
 oi/OIGenerator.h                              |  29 +-
 oi/type_graph/CMakeLists.txt                  |   2 +
 oi/type_graph/ClangTypeParser.cpp             | 416 ++++++++++++++++++
 oi/type_graph/ClangTypeParser.h               | 131 ++++++
 test/integration/template_template_param.toml |  16 +
 tools/OILGen.cpp                              | 148 +++----
 types/string_type.toml                        |  31 ++
 14 files changed, 911 insertions(+), 290 deletions(-)
 create mode 100644 oi/type_graph/ClangTypeParser.cpp
 create mode 100644 oi/type_graph/ClangTypeParser.h
 create mode 100644 test/integration/template_template_param.toml

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f9ee7..eb18f4a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -104,6 +104,15 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(range-v3)
 
+### cxxopts
+FetchContent_Declare(
+  cxxopts
+  GIT_REPOSITORY https://github.com/jarro2783/cxxopts.git
+  GIT_TAG        eb787304d67ec22f7c3a184ee8b4c481d04357fd # v3.1.1
+  GIT_PROGRESS   TRUE
+)
+FetchContent_MakeAvailable(cxxopts)
+
 set_project_warnings()
 
 if (ASAN)
@@ -289,7 +298,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -363,8 +372,10 @@ add_executable(oilgen
   oi/OIGenerator.cpp
 )
 target_link_libraries(oilgen
+  cxxopts
   drgn_utils
   oicore
+  clangTooling
 )
 
 ### Object Introspection cache Printer (OIP)
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 5a4b9eb..e840104 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -537,7 +537,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1104,15 +1104,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
 }
 
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
-  try {
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
+  if (!registerContainers())
     return false;
-  }
 
   TypeGraph typeGraph;
   try {
@@ -1127,6 +1120,20 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
   return true;
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    size_t i = 0;
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1167,7 +1174,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, containerInfos_, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(Flattener::createPass());
@@ -1282,7 +1289,8 @@ void CodeGen::generate(
   code += "\nusing __ROOT_TYPE__ = " + rootType.name() + ";\n";
   code += "} // namespace\n} // namespace OIInternal\n";
 
-  const auto typeName = SymbolService::getTypeName(drgnType);
+  // const auto typeName = SymbolService::getTypeName(drgnType);
+  const std::string typeName{"typeName"};
   if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
   } else {
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 186ec34..fb0ec20 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -40,8 +40,11 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config) : config_(config) {
+    assert(!config.features[Feature::PolymorphicInheritance]);
+  }
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   /*
@@ -53,6 +56,7 @@ class CodeGen {
                        std::string linkageName,
                        std::string& code);
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -66,7 +70,7 @@ class CodeGen {
 
  private:
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
diff --git a/oi/Config.cpp b/oi/Config.cpp
index 157b279..66798bb 100644
--- a/oi/Config.cpp
+++ b/oi/Config.cpp
@@ -22,6 +22,8 @@
 #include <boost/property_tree/ini_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 #include <filesystem>
+#include <range/v3/action/sort.hpp>
+#include <range/v3/action/unique.hpp>
 
 #include "oi/support/Toml.h"
 
@@ -54,6 +56,9 @@ std::optional<FeatureSet> processConfigFiles(
     enables |= *fs;
   }
 
+  ranges::actions::sort(generatorConfig.containerConfigPaths);
+  ranges::actions::unique(generatorConfig.containerConfigPaths);
+
   // Override anything from the config files with command line options
   for (auto [k, v] : featureMap) {
     enables[k] = v;
@@ -108,8 +113,8 @@ std::optional<FeatureSet> processConfigFile(
           if the right path is absolute, else append the right path to the
           left path.
           */
-          generatorConfig.containerConfigPaths.emplace(configDirectory /
-                                                       el.get());
+          generatorConfig.containerConfigPaths.emplace_back(configDirectory /
+                                                            el.get());
         }
       });
     }
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 99a7dc1..dbe10c6 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -202,7 +202,7 @@ std::regex getMatcher(const std::string& typeName) {
 ContainerInfo::ContainerInfo(const fs::path& path) {
   toml::table container;
   try {
-    container = toml::parse_file(std::string(path));
+    container = toml::parse_file(path.string());
   } catch (const toml::parse_error& err) {
     // Convert into a ContainerInfoError, just to avoid having to include
     // the huge TOML++ header in the caller's file. Use toml::parse_error's
diff --git a/oi/OICodeGen.h b/oi/OICodeGen.h
index a01548f..7226a4e 100644
--- a/oi/OICodeGen.h
+++ b/oi/OICodeGen.h
@@ -66,7 +66,7 @@ class OICodeGen {
     };
 
     FeatureSet features;
-    std::set<std::filesystem::path> containerConfigPaths;
+    std::vector<std::filesystem::path> containerConfigPaths;
     std::set<std::string> defaultHeaders;
     std::set<std::string> defaultNamespaces;
     std::vector<std::pair<std::string, std::string>> membersToStub;
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 1cb2a49..93054bc 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -16,130 +16,150 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
 #include <glog/logging.h>
 
 #include <boost/core/demangle.hpp>
 #include <fstream>
 #include <iostream>
+#include <range/v3/action/transform.hpp>
+#include <range/v3/core.hpp>
+#include <range/v3/view/cache1.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/drop_while.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>  // TODO: remove
 #include <string_view>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
+class ConsumerContext;
 
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
-      templateArgsToSymbolsMap;
-
-  auto symbols = prog.find_all_symbols();
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
+class CreateTypeGraphConsumer;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
-    if (val.first.empty() || val.second.empty()) {
-      continue;
-    }
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  return strongToWeakSymbols;
-}
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
 
-std::unordered_map<std::string, drgn_qualified_type>
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
+ private:
+  ConsumerContext& ctx;
+};
 
-  std::unordered_map<std::string, drgn_qualified_type> out;
+class ConsumerContext {
+ public:
+  ConsumerContext(const std::vector<std::unique_ptr<ContainerInfo>>& cis)
+      : containerInfos{cis} {
+  }
 
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
-    std::string strongLinkageName;
-    {
-      const char* linkageNameCstr;
-      if (auto err = drgnplusplus::error(
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
-        // throw err;
+  type_graph::TypeGraph typeGraph;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
+  std::optional<bool> pic;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+}  // namespace
+
+int OIGenerator::generate(clang::tooling::CompilationDatabase& db,
+                          const std::vector<std::string>& sourcePaths) {
+  std::map<Feature, bool> featuresMap = {
+      {Feature::TypeGraph, true},
+      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},
+      {Feature::PackStructs, true},
+      {Feature::PruneTypeGraph, true},
+  };
+
+  OICodeGen::Config generatorConfig{};
+  OICompiler::Config compilerConfig{};
+
+  auto features = config::processConfigFiles(
+      configFilePaths, featuresMap, compilerConfig, generatorConfig);
+  if (!features) {
+    LOG(ERROR) << "failed to process config file";
+    return -1;
+  }
+  generatorConfig.features = *features;
+  compilerConfig.features = *features;
+
+  std::vector<std::unique_ptr<ContainerInfo>> containerInfos;
+  containerInfos.reserve(generatorConfig.containerConfigPaths.size());
+  try {
+    for (const auto& path : generatorConfig.containerConfigPaths) {
+      auto info = std::make_unique<ContainerInfo>(path);
+      if (info->requiredFeatures != (*features & info->requiredFeatures)) {
+        VLOG(1) << "Skipping container (feature conflict): " << info->typeName;
         continue;
       }
-      strongLinkageName = linkageNameCstr;
+      containerInfos.emplace_back(std::move(info));
     }
-
-    std::string weakLinkageName;
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return -1;
   }
 
-  return out;
-}
+  ConsumerContext ctx{containerInfos};
+  CreateTypeGraphActionFactory factory{ctx};
 
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
-                                      const OICompiler::Config& compilerConfig,
-                                      const drgn_qualified_type& type,
-                                      const std::string& linkageName,
-                                      SymbolService& symbols) {
-  CodeGen codegen{generatorConfig, symbols};
+  clang::tooling::ClangTool tool{db, sourcePaths};
+  if (auto ret = tool.run(&factory); ret != 0) {
+    return ret;
+  }
+
+  if (ctx.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one type to generate for but we can't currently "
+        "handle this case");
+
+  compilerConfig.usePIC = ctx.pic.value();
+  CodeGen codegen{generatorConfig};
+  for (auto&& ptr : containerInfos)
+    codegen.registerContainer(std::move(ptr));
+  codegen.transform(ctx.typeGraph);
 
   std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
+  codegen.generate(ctx.typeGraph, code, nullptr);
 
   std::string sourcePath = sourceFileDumpPath;
   if (sourceFileDumpPath.empty()) {
@@ -152,78 +172,110 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
   }
 
   OICompiler compiler{{}, compilerConfig};
-
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
-
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
 }
 
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
-  drgnplusplus::program prog;
+namespace {
 
-  {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(
-            drgn_program_load_debug_info(prog.get(),
-                                         std::data(objectPaths),
-                                         std::size(objectPaths),
-                                         false,
-                                         false))) {
-      LOG(ERROR) << "error loading debug info program: " << err;
-      throw err;
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx(ctx_) {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
     }
-  }
 
-  auto oilTypes = findOilTypesAndNames(prog);
-
-  std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
-      {Feature::PruneTypeGraph, true},
-  };
-
-  OICodeGen::Config generatorConfig{};
-  OICompiler::Config compilerConfig{};
-  compilerConfig.usePIC = pic;
-
-  auto features = config::processConfigFiles(
-      configFilePaths, featuresMap, compilerConfig, generatorConfig);
-  if (!features) {
-    LOG(ERROR) << "failed to process config file";
-    return -1;
-  }
-  generatorConfig.features = *features;
-  compilerConfig.features = *features;
-
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
+    auto introspectImpl =
+        std::move(oi_namespaces) |
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
+        ranges::views::transform([](auto* p) {
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
+        }) |
+        ranges::views::filter([](auto* td) {
+          return td != nullptr && td->getName() == "introspectImpl";
+        }) |
+        ranges::views::take(1) | ranges::to<std::vector>();
+    if (introspectImpl.empty()) {
+      LOG(WARNING)
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
+             "you compile with `OIL_AOT_COMPILATION=1`?";
+      return;
     }
-  }
 
-  size_t successes = oilTypes.size() - failures;
-  LOG(INFO) << "object introspection generation complete. " << successes
-            << " successes and " << failures << " failures.";
+    auto nameToClangTypeMap =
+        ranges::views::single(introspectImpl[0]) |
+        ranges::views::for_each(
+            [](auto* td) { return td->specializations(); }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
 
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
-    return -1;
+              assert(fd->getNumParams() == 1);
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, opts};
+
+    auto& Sema = *ctx.sema;
+    ctx.nameToTypeMap =
+        nameToClangTypeMap |
+        ranges::views::transform(
+            [&parser, &Context, &Sema](
+                auto& p) -> std::pair<std::string, type_graph::Type*> {
+              return {p.first, &parser.parse(Context, Sema, *p.second)};
+            }) |
+        ranges::to<std::unordered_map>();
+
+    for (const auto& [name, type] : ctx.nameToTypeMap)
+      ctx.typeGraph.addRoot(*type);
   }
-  return 0;
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  // Compile the output as position independent if any input is position
+  // independent
+  bool pic = CI.getCodeGenOpts().RelocationModel == llvm::Reloc::PIC_;
+  ctx.pic = ctx.pic.value_or(false) || pic;
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
 }
 
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+}  // namespace
 }  // namespace oi::detail
diff --git a/oi/OIGenerator.h b/oi/OIGenerator.h
index dd38384..fac947a 100644
--- a/oi/OIGenerator.h
+++ b/oi/OIGenerator.h
@@ -23,11 +23,19 @@
 #include "oi/OICodeGen.h"
 #include "oi/OICompiler.h"
 
+namespace clang::tooling {
+class CompilationDatabase;
+}
+
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate(clang::tooling::CompilationDatabase&,
+               const std::vector<std::string>&);
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -41,8 +49,8 @@ class OIGenerator {
   void setFailIfNothingGenerated(bool fail) {
     failIfNothingGenerated = fail;
   }
-  void setUsePIC(bool pic_) {
-    pic = pic_;
+  void setClangArgs(std::vector<std::string> args_) {
+    clangArgs = std::move(args_);
   }
 
  private:
@@ -50,20 +58,7 @@ class OIGenerator {
   std::vector<std::filesystem::path> configFilePaths;
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
-  bool pic = false;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
+  std::vector<std::string> clangArgs;
 };
 
 }  // namespace oi::detail
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 4e8f622..54aba34 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
   Flattener.cpp
@@ -25,3 +26,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
new file mode 100644
index 0000000..67f7bfa
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -0,0 +1,416 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Basic/DiagnosticSema.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <iostream>  // TODO: maybe remove
+#include <regex>
+#include <stdexcept>  // TODO: remove
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (VLOG_IS_ON(3)) {
+    std::string fqName = clang::TypeName::getFullyQualifiedName(
+        clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+    VLOG(3) << std::string(depth_ * 2, ' ') << fqName;
+  }
+
+  if (!requireCompleteType(*sema, ty))
+    return makeType<Incomplete>(ty, "incomplete (TODO naming)");
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(llvm::cast<const clang::RecordType>(ty));
+    case clang::Type::LValueReference:
+      return enumerateReference(
+          llvm::cast<const clang::LValueReferenceType>(ty));
+    case clang::Type::Pointer:
+      return enumeratePointer(llvm::cast<const clang::PointerType>(ty));
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          llvm::cast<const clang::SubstTemplateTypeParmType>(ty));
+    case clang::Type::Builtin:
+      return enumeratePrimitive(llvm::cast<const clang::BuiltinType>(ty));
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(
+          llvm::cast<const clang::ElaboratedType>(ty));
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          llvm::cast<const clang::TemplateSpecializationType>(ty));
+    case clang::Type::UnaryTransform:
+      return enumerateUnaryTransformType(
+          llvm::cast<const clang::UnaryTransformType>(ty));
+    case clang::Type::Decltype:
+      return enumerateDecltypeType(llvm::cast<const clang::DecltypeType>(ty));
+    case clang::Type::Typedef:
+      return enumerateTypedef(llvm::cast<const clang::TypedefType>(ty));
+    case clang::Type::Using:
+      return enumerateUsing(llvm::cast<const clang::UsingType>(ty));
+    case clang::Type::ConstantArray:
+      return enumerateArray(llvm::cast<const clang::ConstantArrayType>(ty));
+    case clang::Type::Enum:
+      return enumerateEnum(llvm::cast<const clang::EnumType>(ty));
+
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Type& ClangTypeParser::enumerateDecltypeType(const clang::DecltypeType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Type& ClangTypeParser::enumerateUnaryTransformType(
+    const clang::UnaryTransformType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Typedef& ClangTypeParser::enumerateUsing(const clang::UsingType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+  std::string name = ty.getFoundDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+
+  std::string name = ty.getDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string name = ty.getDecl()->getNameAsString();
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  std::map<int64_t, std::string> enumeratorMap;
+  if (options_.readEnumValues) {
+    for (const auto* enumerator : ty.getDecl()->enumerators()) {
+      enumeratorMap.emplace(enumerator->getInitVal().getExtValue(),
+                            enumerator->getNameAsString());
+    }
+  }
+
+  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+}
+
+Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
+  uint64_t len = ty.getSize().getLimitedValue();
+  auto& t = enumerateType(*ty.getElementType());
+  return makeType<Array>(ty, t, len);
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  if (auto* info = getContainerInfo(fqName)) {
+    auto& c = makeType<Container>(ty, *info, size, nullptr);
+    enumerateClassTemplateParams(ty, c.templateParams);
+    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)));
+    return c;
+  }
+
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+
+  params.reserve(list.size());
+  for (const auto& arg : list.asArray()) {
+    if (auto p = enumerateTemplateParam(arg))
+      params.emplace_back(std::move(p.value()));
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return TemplateParam{ttype, qualifiers};
+    }
+    case clang::TemplateArgument::Integral: {
+      auto& ty = enumerateType(*p.getIntegralType());
+      llvm::SmallString<32> val;
+      p.getAsIntegral().toString(val);
+      return TemplateParam{ty, std::string(val)};
+    }
+    case clang::TemplateArgument::Template: {
+      return enumerateTemplateTemplateParam(p.getAsTemplate());
+    }
+
+#define X(name)                       \
+  case clang::TemplateArgument::name: \
+    throw std::logic_error("unsupported template argument kind: " #name);
+
+      X(Null)
+      X(Declaration)
+      X(NullPtr)
+      X(TemplateExpansion)
+      X(Expression)
+      X(Pack)
+#undef X
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateTemplateParam(
+    const clang::TemplateName& tn) {
+  switch (tn.getKind()) {
+    case clang::TemplateName::Template:
+      return std::nullopt;
+
+#define X(name)                   \
+  case clang::TemplateName::name: \
+    throw std::logic_error("unsupported template name kind: " #name);
+
+      X(OverloadedTemplate)
+      X(AssumedTemplate)
+      X(QualifiedTemplate)
+      X(DependentTemplate)
+      X(SubstTemplateTemplateParm)
+      X(SubstTemplateTemplateParmPack)
+      X(UsingTemplate)
+#undef X
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function references
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Float:
+      return makeType<Primitive>(ty, Primitive::Kind::Float32);
+    case clang::BuiltinType::Double:
+    case clang::BuiltinType::LongDouble:
+      return makeType<Primitive>(ty, Primitive::Kind::Float64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+ContainerInfo* ClangTypeParser::getContainerInfo(
+    const std::string& fqName) const {
+  for (const auto& containerInfo : containers_) {
+    if (std::regex_search(fqName, containerInfo->matcher)) {
+      return containerInfo.get();
+    }
+  }
+  return nullptr;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
+    return true;  // treat as complete
+
+  // TODO: This is a terrible warning.
+  return !sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      clang::diag::warn_nsconsumed_attribute_mismatch);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
new file mode 100644
index 0000000..10eea87
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.h
@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ASTContext;
+class BuiltinType;
+class ConstantArrayType;
+class DecltypeType;
+class ElaboratedType;
+class EnumType;
+class LValueReferenceType;
+class PointerType;
+class RecordType;
+class Sema;
+class SubstTemplateTypeParmType;
+class TemplateArgument;
+class TemplateName;
+class TemplateSpecializationType;
+class Type;
+class TypedefType;
+class UnaryTransformType;
+class UsingType;
+}  // namespace clang
+
+struct ContainerInfo;
+
+namespace oi::detail::type_graph {
+
+class Array;
+class Class;
+class Enum;
+class Member;
+class Primitive;
+class Reference;
+class Type;
+class TypeGraph;
+class Typedef;
+struct TemplateParam;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+  bool readEnumValues = false;
+};
+
+/*
+ * ClangTypeParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given drgn_type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph,
+                  const std::vector<std::unique_ptr<ContainerInfo>>& containers,
+                  ClangTypeParserOptions options)
+      : typeGraph_{typeGraph}, containers_{containers}, options_{options} {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containers_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Type& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+  Typedef& enumerateTypedef(const clang::TypedefType&);
+  Typedef& enumerateUsing(const clang::UsingType&);
+  Type& enumerateUnaryTransformType(const clang::UnaryTransformType&);
+  Type& enumerateDecltypeType(const clang::DecltypeType&);
+
+  Array& enumerateArray(const clang::ConstantArrayType&);
+  Enum& enumerateEnum(const clang::EnumType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  std::optional<TemplateParam> enumerateTemplateParam(
+      const clang::TemplateArgument&);
+  std::optional<TemplateParam> enumerateTemplateTemplateParam(
+      const clang::TemplateName&);
+
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  ContainerInfo* getContainerInfo(const std::string& fqName) const;
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph
diff --git a/test/integration/template_template_param.toml b/test/integration/template_template_param.toml
new file mode 100644
index 0000000..c64590d
--- /dev/null
+++ b/test/integration/template_template_param.toml
@@ -0,0 +1,16 @@
+includes = ["vector", "utility", "string"]
+
+definitions = '''
+template <typename K, typename V, template <typename, typename> typename Pair>
+struct bad_map {
+  std::vector<Pair<K, K>> keys;
+  std::vector<Pair<V, V>> values;
+};
+'''
+
+[cases]
+  [cases.int_int_empty]
+    param_types = ["const bad_map<int, std::string, std::pair>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index d1e345d..795389e 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -14,131 +14,81 @@
  * limitations under the License.
  */
 
+#include <clang/Tooling/CommonOptionsParser.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include <cstdlib>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/action/drop_while.hpp>
+#include <range/v3/algorithm/find.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/all.hpp>
+#include <range/v3/view/transform.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
 
-constexpr static OIOpts opts{
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
+static llvm::cl::OptionCategory OilgenCategory("oilgen options");
 
-void usage() {
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
-  std::cerr << opts;
+static llvm::cl::list<std::string> ConfigFiles("config-file",
+                                               llvm::cl::desc(R"(TODO HELP)"),
+                                               llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<int> DebugLevel("debug-level",
+                                     llvm::cl::desc(R"(TODO HELP)"),
+                                     llvm::cl::init(-1),
+                                     llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> DumpJit("dump-jit",
+                                          llvm::cl::desc(R"(TODO HELP)"),
+                                          llvm::cl::init("jit.cpp"),
+                                          llvm::cl::cat(OilgenCategory));
 
-  std::cerr << std::endl
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
-
-int main(int argc, char* argv[]) {
+int main(int argc, const char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
-  std::vector<fs::path> configFilePaths;
-  fs::path sourceFileDumpPath = "";
-  bool exitCode = false;
-  bool pic = false;
-
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
+  auto expectedParser =
+      clang::tooling::CommonOptionsParser::create(argc, argv, OilgenCategory);
+  if (!expectedParser) {
+    llvm::errs() << expectedParser.takeError();
+    return -1;
   }
+  clang::tooling::CommonOptionsParser& options = expectedParser.get();
+  auto& compilations = options.getCompilations();
 
-  if (optind >= argc) {
-    usage();
-    return EXIT_FAILURE;
-  }
-  fs::path primaryObject = argv[optind];
-
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
+  if (DebugLevel.getNumOccurrences()) {
+    google::LogToStderr();
+    google::SetStderrLogging(google::INFO);
+    google::SetVLOGLevel("*", DebugLevel);
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
+    gflags::SetCommandLineOption("minloglevel", "0");
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
-  oigen.setConfigFilePaths(std::move(configFilePaths));
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
-  oigen.setFailIfNothingGenerated(exitCode);
-  oigen.setUsePIC(pic);
+  oigen.setConfigFilePaths(ConfigFiles |
+                           ranges::views::transform([](const auto& p) {
+                             return std::filesystem::path(p);
+                           }) |
+                           ranges::to<std::vector>());
+  if (DumpJit.getNumOccurrences())
+    oigen.setSourceFileDumpPath(DumpJit.getValue());
 
-  SymbolService symbols(primaryObject);
+  oigen.setOutputPath("a.o");  // TODO: make this an opt
 
-  return oigen.generate(primaryObject, symbols);
+  // TODO: can we extract this from the original arguments?
+  // oigen.setUsePIC(args["pic"].as<bool>());
+
+  oigen.setFailIfNothingGenerated(true);
+  // compilations.getAllFiles();
+  // return oigen.generate(compilations,
+  // {"/data/users/jakehillion/fbsource/fbcode/object-introspection/oil/examples/compile-time/VectorOfStrings-Introspect.cpp"});
+  return oigen.generate(compilations, options.getSourcePathList());
 }
diff --git a/types/string_type.toml b/types/string_type.toml
index fa4a2ae..0d2ab0d 100644
--- a/types/string_type.toml
+++ b/types/string_type.toml
@@ -35,3 +35,34 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
     );
 }
 """
+
+traversal_func = """
+    bool sso = ((uintptr_t)container.data() <
+                (uintptr_t)(&container + sizeof(std::__cxx11::basic_string<T0>))) &&
+               ((uintptr_t)container.data() >= (uintptr_t)&container);
+
+    return returnArg.write(container.capacity())
+        .write(sso)
+        .write(container.size());
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+uint64_t capacity = std::get<ParsedData::VarInt>(d.val).value;
+el.container_stats.emplace(result::Element::ContainerStats { .capacity = capacity });
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+bool sso = std::get<ParsedData::VarInt>(d.val).value;
+if (!sso)
+  el.exclusive_size += el.container_stats->capacity * sizeof(T0);
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats->length = std::get<ParsedData::VarInt>(d.val).value;
+"""

From 6f5091c95354e6c0a04cc33945157971894158ef Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 14 Dec 2023 13:34:32 +0000
Subject: [PATCH 054/188] type_graph: Add reference type

Our existing drgn parser emits pointers, regardless of whether the type in the
DWARF is really a reference. Newer parsers like the LLDB/Clang parsers can
differentiate between a pointer and a reference. We should do this as
references can be safely followed according to the language rules with
`ChaseRawPointers` disabled, while pointers cannot.

Test Plan:
- TBD
---
 oi/type_graph/NameGen.cpp               |  8 ++++
 oi/type_graph/NameGen.h                 |  1 +
 oi/type_graph/Printer.cpp               | 11 +++++
 oi/type_graph/Printer.h                 |  1 +
 oi/type_graph/RemoveTopLevelPointer.cpp |  4 ++
 oi/type_graph/RemoveTopLevelPointer.h   |  1 +
 oi/type_graph/Types.h                   | 56 +++++++++++++++++++++++++
 oi/type_graph/Visitor.h                 |  7 ++++
 8 files changed, 89 insertions(+)

diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index c792766..5213b3d 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -194,6 +194,14 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& p) {
+  RecursiveVisitor::visit(p);
+  p.regenerateName();
+  std::string inputName{p.pointeeType().inputName()};
+  inputName += '*';
+  p.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..e6437d0 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index e7d70eb..5913977 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -147,6 +147,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& p) {
+  if (prefix(p))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = p.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(p.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index a66581e..d7d60f3 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& p) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;
diff --git a/oi/type_graph/RemoveTopLevelPointer.cpp b/oi/type_graph/RemoveTopLevelPointer.cpp
index 9ecaaaa..e1909a7 100644
--- a/oi/type_graph/RemoveTopLevelPointer.cpp
+++ b/oi/type_graph/RemoveTopLevelPointer.cpp
@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& p) {
+  topLevelType_ = &p.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/RemoveTopLevelPointer.h b/oi/type_graph/RemoveTopLevelPointer.h
index 611429f..6b70545 100644
--- a/oi/type_graph/RemoveTopLevelPointer.h
+++ b/oi/type_graph/RemoveTopLevelPointer.h
@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
 
  private:
   Type* topLevelType_ = nullptr;
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f503886..8aba760 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -715,6 +716,61 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    name_ = pointeeType_.get().name() + "&";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index 7e4395e..b2fb7a2 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -121,6 +121,9 @@ class RecursiveVisitor : public Visitor {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -182,6 +185,10 @@ class RecursiveMutator : public Mutator {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }

From 8b94447145312436513f1aab2cd8345bce7f9810 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jake@hillion.co.uk>
Date: Thu, 14 Dec 2023 15:18:24 +0000
Subject: [PATCH 055/188] ci: move formatting checks to nix

---
 .circleci/config.yml | 31 ++++-------------
 flake.lock           | 82 ++++++++++++++++++++++++++++++++++++++++++++
 flake.nix            | 30 ++++++++++++++++
 3 files changed, 119 insertions(+), 24 deletions(-)
 create mode 100644 flake.lock
 create mode 100644 flake.nix

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 7e805ad..80439f8 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -51,6 +51,10 @@ workflows:
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_a"
 
 executors:
+  nix-docker:
+    docker:
+      - image: nixos/nix:latest
+    resource_class: small
   ubuntu-docker:
     docker:
       - image: ubuntu:jammy
@@ -62,33 +66,12 @@ executors:
 
 jobs:
   lint:
-    executor: ubuntu-docker
+    executor: nix-docker
     steps:
-      - run:
-          name: Install dependencies
-          command: |
-            apt-get update
-            apt-get install -y \
-              clang-format \
-              git \
-              python3-pip
-            # click broke semver with 8.1.0, causing issues for black
-            pip install click==8.0.0 black isort
-          environment:
-            DEBIAN_FRONTEND: noninteractive
       - checkout
       - run:
-          name: clang-format
-          command: |
-            git ls-files '*.cpp' '*.h' | xargs clang-format --fallback-style=Google -i
-            git ls-files '*.py' | xargs black
-            git ls-files '*.py' | xargs isort
-            git diff --exit-code
-      - run:
-          name: python linting
-          command: |
-            black --check --diff test/
-            isort --check --diff test/
+          name: Flake check
+          command: nix --experimental-features 'nix-command flakes' flake check
 
   build:
     # TODO this job could be run in Docker
diff --git a/flake.lock b/flake.lock
new file mode 100644
index 0000000..927748e
--- /dev/null
+++ b/flake.lock
@@ -0,0 +1,82 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1694529238,
+        "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "ff7b65b44d01cf9ba6a71320833626af21126384",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1696375444,
+        "narHash": "sha256-Sv0ICt/pXfpnFhTGYTsX6lUr1SljnuXWejYTI2ZqHa4=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "81e8f48ebdecf07aab321182011b067aafc78896",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs",
+        "treefmt-nix": "treefmt-nix"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1695822946,
+        "narHash": "sha256-IQU3fYo0H+oGlqX5YrgZU3VRhbt2Oqe6KmslQKUO4II=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "720bd006d855b08e60664e4683ccddb7a9ff614a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
diff --git a/flake.nix b/flake.nix
new file mode 100644
index 0000000..b35d91e
--- /dev/null
+++ b/flake.nix
@@ -0,0 +1,30 @@
+{
+  description = "A flake for building Object Introspection.";
+
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
+
+    flake-utils.url = "github:numtide/flake-utils";
+
+    treefmt-nix.url = "github:numtide/treefmt-nix";
+    treefmt-nix.inputs.nixpkgs.follows = "nixpkgs";
+  };
+
+  outputs = { self, nixpkgs, flake-utils, treefmt-nix, ... }@inputs:
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        pkgs = nixpkgs.legacyPackages.${system};
+        treefmtEval = treefmt-nix.lib.evalModule pkgs (pkgs: {
+          projectRootFile = "flake.nix";
+          settings.global.excludes = [ "./extern/**" ];
+
+          programs.nixfmt.enable = true;
+          programs.clang-format.enable = true;
+          programs.black.enable = true;
+          programs.isort.enable = true;
+        });
+      in {
+        formatter = treefmtEval.config.build.wrapper;
+        checks.formatting = treefmtEval.config.build.check self;
+      });
+}

From 71b8f3406a74f0678a139c7a9dd65d25f8a37d78 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 14 Dec 2023 15:46:20 +0000
Subject: [PATCH 056/188] capture_keys: store dynamic type path components more
 efficiently

---
 include/oi/IntrospectionResult.h | 10 ++--
 oi/IntrospectionResult.cpp       | 82 +++++++++++++++++++-------------
 2 files changed, 53 insertions(+), 39 deletions(-)

diff --git a/include/oi/IntrospectionResult.h b/include/oi/IntrospectionResult.h
index 427463b..091028f 100644
--- a/include/oi/IntrospectionResult.h
+++ b/include/oi/IntrospectionResult.h
@@ -56,13 +56,9 @@ class IntrospectionResult {
     std::optional<result::Element> next_;
 
     std::vector<std::string_view> type_path_;
-    // This field could be more space efficient as these strings are primarily
-    // empty. They are used when the string isn't stored in the .rodata section,
-    // currently when performing key capture. It needs reference stability as we
-    // keep views in type_path_. A std::unique_ptr<std::string> would be an
-    // improvement but it isn't copyable. A string type with size fixed at
-    // construction would also be good.
-    std::list<std::string> dynamic_type_path_;
+    // Holds a pair of the type path entry this represents and the owned string
+    // that type_path_ has a view of.
+    std::list<std::pair<size_t, std::string>> dynamic_type_path_;
 
     // We cannot track the position in the iteration solely by the underlying
     // iterator as some fields do not extract data (for example, primitives).
diff --git a/oi/IntrospectionResult.cpp b/oi/IntrospectionResult.cpp
index efb11a2..bc59f5e 100644
--- a/oi/IntrospectionResult.cpp
+++ b/oi/IntrospectionResult.cpp
@@ -19,13 +19,19 @@
 
 #include <cassert>
 #include <iterator>
+#include <optional>
 #include <sstream>
 #include <stdexcept>
+#include <string>
 
 template <typename T>
 inline constexpr bool always_false_v = false;
 
 namespace oi {
+namespace {
+std::optional<std::string> genNameFromData(
+    const decltype(result::Element::data)&);
+}
 
 IntrospectionResult::const_iterator&
 IntrospectionResult::const_iterator::operator++() {
@@ -45,8 +51,10 @@ IntrospectionResult::const_iterator::operator++() {
       [this](auto&& r) -> IntrospectionResult::const_iterator& {
         using U = std::decay_t<decltype(r)>;
         if constexpr (std::is_same_v<U, exporters::inst::PopTypePath>) {
+          if (!dynamic_type_path_.empty() &&
+              dynamic_type_path_.back().first == type_path_.size())
+            dynamic_type_path_.pop_back();
           type_path_.pop_back();
-          dynamic_type_path_.pop_back();
           return operator++();
         } else if constexpr (std::is_same_v<U, exporters::inst::Repeat>) {
           if (r.n-- != 0) {
@@ -78,37 +86,15 @@ IntrospectionResult::const_iterator::operator++() {
                   *next_, [this](auto i) { stack_.emplace(i); }, parsed);
             }
 
-            std::string& new_name = dynamic_type_path_.emplace_back(std::visit(
-                [](const auto& d) -> std::string {
-                  using V = std::decay_t<decltype(d)>;
-                  if constexpr (std::is_same_v<std::string, V>) {
-                    std::string out = "[";
-                    out.reserve(d.size() + 2);
-                    out += d;
-                    out += "]";
-                    return out;
-                  } else if constexpr (std::is_same_v<result::Element::Pointer,
-                                                      V>) {
-                    std::stringstream out;
-                    out << '[' << reinterpret_cast<void*>(d.p) << ']';
-                    return out.str();
-                  } else if constexpr (std::is_same_v<result::Element::Scalar,
-                                                      V>) {
-                    std::string out = "[";
-                    out += std::to_string(d.n);
-                    out += ']';
-                    return out;
-                  } else if constexpr (std::is_same_v<std::nullopt_t, V>) {
-                    return "";
-                  } else {
-                    static_assert(always_false_v<V>, "missing variant");
-                  }
-                },
-                next_->data));
-            if (!new_name.empty()) {
-              type_path_.back() = new_name;
-              next_->type_path.back() = new_name;
-              next_->name = new_name;
+            if (auto new_name = genNameFromData(next_->data)) {
+              std::string& new_name_ref =
+                  dynamic_type_path_
+                      .emplace_back(type_path_.size(), std::move(*new_name))
+                      .second;
+
+              type_path_.back() = new_name_ref;
+              next_->type_path.back() = new_name_ref;
+              next_->name = new_name_ref;
             }
 
             for (auto it = ty.fields.rbegin(); it != ty.fields.rend(); ++it) {
@@ -124,4 +110,36 @@ IntrospectionResult::const_iterator::operator++() {
       el);
 }
 
+namespace {
+
+std::optional<std::string> genNameFromData(
+    const decltype(result::Element::data)& d) {
+  return std::visit(
+      [](const auto& d) -> std::optional<std::string> {
+        using V = std::decay_t<decltype(d)>;
+        if constexpr (std::is_same_v<std::string, V>) {
+          std::string out = "[";
+          out.reserve(d.size() + 2);
+          out += d;
+          out += "]";
+          return out;
+        } else if constexpr (std::is_same_v<result::Element::Pointer, V>) {
+          std::stringstream out;
+          out << '[' << reinterpret_cast<void*>(d.p) << ']';
+          return out.str();
+        } else if constexpr (std::is_same_v<result::Element::Scalar, V>) {
+          std::string out = "[";
+          out += std::to_string(d.n);
+          out += ']';
+          return out;
+        } else if constexpr (std::is_same_v<std::nullopt_t, V>) {
+          return std::nullopt;
+        } else {
+          static_assert(always_false_v<V>, "missing variant");
+        }
+      },
+      d);
+}
+
+}  // namespace
 }  // namespace oi

From d468ebf3d2f0531c3c2b3c7ad626ca65c54115a8 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 14 Dec 2023 18:19:56 +0000
Subject: [PATCH 057/188] type_graph: Add reference type

Our existing drgn parser emits pointers, regardless of whether the type in the
DWARF is really a reference. Newer parsers like the LLDB/Clang parsers can
differentiate between a pointer and a reference. We should do this as
references can be safely followed according to the language rules with
`ChaseRawPointers` disabled, while pointers cannot.

Test Plan:
- TBD
---
 oi/type_graph/NameGen.cpp               |  8 ++++
 oi/type_graph/NameGen.h                 |  1 +
 oi/type_graph/Printer.cpp               | 11 +++++
 oi/type_graph/Printer.h                 |  1 +
 oi/type_graph/RemoveTopLevelPointer.cpp |  4 ++
 oi/type_graph/RemoveTopLevelPointer.h   |  1 +
 oi/type_graph/Types.h                   | 56 +++++++++++++++++++++++++
 oi/type_graph/Visitor.h                 |  7 ++++
 8 files changed, 89 insertions(+)

diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index b1d1e8d..f016c13 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -200,6 +200,14 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& p) {
+  RecursiveVisitor::visit(p);
+  p.regenerateName();
+  std::string inputName{p.pointeeType().inputName()};
+  inputName += '*';
+  p.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..e6437d0 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index 0dae0ab..49165ab 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -150,6 +150,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& p) {
+  if (prefix(p))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = p.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(p.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index 8d7138a..e08a943 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& p) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;
diff --git a/oi/type_graph/RemoveTopLevelPointer.cpp b/oi/type_graph/RemoveTopLevelPointer.cpp
index 9ecaaaa..e1909a7 100644
--- a/oi/type_graph/RemoveTopLevelPointer.cpp
+++ b/oi/type_graph/RemoveTopLevelPointer.cpp
@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& p) {
+  topLevelType_ = &p.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/RemoveTopLevelPointer.h b/oi/type_graph/RemoveTopLevelPointer.h
index 611429f..6b70545 100644
--- a/oi/type_graph/RemoveTopLevelPointer.h
+++ b/oi/type_graph/RemoveTopLevelPointer.h
@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
 
  private:
   Type* topLevelType_ = nullptr;
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index e05d224..18e2d38 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -733,6 +734,61 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    name_ = pointeeType_.get().name() + "&";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index a7f7eec..f055aea 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -108,6 +108,9 @@ class RecursiveVisitor : public Visitor<void> {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -175,6 +178,10 @@ class RecursiveMutator : public Visitor<Type&> {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }

From 9ef5d54c6edddc57ab31108a0c377ee20877dfd1 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 14 Dec 2023 18:19:56 +0000
Subject: [PATCH 058/188] oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD
---
 CMakeLists.txt                                |  13 +-
 oi/CodeGen.cpp                                |  30 +-
 oi/CodeGen.h                                  |   8 +-
 oi/Config.cpp                                 |   9 +-
 oi/ContainerInfo.cpp                          |   2 +-
 oi/OICodeGen.h                                |   2 +-
 oi/OIGenerator.cpp                            | 364 ++++++++-------
 oi/OIGenerator.h                              |  29 +-
 oi/type_graph/CMakeLists.txt                  |   2 +
 oi/type_graph/ClangTypeParser.cpp             | 416 ++++++++++++++++++
 oi/type_graph/ClangTypeParser.h               | 131 ++++++
 test/integration/template_template_param.toml |  16 +
 tools/OILGen.cpp                              | 148 +++----
 types/string_type.toml                        |  31 ++
 14 files changed, 911 insertions(+), 290 deletions(-)
 create mode 100644 oi/type_graph/ClangTypeParser.cpp
 create mode 100644 oi/type_graph/ClangTypeParser.h
 create mode 100644 test/integration/template_template_param.toml

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f9ee7..eb18f4a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -104,6 +104,15 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(range-v3)
 
+### cxxopts
+FetchContent_Declare(
+  cxxopts
+  GIT_REPOSITORY https://github.com/jarro2783/cxxopts.git
+  GIT_TAG        eb787304d67ec22f7c3a184ee8b4c481d04357fd # v3.1.1
+  GIT_PROGRESS   TRUE
+)
+FetchContent_MakeAvailable(cxxopts)
+
 set_project_warnings()
 
 if (ASAN)
@@ -289,7 +298,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -363,8 +372,10 @@ add_executable(oilgen
   oi/OIGenerator.cpp
 )
 target_link_libraries(oilgen
+  cxxopts
   drgn_utils
   oicore
+  clangTooling
 )
 
 ### Object Introspection cache Printer (OIP)
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index ea8e4e3..20921ae 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -539,7 +539,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1106,15 +1106,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
 }
 
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
-  try {
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
+  if (!registerContainers())
     return false;
-  }
 
   TypeGraph typeGraph;
   try {
@@ -1129,6 +1122,20 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
   return true;
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    size_t i = 0;
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1170,7 +1177,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(IdentifyContainers::createPass(containerInfos_));
@@ -1286,7 +1293,8 @@ void CodeGen::generate(
   code += "\nusing __ROOT_TYPE__ = " + rootType.name() + ";\n";
   code += "} // namespace\n} // namespace OIInternal\n";
 
-  const auto typeName = SymbolService::getTypeName(drgnType);
+  // const auto typeName = SymbolService::getTypeName(drgnType);
+  const std::string typeName{"typeName"};
   if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
   } else {
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 186ec34..fb0ec20 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -40,8 +40,11 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config) : config_(config) {
+    assert(!config.features[Feature::PolymorphicInheritance]);
+  }
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   /*
@@ -53,6 +56,7 @@ class CodeGen {
                        std::string linkageName,
                        std::string& code);
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -66,7 +70,7 @@ class CodeGen {
 
  private:
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
diff --git a/oi/Config.cpp b/oi/Config.cpp
index 157b279..66798bb 100644
--- a/oi/Config.cpp
+++ b/oi/Config.cpp
@@ -22,6 +22,8 @@
 #include <boost/property_tree/ini_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 #include <filesystem>
+#include <range/v3/action/sort.hpp>
+#include <range/v3/action/unique.hpp>
 
 #include "oi/support/Toml.h"
 
@@ -54,6 +56,9 @@ std::optional<FeatureSet> processConfigFiles(
     enables |= *fs;
   }
 
+  ranges::actions::sort(generatorConfig.containerConfigPaths);
+  ranges::actions::unique(generatorConfig.containerConfigPaths);
+
   // Override anything from the config files with command line options
   for (auto [k, v] : featureMap) {
     enables[k] = v;
@@ -108,8 +113,8 @@ std::optional<FeatureSet> processConfigFile(
           if the right path is absolute, else append the right path to the
           left path.
           */
-          generatorConfig.containerConfigPaths.emplace(configDirectory /
-                                                       el.get());
+          generatorConfig.containerConfigPaths.emplace_back(configDirectory /
+                                                            el.get());
         }
       });
     }
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 99a7dc1..dbe10c6 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -202,7 +202,7 @@ std::regex getMatcher(const std::string& typeName) {
 ContainerInfo::ContainerInfo(const fs::path& path) {
   toml::table container;
   try {
-    container = toml::parse_file(std::string(path));
+    container = toml::parse_file(path.string());
   } catch (const toml::parse_error& err) {
     // Convert into a ContainerInfoError, just to avoid having to include
     // the huge TOML++ header in the caller's file. Use toml::parse_error's
diff --git a/oi/OICodeGen.h b/oi/OICodeGen.h
index a01548f..7226a4e 100644
--- a/oi/OICodeGen.h
+++ b/oi/OICodeGen.h
@@ -66,7 +66,7 @@ class OICodeGen {
     };
 
     FeatureSet features;
-    std::set<std::filesystem::path> containerConfigPaths;
+    std::vector<std::filesystem::path> containerConfigPaths;
     std::set<std::string> defaultHeaders;
     std::set<std::string> defaultNamespaces;
     std::vector<std::pair<std::string, std::string>> membersToStub;
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 1cb2a49..93054bc 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -16,130 +16,150 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
 #include <glog/logging.h>
 
 #include <boost/core/demangle.hpp>
 #include <fstream>
 #include <iostream>
+#include <range/v3/action/transform.hpp>
+#include <range/v3/core.hpp>
+#include <range/v3/view/cache1.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/drop_while.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>  // TODO: remove
 #include <string_view>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
+class ConsumerContext;
 
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
-      templateArgsToSymbolsMap;
-
-  auto symbols = prog.find_all_symbols();
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
+class CreateTypeGraphConsumer;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
-    if (val.first.empty() || val.second.empty()) {
-      continue;
-    }
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  return strongToWeakSymbols;
-}
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
 
-std::unordered_map<std::string, drgn_qualified_type>
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
+ private:
+  ConsumerContext& ctx;
+};
 
-  std::unordered_map<std::string, drgn_qualified_type> out;
+class ConsumerContext {
+ public:
+  ConsumerContext(const std::vector<std::unique_ptr<ContainerInfo>>& cis)
+      : containerInfos{cis} {
+  }
 
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
-    std::string strongLinkageName;
-    {
-      const char* linkageNameCstr;
-      if (auto err = drgnplusplus::error(
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
-        // throw err;
+  type_graph::TypeGraph typeGraph;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
+  std::optional<bool> pic;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+}  // namespace
+
+int OIGenerator::generate(clang::tooling::CompilationDatabase& db,
+                          const std::vector<std::string>& sourcePaths) {
+  std::map<Feature, bool> featuresMap = {
+      {Feature::TypeGraph, true},
+      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},
+      {Feature::PackStructs, true},
+      {Feature::PruneTypeGraph, true},
+  };
+
+  OICodeGen::Config generatorConfig{};
+  OICompiler::Config compilerConfig{};
+
+  auto features = config::processConfigFiles(
+      configFilePaths, featuresMap, compilerConfig, generatorConfig);
+  if (!features) {
+    LOG(ERROR) << "failed to process config file";
+    return -1;
+  }
+  generatorConfig.features = *features;
+  compilerConfig.features = *features;
+
+  std::vector<std::unique_ptr<ContainerInfo>> containerInfos;
+  containerInfos.reserve(generatorConfig.containerConfigPaths.size());
+  try {
+    for (const auto& path : generatorConfig.containerConfigPaths) {
+      auto info = std::make_unique<ContainerInfo>(path);
+      if (info->requiredFeatures != (*features & info->requiredFeatures)) {
+        VLOG(1) << "Skipping container (feature conflict): " << info->typeName;
         continue;
       }
-      strongLinkageName = linkageNameCstr;
+      containerInfos.emplace_back(std::move(info));
     }
-
-    std::string weakLinkageName;
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return -1;
   }
 
-  return out;
-}
+  ConsumerContext ctx{containerInfos};
+  CreateTypeGraphActionFactory factory{ctx};
 
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
-                                      const OICompiler::Config& compilerConfig,
-                                      const drgn_qualified_type& type,
-                                      const std::string& linkageName,
-                                      SymbolService& symbols) {
-  CodeGen codegen{generatorConfig, symbols};
+  clang::tooling::ClangTool tool{db, sourcePaths};
+  if (auto ret = tool.run(&factory); ret != 0) {
+    return ret;
+  }
+
+  if (ctx.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one type to generate for but we can't currently "
+        "handle this case");
+
+  compilerConfig.usePIC = ctx.pic.value();
+  CodeGen codegen{generatorConfig};
+  for (auto&& ptr : containerInfos)
+    codegen.registerContainer(std::move(ptr));
+  codegen.transform(ctx.typeGraph);
 
   std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
+  codegen.generate(ctx.typeGraph, code, nullptr);
 
   std::string sourcePath = sourceFileDumpPath;
   if (sourceFileDumpPath.empty()) {
@@ -152,78 +172,110 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
   }
 
   OICompiler compiler{{}, compilerConfig};
-
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
-
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
 }
 
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
-  drgnplusplus::program prog;
+namespace {
 
-  {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(
-            drgn_program_load_debug_info(prog.get(),
-                                         std::data(objectPaths),
-                                         std::size(objectPaths),
-                                         false,
-                                         false))) {
-      LOG(ERROR) << "error loading debug info program: " << err;
-      throw err;
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx(ctx_) {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
     }
-  }
 
-  auto oilTypes = findOilTypesAndNames(prog);
-
-  std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
-      {Feature::PruneTypeGraph, true},
-  };
-
-  OICodeGen::Config generatorConfig{};
-  OICompiler::Config compilerConfig{};
-  compilerConfig.usePIC = pic;
-
-  auto features = config::processConfigFiles(
-      configFilePaths, featuresMap, compilerConfig, generatorConfig);
-  if (!features) {
-    LOG(ERROR) << "failed to process config file";
-    return -1;
-  }
-  generatorConfig.features = *features;
-  compilerConfig.features = *features;
-
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
+    auto introspectImpl =
+        std::move(oi_namespaces) |
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
+        ranges::views::transform([](auto* p) {
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
+        }) |
+        ranges::views::filter([](auto* td) {
+          return td != nullptr && td->getName() == "introspectImpl";
+        }) |
+        ranges::views::take(1) | ranges::to<std::vector>();
+    if (introspectImpl.empty()) {
+      LOG(WARNING)
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
+             "you compile with `OIL_AOT_COMPILATION=1`?";
+      return;
     }
-  }
 
-  size_t successes = oilTypes.size() - failures;
-  LOG(INFO) << "object introspection generation complete. " << successes
-            << " successes and " << failures << " failures.";
+    auto nameToClangTypeMap =
+        ranges::views::single(introspectImpl[0]) |
+        ranges::views::for_each(
+            [](auto* td) { return td->specializations(); }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
 
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
-    return -1;
+              assert(fd->getNumParams() == 1);
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, opts};
+
+    auto& Sema = *ctx.sema;
+    ctx.nameToTypeMap =
+        nameToClangTypeMap |
+        ranges::views::transform(
+            [&parser, &Context, &Sema](
+                auto& p) -> std::pair<std::string, type_graph::Type*> {
+              return {p.first, &parser.parse(Context, Sema, *p.second)};
+            }) |
+        ranges::to<std::unordered_map>();
+
+    for (const auto& [name, type] : ctx.nameToTypeMap)
+      ctx.typeGraph.addRoot(*type);
   }
-  return 0;
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  // Compile the output as position independent if any input is position
+  // independent
+  bool pic = CI.getCodeGenOpts().RelocationModel == llvm::Reloc::PIC_;
+  ctx.pic = ctx.pic.value_or(false) || pic;
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
 }
 
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+}  // namespace
 }  // namespace oi::detail
diff --git a/oi/OIGenerator.h b/oi/OIGenerator.h
index dd38384..fac947a 100644
--- a/oi/OIGenerator.h
+++ b/oi/OIGenerator.h
@@ -23,11 +23,19 @@
 #include "oi/OICodeGen.h"
 #include "oi/OICompiler.h"
 
+namespace clang::tooling {
+class CompilationDatabase;
+}
+
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate(clang::tooling::CompilationDatabase&,
+               const std::vector<std::string>&);
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -41,8 +49,8 @@ class OIGenerator {
   void setFailIfNothingGenerated(bool fail) {
     failIfNothingGenerated = fail;
   }
-  void setUsePIC(bool pic_) {
-    pic = pic_;
+  void setClangArgs(std::vector<std::string> args_) {
+    clangArgs = std::move(args_);
   }
 
  private:
@@ -50,20 +58,7 @@ class OIGenerator {
   std::vector<std::filesystem::path> configFilePaths;
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
-  bool pic = false;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
+  std::vector<std::string> clangArgs;
 };
 
 }  // namespace oi::detail
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 940c031..46add03 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
   Flattener.cpp
@@ -26,3 +27,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
new file mode 100644
index 0000000..67f7bfa
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -0,0 +1,416 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Basic/DiagnosticSema.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <iostream>  // TODO: maybe remove
+#include <regex>
+#include <stdexcept>  // TODO: remove
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (VLOG_IS_ON(3)) {
+    std::string fqName = clang::TypeName::getFullyQualifiedName(
+        clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+    VLOG(3) << std::string(depth_ * 2, ' ') << fqName;
+  }
+
+  if (!requireCompleteType(*sema, ty))
+    return makeType<Incomplete>(ty, "incomplete (TODO naming)");
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(llvm::cast<const clang::RecordType>(ty));
+    case clang::Type::LValueReference:
+      return enumerateReference(
+          llvm::cast<const clang::LValueReferenceType>(ty));
+    case clang::Type::Pointer:
+      return enumeratePointer(llvm::cast<const clang::PointerType>(ty));
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          llvm::cast<const clang::SubstTemplateTypeParmType>(ty));
+    case clang::Type::Builtin:
+      return enumeratePrimitive(llvm::cast<const clang::BuiltinType>(ty));
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(
+          llvm::cast<const clang::ElaboratedType>(ty));
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          llvm::cast<const clang::TemplateSpecializationType>(ty));
+    case clang::Type::UnaryTransform:
+      return enumerateUnaryTransformType(
+          llvm::cast<const clang::UnaryTransformType>(ty));
+    case clang::Type::Decltype:
+      return enumerateDecltypeType(llvm::cast<const clang::DecltypeType>(ty));
+    case clang::Type::Typedef:
+      return enumerateTypedef(llvm::cast<const clang::TypedefType>(ty));
+    case clang::Type::Using:
+      return enumerateUsing(llvm::cast<const clang::UsingType>(ty));
+    case clang::Type::ConstantArray:
+      return enumerateArray(llvm::cast<const clang::ConstantArrayType>(ty));
+    case clang::Type::Enum:
+      return enumerateEnum(llvm::cast<const clang::EnumType>(ty));
+
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Type& ClangTypeParser::enumerateDecltypeType(const clang::DecltypeType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Type& ClangTypeParser::enumerateUnaryTransformType(
+    const clang::UnaryTransformType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Typedef& ClangTypeParser::enumerateUsing(const clang::UsingType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+  std::string name = ty.getFoundDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+
+  std::string name = ty.getDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string name = ty.getDecl()->getNameAsString();
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  std::map<int64_t, std::string> enumeratorMap;
+  if (options_.readEnumValues) {
+    for (const auto* enumerator : ty.getDecl()->enumerators()) {
+      enumeratorMap.emplace(enumerator->getInitVal().getExtValue(),
+                            enumerator->getNameAsString());
+    }
+  }
+
+  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+}
+
+Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
+  uint64_t len = ty.getSize().getLimitedValue();
+  auto& t = enumerateType(*ty.getElementType());
+  return makeType<Array>(ty, t, len);
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  if (auto* info = getContainerInfo(fqName)) {
+    auto& c = makeType<Container>(ty, *info, size, nullptr);
+    enumerateClassTemplateParams(ty, c.templateParams);
+    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)));
+    return c;
+  }
+
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+
+  params.reserve(list.size());
+  for (const auto& arg : list.asArray()) {
+    if (auto p = enumerateTemplateParam(arg))
+      params.emplace_back(std::move(p.value()));
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return TemplateParam{ttype, qualifiers};
+    }
+    case clang::TemplateArgument::Integral: {
+      auto& ty = enumerateType(*p.getIntegralType());
+      llvm::SmallString<32> val;
+      p.getAsIntegral().toString(val);
+      return TemplateParam{ty, std::string(val)};
+    }
+    case clang::TemplateArgument::Template: {
+      return enumerateTemplateTemplateParam(p.getAsTemplate());
+    }
+
+#define X(name)                       \
+  case clang::TemplateArgument::name: \
+    throw std::logic_error("unsupported template argument kind: " #name);
+
+      X(Null)
+      X(Declaration)
+      X(NullPtr)
+      X(TemplateExpansion)
+      X(Expression)
+      X(Pack)
+#undef X
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateTemplateParam(
+    const clang::TemplateName& tn) {
+  switch (tn.getKind()) {
+    case clang::TemplateName::Template:
+      return std::nullopt;
+
+#define X(name)                   \
+  case clang::TemplateName::name: \
+    throw std::logic_error("unsupported template name kind: " #name);
+
+      X(OverloadedTemplate)
+      X(AssumedTemplate)
+      X(QualifiedTemplate)
+      X(DependentTemplate)
+      X(SubstTemplateTemplateParm)
+      X(SubstTemplateTemplateParmPack)
+      X(UsingTemplate)
+#undef X
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function references
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Float:
+      return makeType<Primitive>(ty, Primitive::Kind::Float32);
+    case clang::BuiltinType::Double:
+    case clang::BuiltinType::LongDouble:
+      return makeType<Primitive>(ty, Primitive::Kind::Float64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+ContainerInfo* ClangTypeParser::getContainerInfo(
+    const std::string& fqName) const {
+  for (const auto& containerInfo : containers_) {
+    if (std::regex_search(fqName, containerInfo->matcher)) {
+      return containerInfo.get();
+    }
+  }
+  return nullptr;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
+    return true;  // treat as complete
+
+  // TODO: This is a terrible warning.
+  return !sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      clang::diag::warn_nsconsumed_attribute_mismatch);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
new file mode 100644
index 0000000..10eea87
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.h
@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ASTContext;
+class BuiltinType;
+class ConstantArrayType;
+class DecltypeType;
+class ElaboratedType;
+class EnumType;
+class LValueReferenceType;
+class PointerType;
+class RecordType;
+class Sema;
+class SubstTemplateTypeParmType;
+class TemplateArgument;
+class TemplateName;
+class TemplateSpecializationType;
+class Type;
+class TypedefType;
+class UnaryTransformType;
+class UsingType;
+}  // namespace clang
+
+struct ContainerInfo;
+
+namespace oi::detail::type_graph {
+
+class Array;
+class Class;
+class Enum;
+class Member;
+class Primitive;
+class Reference;
+class Type;
+class TypeGraph;
+class Typedef;
+struct TemplateParam;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+  bool readEnumValues = false;
+};
+
+/*
+ * ClangTypeParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given drgn_type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph,
+                  const std::vector<std::unique_ptr<ContainerInfo>>& containers,
+                  ClangTypeParserOptions options)
+      : typeGraph_{typeGraph}, containers_{containers}, options_{options} {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containers_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Type& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+  Typedef& enumerateTypedef(const clang::TypedefType&);
+  Typedef& enumerateUsing(const clang::UsingType&);
+  Type& enumerateUnaryTransformType(const clang::UnaryTransformType&);
+  Type& enumerateDecltypeType(const clang::DecltypeType&);
+
+  Array& enumerateArray(const clang::ConstantArrayType&);
+  Enum& enumerateEnum(const clang::EnumType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  std::optional<TemplateParam> enumerateTemplateParam(
+      const clang::TemplateArgument&);
+  std::optional<TemplateParam> enumerateTemplateTemplateParam(
+      const clang::TemplateName&);
+
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  ContainerInfo* getContainerInfo(const std::string& fqName) const;
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph
diff --git a/test/integration/template_template_param.toml b/test/integration/template_template_param.toml
new file mode 100644
index 0000000..c64590d
--- /dev/null
+++ b/test/integration/template_template_param.toml
@@ -0,0 +1,16 @@
+includes = ["vector", "utility", "string"]
+
+definitions = '''
+template <typename K, typename V, template <typename, typename> typename Pair>
+struct bad_map {
+  std::vector<Pair<K, K>> keys;
+  std::vector<Pair<V, V>> values;
+};
+'''
+
+[cases]
+  [cases.int_int_empty]
+    param_types = ["const bad_map<int, std::string, std::pair>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index d1e345d..795389e 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -14,131 +14,81 @@
  * limitations under the License.
  */
 
+#include <clang/Tooling/CommonOptionsParser.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include <cstdlib>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/action/drop_while.hpp>
+#include <range/v3/algorithm/find.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/all.hpp>
+#include <range/v3/view/transform.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
 
-constexpr static OIOpts opts{
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
+static llvm::cl::OptionCategory OilgenCategory("oilgen options");
 
-void usage() {
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
-  std::cerr << opts;
+static llvm::cl::list<std::string> ConfigFiles("config-file",
+                                               llvm::cl::desc(R"(TODO HELP)"),
+                                               llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<int> DebugLevel("debug-level",
+                                     llvm::cl::desc(R"(TODO HELP)"),
+                                     llvm::cl::init(-1),
+                                     llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> DumpJit("dump-jit",
+                                          llvm::cl::desc(R"(TODO HELP)"),
+                                          llvm::cl::init("jit.cpp"),
+                                          llvm::cl::cat(OilgenCategory));
 
-  std::cerr << std::endl
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
-
-int main(int argc, char* argv[]) {
+int main(int argc, const char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
-  std::vector<fs::path> configFilePaths;
-  fs::path sourceFileDumpPath = "";
-  bool exitCode = false;
-  bool pic = false;
-
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
+  auto expectedParser =
+      clang::tooling::CommonOptionsParser::create(argc, argv, OilgenCategory);
+  if (!expectedParser) {
+    llvm::errs() << expectedParser.takeError();
+    return -1;
   }
+  clang::tooling::CommonOptionsParser& options = expectedParser.get();
+  auto& compilations = options.getCompilations();
 
-  if (optind >= argc) {
-    usage();
-    return EXIT_FAILURE;
-  }
-  fs::path primaryObject = argv[optind];
-
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
+  if (DebugLevel.getNumOccurrences()) {
+    google::LogToStderr();
+    google::SetStderrLogging(google::INFO);
+    google::SetVLOGLevel("*", DebugLevel);
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
+    gflags::SetCommandLineOption("minloglevel", "0");
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
-  oigen.setConfigFilePaths(std::move(configFilePaths));
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
-  oigen.setFailIfNothingGenerated(exitCode);
-  oigen.setUsePIC(pic);
+  oigen.setConfigFilePaths(ConfigFiles |
+                           ranges::views::transform([](const auto& p) {
+                             return std::filesystem::path(p);
+                           }) |
+                           ranges::to<std::vector>());
+  if (DumpJit.getNumOccurrences())
+    oigen.setSourceFileDumpPath(DumpJit.getValue());
 
-  SymbolService symbols(primaryObject);
+  oigen.setOutputPath("a.o");  // TODO: make this an opt
 
-  return oigen.generate(primaryObject, symbols);
+  // TODO: can we extract this from the original arguments?
+  // oigen.setUsePIC(args["pic"].as<bool>());
+
+  oigen.setFailIfNothingGenerated(true);
+  // compilations.getAllFiles();
+  // return oigen.generate(compilations,
+  // {"/data/users/jakehillion/fbsource/fbcode/object-introspection/oil/examples/compile-time/VectorOfStrings-Introspect.cpp"});
+  return oigen.generate(compilations, options.getSourcePathList());
 }
diff --git a/types/string_type.toml b/types/string_type.toml
index fa4a2ae..0d2ab0d 100644
--- a/types/string_type.toml
+++ b/types/string_type.toml
@@ -35,3 +35,34 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
     );
 }
 """
+
+traversal_func = """
+    bool sso = ((uintptr_t)container.data() <
+                (uintptr_t)(&container + sizeof(std::__cxx11::basic_string<T0>))) &&
+               ((uintptr_t)container.data() >= (uintptr_t)&container);
+
+    return returnArg.write(container.capacity())
+        .write(sso)
+        .write(container.size());
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+uint64_t capacity = std::get<ParsedData::VarInt>(d.val).value;
+el.container_stats.emplace(result::Element::ContainerStats { .capacity = capacity });
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+bool sso = std::get<ParsedData::VarInt>(d.val).value;
+if (!sso)
+  el.exclusive_size += el.container_stats->capacity * sizeof(T0);
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats->length = std::get<ParsedData::VarInt>(d.val).value;
+"""

From 7712e2aa63c6eb377b7b591931c012eb39d8a83d Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 14 Dec 2023 18:20:37 +0000
Subject: [PATCH 059/188] incomplete: name type in compiler errors

Summary:

We have a good type representation in the Type Graph of an incomplete type and
the underlying type that represents. However, this incomplete type still ends
up in the generated code as `void` which loses information. For example, a
container that can't contain void may fail to compile because it was
initialised with `void` but really its because the type it was supposed to be
initialised with (say, `Foo`) had incomplete debug information.

This change identifies that a type is incomplete in the output by generating it
as an incomplete type `struct Incomplete<struct Foo>`. This allows us to name
the type correctly in the TreeBuilder output and filter for incomplete types,
as well as getting appropriate compiler errors if it mustn't be incomplete.

Test Plan:
- CI
- Added a unit test to namegen.
- Enabled and added an extra pointers_incomplete test.

This change is tricky to test because it isn't really user visible. The types
still use their `inputName` which is unchanged in any successful output - this
change is used so the compiler fails with a more detailed error.
---
 oi/CodeGen.cpp                            |  9 ++++
 oi/OITraceCode.cpp                        | 15 +++++++
 oi/type_graph/NameGen.cpp                 |  5 +++
 oi/type_graph/NameGen.h                   |  1 +
 oi/type_graph/Printer.cpp                 |  4 +-
 oi/type_graph/TopoSorter.cpp              |  4 ++
 oi/type_graph/TopoSorter.h                |  1 +
 oi/type_graph/Types.cpp                   |  2 -
 oi/type_graph/Types.h                     | 53 +++++++++++++++++------
 test/TypeGraphParser.cpp                  |  5 ++-
 test/integration/pointers_incomplete.toml | 32 ++++++++++++--
 test/test_drgn_parser.cpp                 |  4 +-
 test/test_enforce_compatibility.cpp       |  4 +-
 test/test_flattener.cpp                   |  2 +-
 test/test_name_gen.cpp                    | 13 ++++++
 test/test_remove_members.cpp              |  2 +-
 types/shrd_ptr_type.toml                  |  6 +--
 types/uniq_ptr_type.toml                  |  6 +--
 18 files changed, 133 insertions(+), 35 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index ea8e4e3..bb7203b 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -57,6 +57,7 @@ using type_graph::EnforceCompatibility;
 using type_graph::Enum;
 using type_graph::Flattener;
 using type_graph::IdentifyContainers;
+using type_graph::Incomplete;
 using type_graph::KeyCapture;
 using type_graph::Member;
 using type_graph::NameGen;
@@ -178,12 +179,20 @@ void genDeclsEnum(const Enum& e, std::string& code) {
   code += " {};\n";
 }
 
+void genDeclsIncomplete(const Incomplete& i, std::string& code) {
+  code += "template<> struct ";
+  code += i.name();
+  code += ";\n";
+}
+
 void genDecls(const TypeGraph& typeGraph, std::string& code) {
   for (const Type& t : typeGraph.finalTypes) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
       genDeclsClass(*c, code);
     } else if (const auto* e = dynamic_cast<const Enum*>(&t)) {
       genDeclsEnum(*e, code);
+    } else if (const auto* i = dynamic_cast<const Incomplete*>(&t)) {
+      genDeclsIncomplete(*i, code);
     }
   }
 }
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index 31a873b..416d81c 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -182,3 +182,18 @@ bool isStorageInline(const auto& c) {
   return (uintptr_t)std::data(c) < (uintptr_t)(&c + sizeof(c)) &&
          (uintptr_t)std::data(c) >= (uintptr_t)&c;
 }
+
+namespace OIInternal {
+namespace {
+
+template <typename T>
+struct Incomplete;
+
+template <typename T>
+constexpr bool oi_is_complete = true;
+
+template <typename T>
+constexpr bool oi_is_complete<Incomplete<T>> = false;
+
+}  // namespace
+}  // namespace OIInternal
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index b1d1e8d..1977e40 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -210,4 +210,9 @@ void NameGen::visit(CaptureKeys& c) {
   c.regenerateName();
 }
 
+void NameGen::visit(Incomplete& i) {
+  RecursiveVisitor::visit(i);
+  i.regenerateName();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..4857ded 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -48,6 +48,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
+  void visit(Incomplete& i) override;
 
   static const inline std::string AnonPrefix = "__oi_anon";
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index 0dae0ab..d7866b3 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -37,7 +37,9 @@ void Printer::print(const Type& type) {
 }
 
 void Printer::visit(const Incomplete& i) {
-  prefix();
+  if (prefix(i))
+    return;
+
   out_ << "Incomplete";
   if (auto underlyingType = i.underlyingType()) {
     out_ << std::endl;
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index 6ca10af..ff2715a 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -139,6 +139,10 @@ void TopoSorter::visit(CaptureKeys& c) {
   sortedTypes_.push_back(c);
 }
 
+void TopoSorter::visit(Incomplete& i) {
+  sortedTypes_.push_back(i);
+}
+
 /*
  * A type graph may contain cycles, so we need to slightly tweak the standard
  * topological sorting algorithm. Cycles can only be introduced by certain
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 374e943..44f2364 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -48,6 +48,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
+  void visit(Incomplete& i) override;
 
  private:
   std::unordered_set<Type*> visited_;
diff --git a/oi/type_graph/Types.cpp b/oi/type_graph/Types.cpp
index 332c145..035feae 100644
--- a/oi/type_graph/Types.cpp
+++ b/oi/type_graph/Types.cpp
@@ -32,8 +32,6 @@ namespace oi::detail::type_graph {
 OI_TYPE_LIST
 #undef X
 
-const std::string Incomplete::kName = "void";
-
 std::string Primitive::getName(Kind kind) {
   switch (kind) {
     case Kind::Int8:
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index e05d224..a4aaa4d 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -29,6 +29,7 @@
  * debugging.
  */
 
+#include <algorithm>
 #include <cstddef>
 #include <map>
 #include <optional>
@@ -207,29 +208,33 @@ struct TemplateParam {
  */
 class Incomplete : public Type {
  public:
-  Incomplete(Type& underlyingType) : underlyingType_(underlyingType) {
+  Incomplete(NodeId id, Type& underlyingType)
+      : id_(id), underlyingType_(underlyingType) {
   }
 
-  Incomplete(std::string underlyingTypeName)
-      : underlyingType_(std::move(underlyingTypeName)) {
+  Incomplete(NodeId id, std::string underlyingTypeName)
+      : id_(id), underlyingType_(std::move(underlyingTypeName)) {
   }
 
-  static inline constexpr bool has_node_id = false;
+  static inline constexpr bool has_node_id = true;
 
   DECLARE_ACCEPT
 
   const std::string& name() const override {
-    return kName;
+    return name_;
   }
 
   std::string_view inputName() const override {
-    if (std::holds_alternative<std::string>(underlyingType_)) {
-      return std::get<std::string>(underlyingType_);
-    }
-
-    return std::get<std::reference_wrapper<Type>>(underlyingType_)
-        .get()
-        .inputName();
+    return std::visit(
+        [](const auto& el) -> std::string_view {
+          using T = std::decay_t<decltype(el)>;
+          if constexpr (std::is_same_v<T, std::string>) {
+            return el;
+          } else {
+            return el.get().inputName();
+          }
+        },
+        underlyingType_);
   }
 
   size_t size() const override {
@@ -241,7 +246,26 @@ class Incomplete : public Type {
   }
 
   NodeId id() const override {
-    return -1;
+    return id_;
+  }
+
+  void regenerateName() {
+    std::string_view inputName = this->inputName();
+
+    constexpr std::string_view kPrefix{"Incomplete<struct "};
+    name_.clear();
+    name_.reserve(kPrefix.size() + inputName.size() + 2);
+
+    name_ = kPrefix;
+    for (const auto c : inputName) {
+      if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') ||
+          (c >= 'A' && c <= 'Z')) {
+        name_ += c;
+      } else {
+        name_ += '_';
+      }
+    }
+    name_ += ">";
   }
 
   std::optional<std::reference_wrapper<Type>> underlyingType() const {
@@ -253,8 +277,9 @@ class Incomplete : public Type {
   }
 
  private:
+  NodeId id_ = -1;
   std::variant<std::string, std::reference_wrapper<Type>> underlyingType_;
-  static const std::string kName;
+  std::string name_;
 };
 
 /*
diff --git a/test/TypeGraphParser.cpp b/test/TypeGraphParser.cpp
index f54fe08..d70236c 100644
--- a/test/TypeGraphParser.cpp
+++ b/test/TypeGraphParser.cpp
@@ -215,10 +215,11 @@ Type& TypeGraphParser::parseType(std::string_view& input, size_t rootIndent) {
       auto nameEndPos = line.find(']', nameStartPos);
       auto underlyingTypeName =
           line.substr(nameStartPos, nameEndPos - nameStartPos);
-      type = &typeGraph_.makeType<Incomplete>(std::string(underlyingTypeName));
+      type =
+          &typeGraph_.makeType<Incomplete>(id, std::string(underlyingTypeName));
     } else {
       auto& underlyingType = parseType(input, indent + 2);
-      type = &typeGraph_.makeType<Incomplete>(underlyingType);
+      type = &typeGraph_.makeType<Incomplete>(id, underlyingType);
     }
   } else if (nodeTypeName == "Class" || nodeTypeName == "Struct" ||
              nodeTypeName == "Union") {
diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml
index 437041c..aeebb24 100644
--- a/test/integration/pointers_incomplete.toml
+++ b/test/integration/pointers_incomplete.toml
@@ -55,7 +55,6 @@ definitions = '''
     }]'''
 
   [cases.unique_ptr]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299
     param_types = ["const std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>&"]
     setup = '''
       auto raw_ptr = static_cast<IncompleteType*>(::operator new(5));
@@ -63,28 +62,29 @@ definitions = '''
         raw_ptr, &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
   [cases.unique_ptr_null]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299
     param_types = ["const std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>&"]
     setup = '''
       return std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>(
         nullptr, &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.shared_ptr]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300
     param_types = ["const std::shared_ptr<IncompleteType>&"]
     setup = '''
       auto raw_ptr = static_cast<IncompleteType*>(::operator new(5));
       return std::shared_ptr<IncompleteType>(raw_ptr , &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
   [cases.shared_ptr_null]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300
     param_types = ["const std::shared_ptr<IncompleteType>"]
     setup = "return nullptr;"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.containing_struct]
     oil_disable = "oil can't chase raw pointers safely"
@@ -117,3 +117,27 @@ definitions = '''
         }
       ]
     }]'''
+
+  [cases.containing_struct_no_follow]
+    param_types = ["const IncompleteTypeContainer&"]
+    setup = "return IncompleteTypeContainer{};"
+    expect_json = '''[{
+      "staticSize": 88,
+      "members": [
+        { "name": "ptrundef", "staticSize": 8 },
+        { "name": "__makePad1", "staticSize": 1 },
+        { "name": "shundef", "staticSize": 16 },
+        { "name": "__makePad2", "staticSize": 1 },
+        { "name": "shoptundef",
+          "staticSize": 24,
+          "length": 0,
+          "capacity": 1
+        },
+        { "name": "__makePad3", "staticSize": 1 },
+        { "name": "optundef",
+          "staticSize": 16,
+          "length": 0,
+          "capacity": 1
+        }
+      ]
+    }]'''
diff --git a/test/test_drgn_parser.cpp b/test/test_drgn_parser.cpp
index a5ebe30..0d09b31 100644
--- a/test/test_drgn_parser.cpp
+++ b/test/test_drgn_parser.cpp
@@ -426,8 +426,8 @@ TEST_F(DrgnParserTest, PointerNoFollow) {
 
 TEST_F(DrgnParserTest, PointerIncomplete) {
   test("oid_test_case_pointers_incomplete_raw", R"(
-[0] Pointer
-      Incomplete: [IncompleteType]
+[1] Pointer
+[0]   Incomplete: [IncompleteType]
 )");
 }
 
diff --git a/test/test_enforce_compatibility.cpp b/test/test_enforce_compatibility.cpp
index 2632823..d91f356 100644
--- a/test/test_enforce_compatibility.cpp
+++ b/test/test_enforce_compatibility.cpp
@@ -38,8 +38,8 @@ TEST(EnforceCompatibilityTest, VoidPointer) {
        R"(
 [0] Class: MyClass (size: 8)
       Member: p (offset: 0)
-[1]     Pointer
-          Incomplete
+[2]     Pointer
+[1]       Incomplete
             Primitive: void
 )",
        R"(
diff --git a/test/test_flattener.cpp b/test/test_flattener.cpp
index 808f4ee..c8944bf 100644
--- a/test/test_flattener.cpp
+++ b/test/test_flattener.cpp
@@ -1007,7 +1007,7 @@ TEST(FlattenerTest, IncompleteParent) {
        R"(
 [0] Class: MyClass (size: 4)
       Parent (offset: 0)
-        Incomplete: [IncompleteParent]
+[1]     Incomplete: [IncompleteParent]
 )",
        R"(
 [0] Class: MyClass (size: 4)
diff --git a/test/test_name_gen.cpp b/test/test_name_gen.cpp
index d0cda26..a83c180 100644
--- a/test/test_name_gen.cpp
+++ b/test/test_name_gen.cpp
@@ -501,3 +501,16 @@ TEST(NameGenTest, AnonymousMembers) {
   EXPECT_EQ(myclass.members[0].inputName, "__oi_anon_0");
   EXPECT_EQ(myclass.members[1].inputName, "__oi_anon_1");
 }
+
+TEST(NameGenTest, IncompleteTypes) {
+  auto myincompletevector = Incomplete{0, "std::vector<int>"};
+
+  auto myint = Primitive{Primitive::Kind::Int32};
+  auto myincompleteint = Incomplete{1, myint};
+
+  NameGen nameGen;
+  nameGen.generateNames({myincompletevector, myincompleteint});
+
+  EXPECT_EQ(myincompletevector.name(), "Incomplete<struct std__vector_int_>");
+  EXPECT_EQ(myincompleteint.name(), "Incomplete<struct int32_t>");
+}
diff --git a/test/test_remove_members.cpp b/test/test_remove_members.cpp
index 0d4a54e..9eff4ed 100644
--- a/test/test_remove_members.cpp
+++ b/test/test_remove_members.cpp
@@ -191,7 +191,7 @@ TEST(RemoveMembersTest, Incomplete) {
       Member: a (offset: 0)
         Primitive: int32_t
       Member: b (offset: 4)
-        Incomplete: [MyIncompleteType]
+[1]     Incomplete: [MyIncompleteType]
       Member: c (offset: 8)
         Primitive: int32_t
 )",
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index bb3a622..de84b71 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (!std::is_void<T>::value && oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
         if (s_ptr && ctx.pointers.add((uintptr_t)(s_ptr.get()))) {
@@ -37,7 +37,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (std::is_void<T0>::value || !oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
@@ -85,7 +85,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index f723f80..09e351e 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T,Deleter>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (!std::is_void<T>::value && oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
         if (u_ptr && ctx.pointers.add((uintptr_t)(u_ptr.get()))) {
@@ -38,7 +38,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (std::is_void<T0>::value || !oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
@@ -71,7 +71,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);

From 038d4f00184512e1a3fae0b573185ff9d8310732 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 19 Dec 2023 10:59:45 +0000
Subject: [PATCH 060/188] incomplete: name type in compiler errors

Summary:

We have a good type representation in the Type Graph of an incomplete type and
the underlying type that represents. However, this incomplete type still ends
up in the generated code as `void` which loses information. For example, a
container that can't contain void may fail to compile because it was
initialised with `void` but really its because the type it was supposed to be
initialised with (say, `Foo`) had incomplete debug information.

This change identifies that a type is incomplete in the output by generating it
as an incomplete type `struct Incomplete<struct Foo>`. This allows us to name
the type correctly in the TreeBuilder output and filter for incomplete types,
as well as getting appropriate compiler errors if it mustn't be incomplete.

Test Plan:
- CI
- Added a unit test to namegen.
- Enabled and added an extra pointers_incomplete test.

This change is tricky to test because it isn't really user visible. The types
still use their `inputName` which is unchanged in any successful output - this
change is used so the compiler fails with a more detailed error.
---
 oi/CodeGen.cpp                            |  9 ++++
 oi/OITraceCode.cpp                        | 15 +++++++
 oi/type_graph/NameGen.cpp                 |  5 +++
 oi/type_graph/NameGen.h                   |  1 +
 oi/type_graph/Printer.cpp                 |  4 +-
 oi/type_graph/TopoSorter.cpp              |  4 ++
 oi/type_graph/TopoSorter.h                |  1 +
 oi/type_graph/Types.cpp                   |  2 -
 oi/type_graph/Types.h                     | 53 +++++++++++++++++------
 test/TypeGraphParser.cpp                  |  5 ++-
 test/integration/pointers_incomplete.toml | 32 ++++++++++++--
 test/test_drgn_parser.cpp                 |  4 +-
 test/test_flattener.cpp                   |  2 +-
 test/test_name_gen.cpp                    | 13 ++++++
 test/test_remove_members.cpp              |  2 +-
 types/shrd_ptr_type.toml                  |  6 +--
 types/uniq_ptr_type.toml                  |  6 +--
 17 files changed, 131 insertions(+), 33 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index d866df3..2037616 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -57,6 +57,7 @@ using type_graph::EnforceCompatibility;
 using type_graph::Enum;
 using type_graph::Flattener;
 using type_graph::IdentifyContainers;
+using type_graph::Incomplete;
 using type_graph::KeyCapture;
 using type_graph::Member;
 using type_graph::NameGen;
@@ -178,12 +179,20 @@ void genDeclsEnum(const Enum& e, std::string& code) {
   code += " {};\n";
 }
 
+void genDeclsIncomplete(const Incomplete& i, std::string& code) {
+  code += "template<> struct ";
+  code += i.name();
+  code += ";\n";
+}
+
 void genDecls(const TypeGraph& typeGraph, std::string& code) {
   for (const Type& t : typeGraph.finalTypes) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
       genDeclsClass(*c, code);
     } else if (const auto* e = dynamic_cast<const Enum*>(&t)) {
       genDeclsEnum(*e, code);
+    } else if (const auto* i = dynamic_cast<const Incomplete*>(&t)) {
+      genDeclsIncomplete(*i, code);
     }
   }
 }
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index 31a873b..416d81c 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -182,3 +182,18 @@ bool isStorageInline(const auto& c) {
   return (uintptr_t)std::data(c) < (uintptr_t)(&c + sizeof(c)) &&
          (uintptr_t)std::data(c) >= (uintptr_t)&c;
 }
+
+namespace OIInternal {
+namespace {
+
+template <typename T>
+struct Incomplete;
+
+template <typename T>
+constexpr bool oi_is_complete = true;
+
+template <typename T>
+constexpr bool oi_is_complete<Incomplete<T>> = false;
+
+}  // namespace
+}  // namespace OIInternal
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index b1d1e8d..1977e40 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -210,4 +210,9 @@ void NameGen::visit(CaptureKeys& c) {
   c.regenerateName();
 }
 
+void NameGen::visit(Incomplete& i) {
+  RecursiveVisitor::visit(i);
+  i.regenerateName();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..4857ded 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -48,6 +48,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
+  void visit(Incomplete& i) override;
 
   static const inline std::string AnonPrefix = "__oi_anon";
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index 0dae0ab..d7866b3 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -37,7 +37,9 @@ void Printer::print(const Type& type) {
 }
 
 void Printer::visit(const Incomplete& i) {
-  prefix();
+  if (prefix(i))
+    return;
+
   out_ << "Incomplete";
   if (auto underlyingType = i.underlyingType()) {
     out_ << std::endl;
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index 6ca10af..ff2715a 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -139,6 +139,10 @@ void TopoSorter::visit(CaptureKeys& c) {
   sortedTypes_.push_back(c);
 }
 
+void TopoSorter::visit(Incomplete& i) {
+  sortedTypes_.push_back(i);
+}
+
 /*
  * A type graph may contain cycles, so we need to slightly tweak the standard
  * topological sorting algorithm. Cycles can only be introduced by certain
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 374e943..44f2364 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -48,6 +48,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
+  void visit(Incomplete& i) override;
 
  private:
   std::unordered_set<Type*> visited_;
diff --git a/oi/type_graph/Types.cpp b/oi/type_graph/Types.cpp
index c6219b5..9b53f49 100644
--- a/oi/type_graph/Types.cpp
+++ b/oi/type_graph/Types.cpp
@@ -35,8 +35,6 @@ namespace oi::detail::type_graph {
 OI_TYPE_LIST
 #undef X
 
-const std::string Incomplete::kName = "void";
-
 std::string Primitive::getName(Kind kind) {
   switch (kind) {
     case Kind::Int8:
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f0b3fe9..76cfe6e 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -29,6 +29,7 @@
  * debugging.
  */
 
+#include <algorithm>
 #include <cstddef>
 #include <map>
 #include <optional>
@@ -210,29 +211,33 @@ struct TemplateParam {
  */
 class Incomplete : public Type {
  public:
-  Incomplete(Type& underlyingType) : underlyingType_(underlyingType) {
+  Incomplete(NodeId id, Type& underlyingType)
+      : id_(id), underlyingType_(underlyingType) {
   }
 
-  Incomplete(std::string underlyingTypeName)
-      : underlyingType_(std::move(underlyingTypeName)) {
+  Incomplete(NodeId id, std::string underlyingTypeName)
+      : id_(id), underlyingType_(std::move(underlyingTypeName)) {
   }
 
-  static inline constexpr bool has_node_id = false;
+  static inline constexpr bool has_node_id = true;
 
   DECLARE_ACCEPT
 
   const std::string& name() const override {
-    return kName;
+    return name_;
   }
 
   std::string_view inputName() const override {
-    if (std::holds_alternative<std::string>(underlyingType_)) {
-      return std::get<std::string>(underlyingType_);
-    }
-
-    return std::get<std::reference_wrapper<Type>>(underlyingType_)
-        .get()
-        .inputName();
+    return std::visit(
+        [](const auto& el) -> std::string_view {
+          using T = std::decay_t<decltype(el)>;
+          if constexpr (std::is_same_v<T, std::string>) {
+            return el;
+          } else {
+            return el.get().inputName();
+          }
+        },
+        underlyingType_);
   }
 
   size_t size() const override {
@@ -244,7 +249,26 @@ class Incomplete : public Type {
   }
 
   NodeId id() const override {
-    return -1;
+    return id_;
+  }
+
+  void regenerateName() {
+    std::string_view inputName = this->inputName();
+
+    constexpr std::string_view kPrefix{"Incomplete<struct "};
+    name_.clear();
+    name_.reserve(kPrefix.size() + inputName.size() + 2);
+
+    name_ = kPrefix;
+    for (const auto c : inputName) {
+      if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') ||
+          (c >= 'A' && c <= 'Z')) {
+        name_ += c;
+      } else {
+        name_ += '_';
+      }
+    }
+    name_ += ">";
   }
 
   std::optional<std::reference_wrapper<Type>> underlyingType() const {
@@ -256,8 +280,9 @@ class Incomplete : public Type {
   }
 
  private:
+  NodeId id_ = -1;
   std::variant<std::string, std::reference_wrapper<Type>> underlyingType_;
-  static const std::string kName;
+  std::string name_;
 };
 
 /*
diff --git a/test/TypeGraphParser.cpp b/test/TypeGraphParser.cpp
index f54fe08..d70236c 100644
--- a/test/TypeGraphParser.cpp
+++ b/test/TypeGraphParser.cpp
@@ -215,10 +215,11 @@ Type& TypeGraphParser::parseType(std::string_view& input, size_t rootIndent) {
       auto nameEndPos = line.find(']', nameStartPos);
       auto underlyingTypeName =
           line.substr(nameStartPos, nameEndPos - nameStartPos);
-      type = &typeGraph_.makeType<Incomplete>(std::string(underlyingTypeName));
+      type =
+          &typeGraph_.makeType<Incomplete>(id, std::string(underlyingTypeName));
     } else {
       auto& underlyingType = parseType(input, indent + 2);
-      type = &typeGraph_.makeType<Incomplete>(underlyingType);
+      type = &typeGraph_.makeType<Incomplete>(id, underlyingType);
     }
   } else if (nodeTypeName == "Class" || nodeTypeName == "Struct" ||
              nodeTypeName == "Union") {
diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml
index 0690511..2d34865 100644
--- a/test/integration/pointers_incomplete.toml
+++ b/test/integration/pointers_incomplete.toml
@@ -55,7 +55,6 @@ definitions = '''
     }]'''
 
   [cases.unique_ptr]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299
     param_types = ["const std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>&"]
     setup = '''
       auto raw_ptr = static_cast<IncompleteType*>(::operator new(5));
@@ -63,28 +62,29 @@ definitions = '''
         raw_ptr, &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
   [cases.unique_ptr_null]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299
     param_types = ["const std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>&"]
     setup = '''
       return std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>(
         nullptr, &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.shared_ptr]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300
     param_types = ["const std::shared_ptr<IncompleteType>&"]
     setup = '''
       auto raw_ptr = static_cast<IncompleteType*>(::operator new(5));
       return std::shared_ptr<IncompleteType>(raw_ptr , &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
   [cases.shared_ptr_null]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300
     param_types = ["const std::shared_ptr<IncompleteType>"]
     setup = "return nullptr;"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.containing_struct]
     oil_disable = "oil can't chase raw pointers safely"
@@ -116,3 +116,27 @@ definitions = '''
         }
       ]
     }]'''
+
+  [cases.containing_struct_no_follow]
+    param_types = ["const IncompleteTypeContainer&"]
+    setup = "return IncompleteTypeContainer{};"
+    expect_json = '''[{
+      "staticSize": 88,
+      "members": [
+        { "name": "ptrundef", "staticSize": 8 },
+        { "name": "__makePad1", "staticSize": 1 },
+        { "name": "shundef", "staticSize": 16 },
+        { "name": "__makePad2", "staticSize": 1 },
+        { "name": "shoptundef",
+          "staticSize": 24,
+          "length": 0,
+          "capacity": 1
+        },
+        { "name": "__makePad3", "staticSize": 1 },
+        { "name": "optundef",
+          "staticSize": 16,
+          "length": 0,
+          "capacity": 1
+        }
+      ]
+    }]'''
diff --git a/test/test_drgn_parser.cpp b/test/test_drgn_parser.cpp
index e8d03e3..c70f5a9 100644
--- a/test/test_drgn_parser.cpp
+++ b/test/test_drgn_parser.cpp
@@ -426,8 +426,8 @@ TEST_F(DrgnParserTest, PointerNoFollow) {
 
 TEST_F(DrgnParserTest, PointerIncomplete) {
   test("oid_test_case_pointers_incomplete_raw", R"(
-[0] Pointer
-      Incomplete: [IncompleteType]
+[1] Pointer
+[0]   Incomplete: [IncompleteType]
 )");
 }
 
diff --git a/test/test_flattener.cpp b/test/test_flattener.cpp
index 808f4ee..c8944bf 100644
--- a/test/test_flattener.cpp
+++ b/test/test_flattener.cpp
@@ -1007,7 +1007,7 @@ TEST(FlattenerTest, IncompleteParent) {
        R"(
 [0] Class: MyClass (size: 4)
       Parent (offset: 0)
-        Incomplete: [IncompleteParent]
+[1]     Incomplete: [IncompleteParent]
 )",
        R"(
 [0] Class: MyClass (size: 4)
diff --git a/test/test_name_gen.cpp b/test/test_name_gen.cpp
index d0cda26..a83c180 100644
--- a/test/test_name_gen.cpp
+++ b/test/test_name_gen.cpp
@@ -501,3 +501,16 @@ TEST(NameGenTest, AnonymousMembers) {
   EXPECT_EQ(myclass.members[0].inputName, "__oi_anon_0");
   EXPECT_EQ(myclass.members[1].inputName, "__oi_anon_1");
 }
+
+TEST(NameGenTest, IncompleteTypes) {
+  auto myincompletevector = Incomplete{0, "std::vector<int>"};
+
+  auto myint = Primitive{Primitive::Kind::Int32};
+  auto myincompleteint = Incomplete{1, myint};
+
+  NameGen nameGen;
+  nameGen.generateNames({myincompletevector, myincompleteint});
+
+  EXPECT_EQ(myincompletevector.name(), "Incomplete<struct std__vector_int_>");
+  EXPECT_EQ(myincompleteint.name(), "Incomplete<struct int32_t>");
+}
diff --git a/test/test_remove_members.cpp b/test/test_remove_members.cpp
index 0d4a54e..9eff4ed 100644
--- a/test/test_remove_members.cpp
+++ b/test/test_remove_members.cpp
@@ -191,7 +191,7 @@ TEST(RemoveMembersTest, Incomplete) {
       Member: a (offset: 0)
         Primitive: int32_t
       Member: b (offset: 4)
-        Incomplete: [MyIncompleteType]
+[1]     Incomplete: [MyIncompleteType]
       Member: c (offset: 8)
         Primitive: int32_t
 )",
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index bb3a622..de84b71 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (!std::is_void<T>::value && oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
         if (s_ptr && ctx.pointers.add((uintptr_t)(s_ptr.get()))) {
@@ -37,7 +37,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (std::is_void<T0>::value || !oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
@@ -85,7 +85,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index f723f80..09e351e 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T,Deleter>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (!std::is_void<T>::value && oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
         if (u_ptr && ctx.pointers.add((uintptr_t)(u_ptr.get()))) {
@@ -38,7 +38,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (std::is_void<T0>::value || !oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
@@ -71,7 +71,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (!std::is_void<T0>::value && oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);

From d9e10281696149b63b88de4eaf2421c0505cad72 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 19 Dec 2023 11:35:10 +0000
Subject: [PATCH 061/188] incomplete: name type in compiler errors

Summary:

We have a good type representation in the Type Graph of an incomplete type and
the underlying type that represents. However, this incomplete type still ends
up in the generated code as `void` which loses information. For example, a
container that can't contain void may fail to compile because it was
initialised with `void` but really its because the type it was supposed to be
initialised with (say, `Foo`) had incomplete debug information.

This change identifies that a type is incomplete in the output by generating it
as an incomplete type `struct Incomplete<struct Foo>`. This allows us to name
the type correctly in the TreeBuilder output and filter for incomplete types,
as well as getting appropriate compiler errors if it mustn't be incomplete.

Test Plan:
- CI
- Added a unit test to namegen.
- Enabled and added an extra pointers_incomplete test.

This change is tricky to test because it isn't really user visible. The types
still use their `inputName` which is unchanged in any successful output - this
change is used so the compiler fails with a more detailed error.
---
 oi/CodeGen.cpp                            |  9 ++++++
 oi/OITraceCode.cpp                        | 16 ++++++++++
 oi/type_graph/DrgnExporter.cpp            |  2 ++
 oi/type_graph/NameGen.cpp                 | 23 +++++++++++++
 oi/type_graph/NameGen.h                   |  1 +
 oi/type_graph/Printer.cpp                 |  4 ++-
 oi/type_graph/TopoSorter.cpp              |  4 +++
 oi/type_graph/TopoSorter.h                |  1 +
 oi/type_graph/Types.cpp                   |  2 --
 oi/type_graph/Types.h                     | 39 +++++++++++++++--------
 test/TypeGraphParser.cpp                  |  5 +--
 test/integration/pointers_incomplete.toml | 32 ++++++++++++++++---
 test/test_drgn_parser.cpp                 |  4 +--
 test/test_flattener.cpp                   |  2 +-
 test/test_name_gen.cpp                    | 13 ++++++++
 test/test_remove_members.cpp              |  2 +-
 types/shrd_ptr_type.toml                  |  6 ++--
 types/uniq_ptr_type.toml                  |  6 ++--
 18 files changed, 138 insertions(+), 33 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index d866df3..2037616 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -57,6 +57,7 @@ using type_graph::EnforceCompatibility;
 using type_graph::Enum;
 using type_graph::Flattener;
 using type_graph::IdentifyContainers;
+using type_graph::Incomplete;
 using type_graph::KeyCapture;
 using type_graph::Member;
 using type_graph::NameGen;
@@ -178,12 +179,20 @@ void genDeclsEnum(const Enum& e, std::string& code) {
   code += " {};\n";
 }
 
+void genDeclsIncomplete(const Incomplete& i, std::string& code) {
+  code += "template<> struct ";
+  code += i.name();
+  code += ";\n";
+}
+
 void genDecls(const TypeGraph& typeGraph, std::string& code) {
   for (const Type& t : typeGraph.finalTypes) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
       genDeclsClass(*c, code);
     } else if (const auto* e = dynamic_cast<const Enum*>(&t)) {
       genDeclsEnum(*e, code);
+    } else if (const auto* i = dynamic_cast<const Incomplete*>(&t)) {
+      genDeclsIncomplete(*i, code);
     }
   }
 }
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index 31a873b..a100900 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -182,3 +182,19 @@ bool isStorageInline(const auto& c) {
   return (uintptr_t)std::data(c) < (uintptr_t)(&c + sizeof(c)) &&
          (uintptr_t)std::data(c) >= (uintptr_t)&c;
 }
+
+namespace OIInternal {
+namespace {
+
+template <typename T>
+struct Incomplete;
+
+template <typename T>
+constexpr bool oi_is_complete = true;
+template <>
+constexpr bool oi_is_complete<void> = false;
+template <typename T>
+constexpr bool oi_is_complete<Incomplete<T>> = false;
+
+}  // namespace
+}  // namespace OIInternal
diff --git a/oi/type_graph/DrgnExporter.cpp b/oi/type_graph/DrgnExporter.cpp
index 802857e..e15fdda 100644
--- a/oi/type_graph/DrgnExporter.cpp
+++ b/oi/type_graph/DrgnExporter.cpp
@@ -100,6 +100,8 @@ drgn_type* DrgnExporter::visit(Container& c) {
     if (auto* p = dynamic_cast<const Primitive*>(&paramType);
         p && p->kind() == Primitive::Kind::Void) {
       return drgnType;
+    } else if (auto* p = dynamic_cast<const Incomplete*>(&paramType)) {
+      return drgnType;
     }
   }
 
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index b1d1e8d..ed797f0 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -210,4 +210,27 @@ void NameGen::visit(CaptureKeys& c) {
   c.regenerateName();
 }
 
+void NameGen::visit(Incomplete& i) {
+  constexpr std::string_view kPrefix{"Incomplete<struct "};
+
+  RecursiveVisitor::visit(i);
+
+  std::string_view inputName = i.inputName();
+
+  std::string name;
+  name.reserve(kPrefix.size() + inputName.size() + 2);
+  name = kPrefix;
+
+  for (const unsigned char c : inputName) {
+    if (std::isalnum(c)) {
+      name += c;
+    } else {
+      name += '_';
+    }
+  }
+  name += ">";
+
+  i.setName(name);
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..4857ded 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -48,6 +48,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
+  void visit(Incomplete& i) override;
 
   static const inline std::string AnonPrefix = "__oi_anon";
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index 0dae0ab..d7866b3 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -37,7 +37,9 @@ void Printer::print(const Type& type) {
 }
 
 void Printer::visit(const Incomplete& i) {
-  prefix();
+  if (prefix(i))
+    return;
+
   out_ << "Incomplete";
   if (auto underlyingType = i.underlyingType()) {
     out_ << std::endl;
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index 6ca10af..ff2715a 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -139,6 +139,10 @@ void TopoSorter::visit(CaptureKeys& c) {
   sortedTypes_.push_back(c);
 }
 
+void TopoSorter::visit(Incomplete& i) {
+  sortedTypes_.push_back(i);
+}
+
 /*
  * A type graph may contain cycles, so we need to slightly tweak the standard
  * topological sorting algorithm. Cycles can only be introduced by certain
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 374e943..44f2364 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -48,6 +48,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
+  void visit(Incomplete& i) override;
 
  private:
   std::unordered_set<Type*> visited_;
diff --git a/oi/type_graph/Types.cpp b/oi/type_graph/Types.cpp
index c6219b5..9b53f49 100644
--- a/oi/type_graph/Types.cpp
+++ b/oi/type_graph/Types.cpp
@@ -35,8 +35,6 @@ namespace oi::detail::type_graph {
 OI_TYPE_LIST
 #undef X
 
-const std::string Incomplete::kName = "void";
-
 std::string Primitive::getName(Kind kind) {
   switch (kind) {
     case Kind::Int8:
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f0b3fe9..9de1ce6 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -29,6 +29,8 @@
  * debugging.
  */
 
+#include <algorithm>
+#include <cctype>
 #include <cstddef>
 #include <map>
 #include <optional>
@@ -210,29 +212,33 @@ struct TemplateParam {
  */
 class Incomplete : public Type {
  public:
-  Incomplete(Type& underlyingType) : underlyingType_(underlyingType) {
+  Incomplete(NodeId id, Type& underlyingType)
+      : id_(id), underlyingType_(underlyingType) {
   }
 
-  Incomplete(std::string underlyingTypeName)
-      : underlyingType_(std::move(underlyingTypeName)) {
+  Incomplete(NodeId id, std::string underlyingTypeName)
+      : id_(id), underlyingType_(std::move(underlyingTypeName)) {
   }
 
-  static inline constexpr bool has_node_id = false;
+  static inline constexpr bool has_node_id = true;
 
   DECLARE_ACCEPT
 
   const std::string& name() const override {
-    return kName;
+    return name_;
   }
 
   std::string_view inputName() const override {
-    if (std::holds_alternative<std::string>(underlyingType_)) {
-      return std::get<std::string>(underlyingType_);
-    }
-
-    return std::get<std::reference_wrapper<Type>>(underlyingType_)
-        .get()
-        .inputName();
+    return std::visit(
+        [](const auto& el) -> std::string_view {
+          using T = std::decay_t<decltype(el)>;
+          if constexpr (std::is_same_v<T, std::string>) {
+            return el;
+          } else {
+            return el.get().inputName();
+          }
+        },
+        underlyingType_);
   }
 
   size_t size() const override {
@@ -244,7 +250,11 @@ class Incomplete : public Type {
   }
 
   NodeId id() const override {
-    return -1;
+    return id_;
+  }
+
+  void setName(std::string name) {
+    name_ = std::move(name);
   }
 
   std::optional<std::reference_wrapper<Type>> underlyingType() const {
@@ -256,8 +266,9 @@ class Incomplete : public Type {
   }
 
  private:
+  NodeId id_ = -1;
   std::variant<std::string, std::reference_wrapper<Type>> underlyingType_;
-  static const std::string kName;
+  std::string name_ = "void";
 };
 
 /*
diff --git a/test/TypeGraphParser.cpp b/test/TypeGraphParser.cpp
index f54fe08..d70236c 100644
--- a/test/TypeGraphParser.cpp
+++ b/test/TypeGraphParser.cpp
@@ -215,10 +215,11 @@ Type& TypeGraphParser::parseType(std::string_view& input, size_t rootIndent) {
       auto nameEndPos = line.find(']', nameStartPos);
       auto underlyingTypeName =
           line.substr(nameStartPos, nameEndPos - nameStartPos);
-      type = &typeGraph_.makeType<Incomplete>(std::string(underlyingTypeName));
+      type =
+          &typeGraph_.makeType<Incomplete>(id, std::string(underlyingTypeName));
     } else {
       auto& underlyingType = parseType(input, indent + 2);
-      type = &typeGraph_.makeType<Incomplete>(underlyingType);
+      type = &typeGraph_.makeType<Incomplete>(id, underlyingType);
     }
   } else if (nodeTypeName == "Class" || nodeTypeName == "Struct" ||
              nodeTypeName == "Union") {
diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml
index 0690511..2d34865 100644
--- a/test/integration/pointers_incomplete.toml
+++ b/test/integration/pointers_incomplete.toml
@@ -55,7 +55,6 @@ definitions = '''
     }]'''
 
   [cases.unique_ptr]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299
     param_types = ["const std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>&"]
     setup = '''
       auto raw_ptr = static_cast<IncompleteType*>(::operator new(5));
@@ -63,28 +62,29 @@ definitions = '''
         raw_ptr, &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
   [cases.unique_ptr_null]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299
     param_types = ["const std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>&"]
     setup = '''
       return std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>(
         nullptr, &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.shared_ptr]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300
     param_types = ["const std::shared_ptr<IncompleteType>&"]
     setup = '''
       auto raw_ptr = static_cast<IncompleteType*>(::operator new(5));
       return std::shared_ptr<IncompleteType>(raw_ptr , &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
   [cases.shared_ptr_null]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300
     param_types = ["const std::shared_ptr<IncompleteType>"]
     setup = "return nullptr;"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.containing_struct]
     oil_disable = "oil can't chase raw pointers safely"
@@ -116,3 +116,27 @@ definitions = '''
         }
       ]
     }]'''
+
+  [cases.containing_struct_no_follow]
+    param_types = ["const IncompleteTypeContainer&"]
+    setup = "return IncompleteTypeContainer{};"
+    expect_json = '''[{
+      "staticSize": 88,
+      "members": [
+        { "name": "ptrundef", "staticSize": 8 },
+        { "name": "__makePad1", "staticSize": 1 },
+        { "name": "shundef", "staticSize": 16 },
+        { "name": "__makePad2", "staticSize": 1 },
+        { "name": "shoptundef",
+          "staticSize": 24,
+          "length": 0,
+          "capacity": 1
+        },
+        { "name": "__makePad3", "staticSize": 1 },
+        { "name": "optundef",
+          "staticSize": 16,
+          "length": 0,
+          "capacity": 1
+        }
+      ]
+    }]'''
diff --git a/test/test_drgn_parser.cpp b/test/test_drgn_parser.cpp
index e8d03e3..c70f5a9 100644
--- a/test/test_drgn_parser.cpp
+++ b/test/test_drgn_parser.cpp
@@ -426,8 +426,8 @@ TEST_F(DrgnParserTest, PointerNoFollow) {
 
 TEST_F(DrgnParserTest, PointerIncomplete) {
   test("oid_test_case_pointers_incomplete_raw", R"(
-[0] Pointer
-      Incomplete: [IncompleteType]
+[1] Pointer
+[0]   Incomplete: [IncompleteType]
 )");
 }
 
diff --git a/test/test_flattener.cpp b/test/test_flattener.cpp
index 808f4ee..c8944bf 100644
--- a/test/test_flattener.cpp
+++ b/test/test_flattener.cpp
@@ -1007,7 +1007,7 @@ TEST(FlattenerTest, IncompleteParent) {
        R"(
 [0] Class: MyClass (size: 4)
       Parent (offset: 0)
-        Incomplete: [IncompleteParent]
+[1]     Incomplete: [IncompleteParent]
 )",
        R"(
 [0] Class: MyClass (size: 4)
diff --git a/test/test_name_gen.cpp b/test/test_name_gen.cpp
index d0cda26..a83c180 100644
--- a/test/test_name_gen.cpp
+++ b/test/test_name_gen.cpp
@@ -501,3 +501,16 @@ TEST(NameGenTest, AnonymousMembers) {
   EXPECT_EQ(myclass.members[0].inputName, "__oi_anon_0");
   EXPECT_EQ(myclass.members[1].inputName, "__oi_anon_1");
 }
+
+TEST(NameGenTest, IncompleteTypes) {
+  auto myincompletevector = Incomplete{0, "std::vector<int>"};
+
+  auto myint = Primitive{Primitive::Kind::Int32};
+  auto myincompleteint = Incomplete{1, myint};
+
+  NameGen nameGen;
+  nameGen.generateNames({myincompletevector, myincompleteint});
+
+  EXPECT_EQ(myincompletevector.name(), "Incomplete<struct std__vector_int_>");
+  EXPECT_EQ(myincompleteint.name(), "Incomplete<struct int32_t>");
+}
diff --git a/test/test_remove_members.cpp b/test/test_remove_members.cpp
index 0d4a54e..9eff4ed 100644
--- a/test/test_remove_members.cpp
+++ b/test/test_remove_members.cpp
@@ -191,7 +191,7 @@ TEST(RemoveMembersTest, Incomplete) {
       Member: a (offset: 0)
         Primitive: int32_t
       Member: b (offset: 4)
-        Incomplete: [MyIncompleteType]
+[1]     Incomplete: [MyIncompleteType]
       Member: c (offset: 8)
         Primitive: int32_t
 )",
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index bb3a622..3439bf3 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
         if (s_ptr && ctx.pointers.add((uintptr_t)(s_ptr.get()))) {
@@ -37,7 +37,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (!oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
@@ -85,7 +85,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index f723f80..f71cff4 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T,Deleter>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
         if (u_ptr && ctx.pointers.add((uintptr_t)(u_ptr.get()))) {
@@ -38,7 +38,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (!oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
@@ -71,7 +71,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);

From 3e41dcfeb8b24303b41e1ef0218e0eb75cbf2baa Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 19 Dec 2023 13:38:23 +0000
Subject: [PATCH 062/188] oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD
---
 CMakeLists.txt                                |  13 +-
 oi/CodeGen.cpp                                |  30 +-
 oi/CodeGen.h                                  |   8 +-
 oi/Config.cpp                                 |   9 +-
 oi/ContainerInfo.cpp                          |   2 +-
 oi/OICodeGen.h                                |   2 +-
 oi/OIGenerator.cpp                            | 364 ++++++++-------
 oi/OIGenerator.h                              |  29 +-
 oi/type_graph/CMakeLists.txt                  |   2 +
 oi/type_graph/ClangTypeParser.cpp             | 416 ++++++++++++++++++
 oi/type_graph/ClangTypeParser.h               | 131 ++++++
 oi/type_graph/NameGen.cpp                     |   8 +
 oi/type_graph/NameGen.h                       |   1 +
 oi/type_graph/Printer.cpp                     |  11 +
 oi/type_graph/Printer.h                       |   1 +
 oi/type_graph/RemoveTopLevelPointer.cpp       |   4 +
 oi/type_graph/RemoveTopLevelPointer.h         |   1 +
 oi/type_graph/Types.h                         |  56 +++
 oi/type_graph/Visitor.h                       |   7 +
 test/integration/template_template_param.toml |  16 +
 tools/OILGen.cpp                              | 148 +++----
 types/string_type.toml                        |  31 ++
 22 files changed, 1000 insertions(+), 290 deletions(-)
 create mode 100644 oi/type_graph/ClangTypeParser.cpp
 create mode 100644 oi/type_graph/ClangTypeParser.h
 create mode 100644 test/integration/template_template_param.toml

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f9ee7..eb18f4a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -104,6 +104,15 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(range-v3)
 
+### cxxopts
+FetchContent_Declare(
+  cxxopts
+  GIT_REPOSITORY https://github.com/jarro2783/cxxopts.git
+  GIT_TAG        eb787304d67ec22f7c3a184ee8b4c481d04357fd # v3.1.1
+  GIT_PROGRESS   TRUE
+)
+FetchContent_MakeAvailable(cxxopts)
+
 set_project_warnings()
 
 if (ASAN)
@@ -289,7 +298,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -363,8 +372,10 @@ add_executable(oilgen
   oi/OIGenerator.cpp
 )
 target_link_libraries(oilgen
+  cxxopts
   drgn_utils
   oicore
+  clangTooling
 )
 
 ### Object Introspection cache Printer (OIP)
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index ea8e4e3..20921ae 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -539,7 +539,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1106,15 +1106,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
 }
 
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
-  try {
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
+  if (!registerContainers())
     return false;
-  }
 
   TypeGraph typeGraph;
   try {
@@ -1129,6 +1122,20 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
   return true;
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    size_t i = 0;
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1170,7 +1177,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(IdentifyContainers::createPass(containerInfos_));
@@ -1286,7 +1293,8 @@ void CodeGen::generate(
   code += "\nusing __ROOT_TYPE__ = " + rootType.name() + ";\n";
   code += "} // namespace\n} // namespace OIInternal\n";
 
-  const auto typeName = SymbolService::getTypeName(drgnType);
+  // const auto typeName = SymbolService::getTypeName(drgnType);
+  const std::string typeName{"typeName"};
   if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
   } else {
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 186ec34..fb0ec20 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -40,8 +40,11 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config) : config_(config) {
+    assert(!config.features[Feature::PolymorphicInheritance]);
+  }
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   /*
@@ -53,6 +56,7 @@ class CodeGen {
                        std::string linkageName,
                        std::string& code);
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -66,7 +70,7 @@ class CodeGen {
 
  private:
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
diff --git a/oi/Config.cpp b/oi/Config.cpp
index 157b279..66798bb 100644
--- a/oi/Config.cpp
+++ b/oi/Config.cpp
@@ -22,6 +22,8 @@
 #include <boost/property_tree/ini_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 #include <filesystem>
+#include <range/v3/action/sort.hpp>
+#include <range/v3/action/unique.hpp>
 
 #include "oi/support/Toml.h"
 
@@ -54,6 +56,9 @@ std::optional<FeatureSet> processConfigFiles(
     enables |= *fs;
   }
 
+  ranges::actions::sort(generatorConfig.containerConfigPaths);
+  ranges::actions::unique(generatorConfig.containerConfigPaths);
+
   // Override anything from the config files with command line options
   for (auto [k, v] : featureMap) {
     enables[k] = v;
@@ -108,8 +113,8 @@ std::optional<FeatureSet> processConfigFile(
           if the right path is absolute, else append the right path to the
           left path.
           */
-          generatorConfig.containerConfigPaths.emplace(configDirectory /
-                                                       el.get());
+          generatorConfig.containerConfigPaths.emplace_back(configDirectory /
+                                                            el.get());
         }
       });
     }
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 99a7dc1..dbe10c6 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -202,7 +202,7 @@ std::regex getMatcher(const std::string& typeName) {
 ContainerInfo::ContainerInfo(const fs::path& path) {
   toml::table container;
   try {
-    container = toml::parse_file(std::string(path));
+    container = toml::parse_file(path.string());
   } catch (const toml::parse_error& err) {
     // Convert into a ContainerInfoError, just to avoid having to include
     // the huge TOML++ header in the caller's file. Use toml::parse_error's
diff --git a/oi/OICodeGen.h b/oi/OICodeGen.h
index a01548f..7226a4e 100644
--- a/oi/OICodeGen.h
+++ b/oi/OICodeGen.h
@@ -66,7 +66,7 @@ class OICodeGen {
     };
 
     FeatureSet features;
-    std::set<std::filesystem::path> containerConfigPaths;
+    std::vector<std::filesystem::path> containerConfigPaths;
     std::set<std::string> defaultHeaders;
     std::set<std::string> defaultNamespaces;
     std::vector<std::pair<std::string, std::string>> membersToStub;
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 1cb2a49..93054bc 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -16,130 +16,150 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
 #include <glog/logging.h>
 
 #include <boost/core/demangle.hpp>
 #include <fstream>
 #include <iostream>
+#include <range/v3/action/transform.hpp>
+#include <range/v3/core.hpp>
+#include <range/v3/view/cache1.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/drop_while.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>  // TODO: remove
 #include <string_view>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
+class ConsumerContext;
 
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
-      templateArgsToSymbolsMap;
-
-  auto symbols = prog.find_all_symbols();
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
+class CreateTypeGraphConsumer;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
-    if (val.first.empty() || val.second.empty()) {
-      continue;
-    }
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  return strongToWeakSymbols;
-}
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
 
-std::unordered_map<std::string, drgn_qualified_type>
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
+ private:
+  ConsumerContext& ctx;
+};
 
-  std::unordered_map<std::string, drgn_qualified_type> out;
+class ConsumerContext {
+ public:
+  ConsumerContext(const std::vector<std::unique_ptr<ContainerInfo>>& cis)
+      : containerInfos{cis} {
+  }
 
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
-    std::string strongLinkageName;
-    {
-      const char* linkageNameCstr;
-      if (auto err = drgnplusplus::error(
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
-        // throw err;
+  type_graph::TypeGraph typeGraph;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
+  std::optional<bool> pic;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+}  // namespace
+
+int OIGenerator::generate(clang::tooling::CompilationDatabase& db,
+                          const std::vector<std::string>& sourcePaths) {
+  std::map<Feature, bool> featuresMap = {
+      {Feature::TypeGraph, true},
+      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},
+      {Feature::PackStructs, true},
+      {Feature::PruneTypeGraph, true},
+  };
+
+  OICodeGen::Config generatorConfig{};
+  OICompiler::Config compilerConfig{};
+
+  auto features = config::processConfigFiles(
+      configFilePaths, featuresMap, compilerConfig, generatorConfig);
+  if (!features) {
+    LOG(ERROR) << "failed to process config file";
+    return -1;
+  }
+  generatorConfig.features = *features;
+  compilerConfig.features = *features;
+
+  std::vector<std::unique_ptr<ContainerInfo>> containerInfos;
+  containerInfos.reserve(generatorConfig.containerConfigPaths.size());
+  try {
+    for (const auto& path : generatorConfig.containerConfigPaths) {
+      auto info = std::make_unique<ContainerInfo>(path);
+      if (info->requiredFeatures != (*features & info->requiredFeatures)) {
+        VLOG(1) << "Skipping container (feature conflict): " << info->typeName;
         continue;
       }
-      strongLinkageName = linkageNameCstr;
+      containerInfos.emplace_back(std::move(info));
     }
-
-    std::string weakLinkageName;
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return -1;
   }
 
-  return out;
-}
+  ConsumerContext ctx{containerInfos};
+  CreateTypeGraphActionFactory factory{ctx};
 
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
-                                      const OICompiler::Config& compilerConfig,
-                                      const drgn_qualified_type& type,
-                                      const std::string& linkageName,
-                                      SymbolService& symbols) {
-  CodeGen codegen{generatorConfig, symbols};
+  clang::tooling::ClangTool tool{db, sourcePaths};
+  if (auto ret = tool.run(&factory); ret != 0) {
+    return ret;
+  }
+
+  if (ctx.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one type to generate for but we can't currently "
+        "handle this case");
+
+  compilerConfig.usePIC = ctx.pic.value();
+  CodeGen codegen{generatorConfig};
+  for (auto&& ptr : containerInfos)
+    codegen.registerContainer(std::move(ptr));
+  codegen.transform(ctx.typeGraph);
 
   std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
+  codegen.generate(ctx.typeGraph, code, nullptr);
 
   std::string sourcePath = sourceFileDumpPath;
   if (sourceFileDumpPath.empty()) {
@@ -152,78 +172,110 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
   }
 
   OICompiler compiler{{}, compilerConfig};
-
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
-
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
 }
 
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
-  drgnplusplus::program prog;
+namespace {
 
-  {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(
-            drgn_program_load_debug_info(prog.get(),
-                                         std::data(objectPaths),
-                                         std::size(objectPaths),
-                                         false,
-                                         false))) {
-      LOG(ERROR) << "error loading debug info program: " << err;
-      throw err;
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx(ctx_) {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
     }
-  }
 
-  auto oilTypes = findOilTypesAndNames(prog);
-
-  std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
-      {Feature::PruneTypeGraph, true},
-  };
-
-  OICodeGen::Config generatorConfig{};
-  OICompiler::Config compilerConfig{};
-  compilerConfig.usePIC = pic;
-
-  auto features = config::processConfigFiles(
-      configFilePaths, featuresMap, compilerConfig, generatorConfig);
-  if (!features) {
-    LOG(ERROR) << "failed to process config file";
-    return -1;
-  }
-  generatorConfig.features = *features;
-  compilerConfig.features = *features;
-
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
+    auto introspectImpl =
+        std::move(oi_namespaces) |
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
+        ranges::views::transform([](auto* p) {
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
+        }) |
+        ranges::views::filter([](auto* td) {
+          return td != nullptr && td->getName() == "introspectImpl";
+        }) |
+        ranges::views::take(1) | ranges::to<std::vector>();
+    if (introspectImpl.empty()) {
+      LOG(WARNING)
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
+             "you compile with `OIL_AOT_COMPILATION=1`?";
+      return;
     }
-  }
 
-  size_t successes = oilTypes.size() - failures;
-  LOG(INFO) << "object introspection generation complete. " << successes
-            << " successes and " << failures << " failures.";
+    auto nameToClangTypeMap =
+        ranges::views::single(introspectImpl[0]) |
+        ranges::views::for_each(
+            [](auto* td) { return td->specializations(); }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
 
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
-    return -1;
+              assert(fd->getNumParams() == 1);
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, opts};
+
+    auto& Sema = *ctx.sema;
+    ctx.nameToTypeMap =
+        nameToClangTypeMap |
+        ranges::views::transform(
+            [&parser, &Context, &Sema](
+                auto& p) -> std::pair<std::string, type_graph::Type*> {
+              return {p.first, &parser.parse(Context, Sema, *p.second)};
+            }) |
+        ranges::to<std::unordered_map>();
+
+    for (const auto& [name, type] : ctx.nameToTypeMap)
+      ctx.typeGraph.addRoot(*type);
   }
-  return 0;
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  // Compile the output as position independent if any input is position
+  // independent
+  bool pic = CI.getCodeGenOpts().RelocationModel == llvm::Reloc::PIC_;
+  ctx.pic = ctx.pic.value_or(false) || pic;
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
 }
 
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+}  // namespace
 }  // namespace oi::detail
diff --git a/oi/OIGenerator.h b/oi/OIGenerator.h
index dd38384..fac947a 100644
--- a/oi/OIGenerator.h
+++ b/oi/OIGenerator.h
@@ -23,11 +23,19 @@
 #include "oi/OICodeGen.h"
 #include "oi/OICompiler.h"
 
+namespace clang::tooling {
+class CompilationDatabase;
+}
+
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate(clang::tooling::CompilationDatabase&,
+               const std::vector<std::string>&);
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -41,8 +49,8 @@ class OIGenerator {
   void setFailIfNothingGenerated(bool fail) {
     failIfNothingGenerated = fail;
   }
-  void setUsePIC(bool pic_) {
-    pic = pic_;
+  void setClangArgs(std::vector<std::string> args_) {
+    clangArgs = std::move(args_);
   }
 
  private:
@@ -50,20 +58,7 @@ class OIGenerator {
   std::vector<std::filesystem::path> configFilePaths;
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
-  bool pic = false;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
+  std::vector<std::string> clangArgs;
 };
 
 }  // namespace oi::detail
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 940c031..46add03 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
   Flattener.cpp
@@ -26,3 +27,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
new file mode 100644
index 0000000..67f7bfa
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -0,0 +1,416 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Basic/DiagnosticSema.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <iostream>  // TODO: maybe remove
+#include <regex>
+#include <stdexcept>  // TODO: remove
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (VLOG_IS_ON(3)) {
+    std::string fqName = clang::TypeName::getFullyQualifiedName(
+        clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+    VLOG(3) << std::string(depth_ * 2, ' ') << fqName;
+  }
+
+  if (!requireCompleteType(*sema, ty))
+    return makeType<Incomplete>(ty, "incomplete (TODO naming)");
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(llvm::cast<const clang::RecordType>(ty));
+    case clang::Type::LValueReference:
+      return enumerateReference(
+          llvm::cast<const clang::LValueReferenceType>(ty));
+    case clang::Type::Pointer:
+      return enumeratePointer(llvm::cast<const clang::PointerType>(ty));
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          llvm::cast<const clang::SubstTemplateTypeParmType>(ty));
+    case clang::Type::Builtin:
+      return enumeratePrimitive(llvm::cast<const clang::BuiltinType>(ty));
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(
+          llvm::cast<const clang::ElaboratedType>(ty));
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          llvm::cast<const clang::TemplateSpecializationType>(ty));
+    case clang::Type::UnaryTransform:
+      return enumerateUnaryTransformType(
+          llvm::cast<const clang::UnaryTransformType>(ty));
+    case clang::Type::Decltype:
+      return enumerateDecltypeType(llvm::cast<const clang::DecltypeType>(ty));
+    case clang::Type::Typedef:
+      return enumerateTypedef(llvm::cast<const clang::TypedefType>(ty));
+    case clang::Type::Using:
+      return enumerateUsing(llvm::cast<const clang::UsingType>(ty));
+    case clang::Type::ConstantArray:
+      return enumerateArray(llvm::cast<const clang::ConstantArrayType>(ty));
+    case clang::Type::Enum:
+      return enumerateEnum(llvm::cast<const clang::EnumType>(ty));
+
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Type& ClangTypeParser::enumerateDecltypeType(const clang::DecltypeType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Type& ClangTypeParser::enumerateUnaryTransformType(
+    const clang::UnaryTransformType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Typedef& ClangTypeParser::enumerateUsing(const clang::UsingType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+  std::string name = ty.getFoundDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+
+  std::string name = ty.getDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string name = ty.getDecl()->getNameAsString();
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  std::map<int64_t, std::string> enumeratorMap;
+  if (options_.readEnumValues) {
+    for (const auto* enumerator : ty.getDecl()->enumerators()) {
+      enumeratorMap.emplace(enumerator->getInitVal().getExtValue(),
+                            enumerator->getNameAsString());
+    }
+  }
+
+  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+}
+
+Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
+  uint64_t len = ty.getSize().getLimitedValue();
+  auto& t = enumerateType(*ty.getElementType());
+  return makeType<Array>(ty, t, len);
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  if (auto* info = getContainerInfo(fqName)) {
+    auto& c = makeType<Container>(ty, *info, size, nullptr);
+    enumerateClassTemplateParams(ty, c.templateParams);
+    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)));
+    return c;
+  }
+
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+
+  params.reserve(list.size());
+  for (const auto& arg : list.asArray()) {
+    if (auto p = enumerateTemplateParam(arg))
+      params.emplace_back(std::move(p.value()));
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return TemplateParam{ttype, qualifiers};
+    }
+    case clang::TemplateArgument::Integral: {
+      auto& ty = enumerateType(*p.getIntegralType());
+      llvm::SmallString<32> val;
+      p.getAsIntegral().toString(val);
+      return TemplateParam{ty, std::string(val)};
+    }
+    case clang::TemplateArgument::Template: {
+      return enumerateTemplateTemplateParam(p.getAsTemplate());
+    }
+
+#define X(name)                       \
+  case clang::TemplateArgument::name: \
+    throw std::logic_error("unsupported template argument kind: " #name);
+
+      X(Null)
+      X(Declaration)
+      X(NullPtr)
+      X(TemplateExpansion)
+      X(Expression)
+      X(Pack)
+#undef X
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateTemplateParam(
+    const clang::TemplateName& tn) {
+  switch (tn.getKind()) {
+    case clang::TemplateName::Template:
+      return std::nullopt;
+
+#define X(name)                   \
+  case clang::TemplateName::name: \
+    throw std::logic_error("unsupported template name kind: " #name);
+
+      X(OverloadedTemplate)
+      X(AssumedTemplate)
+      X(QualifiedTemplate)
+      X(DependentTemplate)
+      X(SubstTemplateTemplateParm)
+      X(SubstTemplateTemplateParmPack)
+      X(UsingTemplate)
+#undef X
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function references
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Float:
+      return makeType<Primitive>(ty, Primitive::Kind::Float32);
+    case clang::BuiltinType::Double:
+    case clang::BuiltinType::LongDouble:
+      return makeType<Primitive>(ty, Primitive::Kind::Float64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+ContainerInfo* ClangTypeParser::getContainerInfo(
+    const std::string& fqName) const {
+  for (const auto& containerInfo : containers_) {
+    if (std::regex_search(fqName, containerInfo->matcher)) {
+      return containerInfo.get();
+    }
+  }
+  return nullptr;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
+    return true;  // treat as complete
+
+  // TODO: This is a terrible warning.
+  return !sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      clang::diag::warn_nsconsumed_attribute_mismatch);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
new file mode 100644
index 0000000..10eea87
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.h
@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ASTContext;
+class BuiltinType;
+class ConstantArrayType;
+class DecltypeType;
+class ElaboratedType;
+class EnumType;
+class LValueReferenceType;
+class PointerType;
+class RecordType;
+class Sema;
+class SubstTemplateTypeParmType;
+class TemplateArgument;
+class TemplateName;
+class TemplateSpecializationType;
+class Type;
+class TypedefType;
+class UnaryTransformType;
+class UsingType;
+}  // namespace clang
+
+struct ContainerInfo;
+
+namespace oi::detail::type_graph {
+
+class Array;
+class Class;
+class Enum;
+class Member;
+class Primitive;
+class Reference;
+class Type;
+class TypeGraph;
+class Typedef;
+struct TemplateParam;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+  bool readEnumValues = false;
+};
+
+/*
+ * ClangTypeParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given drgn_type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph,
+                  const std::vector<std::unique_ptr<ContainerInfo>>& containers,
+                  ClangTypeParserOptions options)
+      : typeGraph_{typeGraph}, containers_{containers}, options_{options} {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containers_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Type& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+  Typedef& enumerateTypedef(const clang::TypedefType&);
+  Typedef& enumerateUsing(const clang::UsingType&);
+  Type& enumerateUnaryTransformType(const clang::UnaryTransformType&);
+  Type& enumerateDecltypeType(const clang::DecltypeType&);
+
+  Array& enumerateArray(const clang::ConstantArrayType&);
+  Enum& enumerateEnum(const clang::EnumType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  std::optional<TemplateParam> enumerateTemplateParam(
+      const clang::TemplateArgument&);
+  std::optional<TemplateParam> enumerateTemplateTemplateParam(
+      const clang::TemplateName&);
+
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  ContainerInfo* getContainerInfo(const std::string& fqName) const;
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index b1d1e8d..f016c13 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -200,6 +200,14 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& p) {
+  RecursiveVisitor::visit(p);
+  p.regenerateName();
+  std::string inputName{p.pointeeType().inputName()};
+  inputName += '*';
+  p.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..e6437d0 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index 0dae0ab..49165ab 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -150,6 +150,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& p) {
+  if (prefix(p))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = p.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(p.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index 8d7138a..e08a943 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& p) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;
diff --git a/oi/type_graph/RemoveTopLevelPointer.cpp b/oi/type_graph/RemoveTopLevelPointer.cpp
index 9ecaaaa..e1909a7 100644
--- a/oi/type_graph/RemoveTopLevelPointer.cpp
+++ b/oi/type_graph/RemoveTopLevelPointer.cpp
@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& p) {
+  topLevelType_ = &p.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/RemoveTopLevelPointer.h b/oi/type_graph/RemoveTopLevelPointer.h
index 611429f..6b70545 100644
--- a/oi/type_graph/RemoveTopLevelPointer.h
+++ b/oi/type_graph/RemoveTopLevelPointer.h
@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
 
  private:
   Type* topLevelType_ = nullptr;
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index e05d224..18e2d38 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -733,6 +734,61 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    name_ = pointeeType_.get().name() + "&";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index a7f7eec..f055aea 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -108,6 +108,9 @@ class RecursiveVisitor : public Visitor<void> {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -175,6 +178,10 @@ class RecursiveMutator : public Visitor<Type&> {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }
diff --git a/test/integration/template_template_param.toml b/test/integration/template_template_param.toml
new file mode 100644
index 0000000..c64590d
--- /dev/null
+++ b/test/integration/template_template_param.toml
@@ -0,0 +1,16 @@
+includes = ["vector", "utility", "string"]
+
+definitions = '''
+template <typename K, typename V, template <typename, typename> typename Pair>
+struct bad_map {
+  std::vector<Pair<K, K>> keys;
+  std::vector<Pair<V, V>> values;
+};
+'''
+
+[cases]
+  [cases.int_int_empty]
+    param_types = ["const bad_map<int, std::string, std::pair>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index d1e345d..795389e 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -14,131 +14,81 @@
  * limitations under the License.
  */
 
+#include <clang/Tooling/CommonOptionsParser.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include <cstdlib>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/action/drop_while.hpp>
+#include <range/v3/algorithm/find.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/all.hpp>
+#include <range/v3/view/transform.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
 
-constexpr static OIOpts opts{
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
+static llvm::cl::OptionCategory OilgenCategory("oilgen options");
 
-void usage() {
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
-  std::cerr << opts;
+static llvm::cl::list<std::string> ConfigFiles("config-file",
+                                               llvm::cl::desc(R"(TODO HELP)"),
+                                               llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<int> DebugLevel("debug-level",
+                                     llvm::cl::desc(R"(TODO HELP)"),
+                                     llvm::cl::init(-1),
+                                     llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> DumpJit("dump-jit",
+                                          llvm::cl::desc(R"(TODO HELP)"),
+                                          llvm::cl::init("jit.cpp"),
+                                          llvm::cl::cat(OilgenCategory));
 
-  std::cerr << std::endl
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
-
-int main(int argc, char* argv[]) {
+int main(int argc, const char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
-  std::vector<fs::path> configFilePaths;
-  fs::path sourceFileDumpPath = "";
-  bool exitCode = false;
-  bool pic = false;
-
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
+  auto expectedParser =
+      clang::tooling::CommonOptionsParser::create(argc, argv, OilgenCategory);
+  if (!expectedParser) {
+    llvm::errs() << expectedParser.takeError();
+    return -1;
   }
+  clang::tooling::CommonOptionsParser& options = expectedParser.get();
+  auto& compilations = options.getCompilations();
 
-  if (optind >= argc) {
-    usage();
-    return EXIT_FAILURE;
-  }
-  fs::path primaryObject = argv[optind];
-
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
+  if (DebugLevel.getNumOccurrences()) {
+    google::LogToStderr();
+    google::SetStderrLogging(google::INFO);
+    google::SetVLOGLevel("*", DebugLevel);
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
+    gflags::SetCommandLineOption("minloglevel", "0");
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
-  oigen.setConfigFilePaths(std::move(configFilePaths));
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
-  oigen.setFailIfNothingGenerated(exitCode);
-  oigen.setUsePIC(pic);
+  oigen.setConfigFilePaths(ConfigFiles |
+                           ranges::views::transform([](const auto& p) {
+                             return std::filesystem::path(p);
+                           }) |
+                           ranges::to<std::vector>());
+  if (DumpJit.getNumOccurrences())
+    oigen.setSourceFileDumpPath(DumpJit.getValue());
 
-  SymbolService symbols(primaryObject);
+  oigen.setOutputPath("a.o");  // TODO: make this an opt
 
-  return oigen.generate(primaryObject, symbols);
+  // TODO: can we extract this from the original arguments?
+  // oigen.setUsePIC(args["pic"].as<bool>());
+
+  oigen.setFailIfNothingGenerated(true);
+  // compilations.getAllFiles();
+  // return oigen.generate(compilations,
+  // {"/data/users/jakehillion/fbsource/fbcode/object-introspection/oil/examples/compile-time/VectorOfStrings-Introspect.cpp"});
+  return oigen.generate(compilations, options.getSourcePathList());
 }
diff --git a/types/string_type.toml b/types/string_type.toml
index fa4a2ae..0d2ab0d 100644
--- a/types/string_type.toml
+++ b/types/string_type.toml
@@ -35,3 +35,34 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
     );
 }
 """
+
+traversal_func = """
+    bool sso = ((uintptr_t)container.data() <
+                (uintptr_t)(&container + sizeof(std::__cxx11::basic_string<T0>))) &&
+               ((uintptr_t)container.data() >= (uintptr_t)&container);
+
+    return returnArg.write(container.capacity())
+        .write(sso)
+        .write(container.size());
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+uint64_t capacity = std::get<ParsedData::VarInt>(d.val).value;
+el.container_stats.emplace(result::Element::ContainerStats { .capacity = capacity });
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+bool sso = std::get<ParsedData::VarInt>(d.val).value;
+if (!sso)
+  el.exclusive_size += el.container_stats->capacity * sizeof(T0);
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats->length = std::get<ParsedData::VarInt>(d.val).value;
+"""

From efe65597e43d8419f17e1f43f17d3e9c7288b20f Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 19 Dec 2023 14:09:53 +0000
Subject: [PATCH 063/188] codegen: remove reliance on drgn type for top level
 name

---
 oi/CodeGen.cpp | 46 ++++++++++++++++++++++++++++++++--------------
 oi/CodeGen.h   | 18 ++++++++++++++----
 2 files changed, 46 insertions(+), 18 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index d866df3..74c6f39 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -43,6 +43,9 @@
 #include "type_graph/TypeIdentifier.h"
 #include "type_graph/Types.h"
 
+template <typename T>
+inline constexpr bool always_false_v = false;
+
 namespace oi::detail {
 
 using type_graph::AddChildren;
@@ -1101,11 +1104,17 @@ void CodeGen::addTypeHandlers(const TypeGraph& typeGraph, std::string& code) {
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
                               std::string linkageName,
                               std::string& code) {
-  linkageName_ = std::move(linkageName);
-  return codegenFromDrgn(drgnType, code);
+  return codegenFromDrgn(drgnType, code, ExactName{std::move(linkageName)});
 }
 
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
+  return codegenFromDrgn(
+      drgnType, code, HashedComponent{SymbolService::getTypeName(drgnType)});
+}
+
+bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
+                              std::string& code,
+                              RootFunctionName name) {
   try {
     containerInfos_.reserve(config_.containerConfigPaths.size());
     for (const auto& path : config_.containerConfigPaths) {
@@ -1124,7 +1133,7 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
   }
 
   transform(typeGraph_);
-  generate(typeGraph_, code, drgnType);
+  generate(typeGraph_, code, std::move(name));
   return true;
 }
 
@@ -1213,11 +1222,9 @@ void CodeGen::transform(TypeGraph& typeGraph) {
   };
 }
 
-void CodeGen::generate(
-    TypeGraph& typeGraph,
-    std::string& code,
-    struct drgn_type* drgnType /* TODO: this argument should not be required */
-) {
+void CodeGen::generate(TypeGraph& typeGraph,
+                       std::string& code,
+                       RootFunctionName rootName) {
   code = headers::oi_OITraceCode_cpp;
   if (!config_.features[Feature::Library]) {
     FuncGen::DeclareExterns(code);
@@ -1296,22 +1303,33 @@ void CodeGen::generate(
   code += "\nusing __ROOT_TYPE__ = " + rootType.name() + ";\n";
   code += "} // namespace\n} // namespace OIInternal\n";
 
-  const auto typeName = SymbolService::getTypeName(drgnType);
+  const auto& typeToHash = std::visit(
+      [](const auto& v) -> const std::string& {
+        using T = std::decay_t<decltype(v)>;
+        if constexpr (std::is_same_v<ExactName, T> ||
+                      std::is_same_v<HashedComponent, T>) {
+          return v.name;
+        } else {
+          static_assert(always_false_v<T>, "missing visit");
+        }
+      },
+      rootName);
+
   if (config_.features[Feature::TreeBuilderV2]) {
-    FuncGen::DefineTopLevelIntrospect(code, typeName);
+    FuncGen::DefineTopLevelIntrospect(code, typeToHash);
   } else {
-    FuncGen::DefineTopLevelGetSizeRef(code, typeName, config_.features);
+    FuncGen::DefineTopLevelGetSizeRef(code, typeToHash, config_.features);
   }
 
   if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTreeBuilderInstructions(code,
-                                           typeName,
+                                           typeToHash,
                                            calculateExclusiveSize(rootType),
                                            enumerateTypeNames(rootType));
   }
 
-  if (!linkageName_.empty())
-    FuncGen::DefineTopLevelIntrospectNamed(code, typeName, linkageName_);
+  if (auto* n = std::get_if<ExactName>(&rootName))
+    FuncGen::DefineTopLevelIntrospectNamed(code, typeToHash, n->name);
 
   if (VLOG_IS_ON(3)) {
     VLOG(3) << "Generated trace code:\n";
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 2c9b5e1..29df5b9 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -40,6 +40,15 @@ class Member;
 namespace oi::detail {
 
 class CodeGen {
+ private:
+  struct ExactName {
+    std::string name;
+  };
+  struct HashedComponent {
+    std::string name;
+  };
+  using RootFunctionName = std::variant<ExactName, HashedComponent>;
+
  public:
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
       : config_(config), symbols_(symbols) {
@@ -64,9 +73,7 @@ class CodeGen {
   void transform(type_graph::TypeGraph& typeGraph);
   void generate(type_graph::TypeGraph& typeGraph,
                 std::string& code,
-                struct drgn_type*
-                    drgnType /* TODO: this argument should not be required */
-  );
+                RootFunctionName rootName);
 
  private:
   type_graph::TypeGraph typeGraph_;
@@ -76,7 +83,10 @@ class CodeGen {
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
       thriftIssetMembers_;
-  std::string linkageName_;
+
+  bool codegenFromDrgn(struct drgn_type* drgnType,
+                       std::string& code,
+                       RootFunctionName name);
 
   void genDefsThrift(const type_graph::TypeGraph& typeGraph, std::string& code);
   void addGetSizeFuncDefs(const type_graph::TypeGraph& typeGraph,

From 9aa0f9b62ba22933a35d8cfe3cc72ff7e830f2cb Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 19 Dec 2023 14:44:39 +0000
Subject: [PATCH 064/188] codegen: remove reliance on drgn type for top level
 name

Currently we rely on `SymbolService::getTypeName` for getting the hash that's
included in the generated function's name. The value of this must stay the same
to match with the value expected by OIDebugger - changing it causes failure to
relocate when attaching with OID and JIT OIL.

Calculate this name in the `codegenFromDrgn` method and pass it through where
appropriate rather than passing the `drgn_type` itself through.

We don't need to name the type like that when using AoT OIL. Let's
hash the linkage name instead as that is more unique.

Test Plan:
- CI
- Ran a quick AoT OIL test, it's fine.
---
 oi/CodeGen.cpp | 46 ++++++++++++++++++++++++++++++++--------------
 oi/CodeGen.h   | 18 ++++++++++++++----
 2 files changed, 46 insertions(+), 18 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index d866df3..74c6f39 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -43,6 +43,9 @@
 #include "type_graph/TypeIdentifier.h"
 #include "type_graph/Types.h"
 
+template <typename T>
+inline constexpr bool always_false_v = false;
+
 namespace oi::detail {
 
 using type_graph::AddChildren;
@@ -1101,11 +1104,17 @@ void CodeGen::addTypeHandlers(const TypeGraph& typeGraph, std::string& code) {
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
                               std::string linkageName,
                               std::string& code) {
-  linkageName_ = std::move(linkageName);
-  return codegenFromDrgn(drgnType, code);
+  return codegenFromDrgn(drgnType, code, ExactName{std::move(linkageName)});
 }
 
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
+  return codegenFromDrgn(
+      drgnType, code, HashedComponent{SymbolService::getTypeName(drgnType)});
+}
+
+bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
+                              std::string& code,
+                              RootFunctionName name) {
   try {
     containerInfos_.reserve(config_.containerConfigPaths.size());
     for (const auto& path : config_.containerConfigPaths) {
@@ -1124,7 +1133,7 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
   }
 
   transform(typeGraph_);
-  generate(typeGraph_, code, drgnType);
+  generate(typeGraph_, code, std::move(name));
   return true;
 }
 
@@ -1213,11 +1222,9 @@ void CodeGen::transform(TypeGraph& typeGraph) {
   };
 }
 
-void CodeGen::generate(
-    TypeGraph& typeGraph,
-    std::string& code,
-    struct drgn_type* drgnType /* TODO: this argument should not be required */
-) {
+void CodeGen::generate(TypeGraph& typeGraph,
+                       std::string& code,
+                       RootFunctionName rootName) {
   code = headers::oi_OITraceCode_cpp;
   if (!config_.features[Feature::Library]) {
     FuncGen::DeclareExterns(code);
@@ -1296,22 +1303,33 @@ void CodeGen::generate(
   code += "\nusing __ROOT_TYPE__ = " + rootType.name() + ";\n";
   code += "} // namespace\n} // namespace OIInternal\n";
 
-  const auto typeName = SymbolService::getTypeName(drgnType);
+  const auto& typeToHash = std::visit(
+      [](const auto& v) -> const std::string& {
+        using T = std::decay_t<decltype(v)>;
+        if constexpr (std::is_same_v<ExactName, T> ||
+                      std::is_same_v<HashedComponent, T>) {
+          return v.name;
+        } else {
+          static_assert(always_false_v<T>, "missing visit");
+        }
+      },
+      rootName);
+
   if (config_.features[Feature::TreeBuilderV2]) {
-    FuncGen::DefineTopLevelIntrospect(code, typeName);
+    FuncGen::DefineTopLevelIntrospect(code, typeToHash);
   } else {
-    FuncGen::DefineTopLevelGetSizeRef(code, typeName, config_.features);
+    FuncGen::DefineTopLevelGetSizeRef(code, typeToHash, config_.features);
   }
 
   if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTreeBuilderInstructions(code,
-                                           typeName,
+                                           typeToHash,
                                            calculateExclusiveSize(rootType),
                                            enumerateTypeNames(rootType));
   }
 
-  if (!linkageName_.empty())
-    FuncGen::DefineTopLevelIntrospectNamed(code, typeName, linkageName_);
+  if (auto* n = std::get_if<ExactName>(&rootName))
+    FuncGen::DefineTopLevelIntrospectNamed(code, typeToHash, n->name);
 
   if (VLOG_IS_ON(3)) {
     VLOG(3) << "Generated trace code:\n";
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 2c9b5e1..29df5b9 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -40,6 +40,15 @@ class Member;
 namespace oi::detail {
 
 class CodeGen {
+ private:
+  struct ExactName {
+    std::string name;
+  };
+  struct HashedComponent {
+    std::string name;
+  };
+  using RootFunctionName = std::variant<ExactName, HashedComponent>;
+
  public:
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
       : config_(config), symbols_(symbols) {
@@ -64,9 +73,7 @@ class CodeGen {
   void transform(type_graph::TypeGraph& typeGraph);
   void generate(type_graph::TypeGraph& typeGraph,
                 std::string& code,
-                struct drgn_type*
-                    drgnType /* TODO: this argument should not be required */
-  );
+                RootFunctionName rootName);
 
  private:
   type_graph::TypeGraph typeGraph_;
@@ -76,7 +83,10 @@ class CodeGen {
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
       thriftIssetMembers_;
-  std::string linkageName_;
+
+  bool codegenFromDrgn(struct drgn_type* drgnType,
+                       std::string& code,
+                       RootFunctionName name);
 
   void genDefsThrift(const type_graph::TypeGraph& typeGraph, std::string& code);
   void addGetSizeFuncDefs(const type_graph::TypeGraph& typeGraph,

From 51bddceea671e714470b80fe804a9a186fcfe18b Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 19 Dec 2023 15:00:35 +0000
Subject: [PATCH 065/188] codegen: remove reliance on drgn type for top level
 name

Currently we rely on `SymbolService::getTypeName` for getting the hash that's
included in the generated function's name. The value of this must stay the same
to match with the value expected by OIDebugger - changing it causes failure to
relocate when attaching with OID and JIT OIL.

Calculate this name in the `codegenFromDrgn` method and pass it through where
appropriate rather than passing the `drgn_type` itself through.

We don't need to name the type like that when using AoT OIL. Let's
hash the linkage name instead as that is more unique.

Test Plan:
- CI
---
 oi/CodeGen.cpp | 46 ++++++++++++++++++++++++++++++++--------------
 oi/CodeGen.h   | 18 ++++++++++++++----
 2 files changed, 46 insertions(+), 18 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index d866df3..74c6f39 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -43,6 +43,9 @@
 #include "type_graph/TypeIdentifier.h"
 #include "type_graph/Types.h"
 
+template <typename T>
+inline constexpr bool always_false_v = false;
+
 namespace oi::detail {
 
 using type_graph::AddChildren;
@@ -1101,11 +1104,17 @@ void CodeGen::addTypeHandlers(const TypeGraph& typeGraph, std::string& code) {
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
                               std::string linkageName,
                               std::string& code) {
-  linkageName_ = std::move(linkageName);
-  return codegenFromDrgn(drgnType, code);
+  return codegenFromDrgn(drgnType, code, ExactName{std::move(linkageName)});
 }
 
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
+  return codegenFromDrgn(
+      drgnType, code, HashedComponent{SymbolService::getTypeName(drgnType)});
+}
+
+bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
+                              std::string& code,
+                              RootFunctionName name) {
   try {
     containerInfos_.reserve(config_.containerConfigPaths.size());
     for (const auto& path : config_.containerConfigPaths) {
@@ -1124,7 +1133,7 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
   }
 
   transform(typeGraph_);
-  generate(typeGraph_, code, drgnType);
+  generate(typeGraph_, code, std::move(name));
   return true;
 }
 
@@ -1213,11 +1222,9 @@ void CodeGen::transform(TypeGraph& typeGraph) {
   };
 }
 
-void CodeGen::generate(
-    TypeGraph& typeGraph,
-    std::string& code,
-    struct drgn_type* drgnType /* TODO: this argument should not be required */
-) {
+void CodeGen::generate(TypeGraph& typeGraph,
+                       std::string& code,
+                       RootFunctionName rootName) {
   code = headers::oi_OITraceCode_cpp;
   if (!config_.features[Feature::Library]) {
     FuncGen::DeclareExterns(code);
@@ -1296,22 +1303,33 @@ void CodeGen::generate(
   code += "\nusing __ROOT_TYPE__ = " + rootType.name() + ";\n";
   code += "} // namespace\n} // namespace OIInternal\n";
 
-  const auto typeName = SymbolService::getTypeName(drgnType);
+  const auto& typeToHash = std::visit(
+      [](const auto& v) -> const std::string& {
+        using T = std::decay_t<decltype(v)>;
+        if constexpr (std::is_same_v<ExactName, T> ||
+                      std::is_same_v<HashedComponent, T>) {
+          return v.name;
+        } else {
+          static_assert(always_false_v<T>, "missing visit");
+        }
+      },
+      rootName);
+
   if (config_.features[Feature::TreeBuilderV2]) {
-    FuncGen::DefineTopLevelIntrospect(code, typeName);
+    FuncGen::DefineTopLevelIntrospect(code, typeToHash);
   } else {
-    FuncGen::DefineTopLevelGetSizeRef(code, typeName, config_.features);
+    FuncGen::DefineTopLevelGetSizeRef(code, typeToHash, config_.features);
   }
 
   if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTreeBuilderInstructions(code,
-                                           typeName,
+                                           typeToHash,
                                            calculateExclusiveSize(rootType),
                                            enumerateTypeNames(rootType));
   }
 
-  if (!linkageName_.empty())
-    FuncGen::DefineTopLevelIntrospectNamed(code, typeName, linkageName_);
+  if (auto* n = std::get_if<ExactName>(&rootName))
+    FuncGen::DefineTopLevelIntrospectNamed(code, typeToHash, n->name);
 
   if (VLOG_IS_ON(3)) {
     VLOG(3) << "Generated trace code:\n";
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 2c9b5e1..29df5b9 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -40,6 +40,15 @@ class Member;
 namespace oi::detail {
 
 class CodeGen {
+ private:
+  struct ExactName {
+    std::string name;
+  };
+  struct HashedComponent {
+    std::string name;
+  };
+  using RootFunctionName = std::variant<ExactName, HashedComponent>;
+
  public:
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
       : config_(config), symbols_(symbols) {
@@ -64,9 +73,7 @@ class CodeGen {
   void transform(type_graph::TypeGraph& typeGraph);
   void generate(type_graph::TypeGraph& typeGraph,
                 std::string& code,
-                struct drgn_type*
-                    drgnType /* TODO: this argument should not be required */
-  );
+                RootFunctionName rootName);
 
  private:
   type_graph::TypeGraph typeGraph_;
@@ -76,7 +83,10 @@ class CodeGen {
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
       thriftIssetMembers_;
-  std::string linkageName_;
+
+  bool codegenFromDrgn(struct drgn_type* drgnType,
+                       std::string& code,
+                       RootFunctionName name);
 
   void genDefsThrift(const type_graph::TypeGraph& typeGraph, std::string& code);
   void addGetSizeFuncDefs(const type_graph::TypeGraph& typeGraph,

From ec21473c8263020b4e8b5f36322f9d252c25e597 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 19 Dec 2023 15:02:10 +0000
Subject: [PATCH 066/188] oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD
---
 CMakeLists.txt                                |   3 +-
 oi/CodeGen.cpp                                |  33 +-
 oi/CodeGen.h                                  |   6 +-
 oi/Config.cpp                                 |   9 +-
 oi/ContainerInfo.cpp                          |   2 +-
 oi/OICodeGen.h                                |   2 +-
 oi/OIGenerator.cpp                            | 364 ++++++++-------
 oi/OIGenerator.h                              |  29 +-
 oi/type_graph/CMakeLists.txt                  |   2 +
 oi/type_graph/ClangTypeParser.cpp             | 416 ++++++++++++++++++
 oi/type_graph/ClangTypeParser.h               | 131 ++++++
 oi/type_graph/DrgnExporter.cpp                |   8 +
 oi/type_graph/DrgnExporter.h                  |   1 +
 oi/type_graph/NameGen.cpp                     |   8 +
 oi/type_graph/NameGen.h                       |   1 +
 oi/type_graph/Printer.cpp                     |  11 +
 oi/type_graph/Printer.h                       |   1 +
 oi/type_graph/RemoveTopLevelPointer.cpp       |   4 +
 oi/type_graph/RemoveTopLevelPointer.h         |   1 +
 oi/type_graph/Types.h                         |  56 +++
 oi/type_graph/Visitor.h                       |   7 +
 test/integration/template_template_param.toml |  16 +
 tools/OILGen.cpp                              | 148 +++----
 types/string_type.toml                        |  31 ++
 24 files changed, 1000 insertions(+), 290 deletions(-)
 create mode 100644 oi/type_graph/ClangTypeParser.cpp
 create mode 100644 oi/type_graph/ClangTypeParser.h
 create mode 100644 test/integration/template_template_param.toml

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f9ee7..b6fecd1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -289,7 +289,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -365,6 +365,7 @@ add_executable(oilgen
 target_link_libraries(oilgen
   drgn_utils
   oicore
+  clangTooling
 )
 
 ### Object Introspection cache Printer (OIP)
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index d866df3..6492756 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -296,6 +296,10 @@ void genDefsThriftClass(const Class& c, std::string& code) {
 
 }  // namespace
 
+CodeGen::CodeGen(const OICodeGen::Config& config) : config_(config) {
+  DCHECK(!config.features[Feature::PolymorphicInheritance]) << "polymorphic inheritance requires symbol service!";
+}
+
 void CodeGen::genDefsThrift(const TypeGraph& typeGraph, std::string& code) {
   for (const Type& t : typeGraph.finalTypes) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
@@ -539,7 +543,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1106,15 +1110,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
 }
 
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
-  try {
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
+  if (!registerContainers())
     return false;
-  }
 
   try {
     addDrgnRoot(drgnType, typeGraph_);
@@ -1139,6 +1136,19 @@ void CodeGen::exportDrgnTypes(TypeHierarchy& th,
   }
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1180,7 +1190,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(IdentifyContainers::createPass(containerInfos_));
@@ -1296,7 +1306,8 @@ void CodeGen::generate(
   code += "\nusing __ROOT_TYPE__ = " + rootType.name() + ";\n";
   code += "} // namespace\n} // namespace OIInternal\n";
 
-  const auto typeName = SymbolService::getTypeName(drgnType);
+  // const auto typeName = SymbolService::getTypeName(drgnType);
+  const std::string typeName{"typeName"};
   if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTopLevelIntrospect(code, typeName);
   } else {
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 2c9b5e1..5b02bb2 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -41,8 +41,9 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config);
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   /*
@@ -57,6 +58,7 @@ class CodeGen {
                        std::list<drgn_type>& drgnTypes,
                        drgn_type** rootType) const;
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -71,7 +73,7 @@ class CodeGen {
  private:
   type_graph::TypeGraph typeGraph_;
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
diff --git a/oi/Config.cpp b/oi/Config.cpp
index 157b279..66798bb 100644
--- a/oi/Config.cpp
+++ b/oi/Config.cpp
@@ -22,6 +22,8 @@
 #include <boost/property_tree/ini_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 #include <filesystem>
+#include <range/v3/action/sort.hpp>
+#include <range/v3/action/unique.hpp>
 
 #include "oi/support/Toml.h"
 
@@ -54,6 +56,9 @@ std::optional<FeatureSet> processConfigFiles(
     enables |= *fs;
   }
 
+  ranges::actions::sort(generatorConfig.containerConfigPaths);
+  ranges::actions::unique(generatorConfig.containerConfigPaths);
+
   // Override anything from the config files with command line options
   for (auto [k, v] : featureMap) {
     enables[k] = v;
@@ -108,8 +113,8 @@ std::optional<FeatureSet> processConfigFile(
           if the right path is absolute, else append the right path to the
           left path.
           */
-          generatorConfig.containerConfigPaths.emplace(configDirectory /
-                                                       el.get());
+          generatorConfig.containerConfigPaths.emplace_back(configDirectory /
+                                                            el.get());
         }
       });
     }
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 4cb20b8..b83b7e4 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -202,7 +202,7 @@ std::regex getMatcher(const std::string& typeName) {
 ContainerInfo::ContainerInfo(const fs::path& path) {
   toml::table container;
   try {
-    container = toml::parse_file(std::string(path));
+    container = toml::parse_file(path.string());
   } catch (const toml::parse_error& err) {
     // Convert into a ContainerInfoError, just to avoid having to include
     // the huge TOML++ header in the caller's file. Use toml::parse_error's
diff --git a/oi/OICodeGen.h b/oi/OICodeGen.h
index a01548f..7226a4e 100644
--- a/oi/OICodeGen.h
+++ b/oi/OICodeGen.h
@@ -66,7 +66,7 @@ class OICodeGen {
     };
 
     FeatureSet features;
-    std::set<std::filesystem::path> containerConfigPaths;
+    std::vector<std::filesystem::path> containerConfigPaths;
     std::set<std::string> defaultHeaders;
     std::set<std::string> defaultNamespaces;
     std::vector<std::pair<std::string, std::string>> membersToStub;
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 1cb2a49..93054bc 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -16,130 +16,150 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
 #include <glog/logging.h>
 
 #include <boost/core/demangle.hpp>
 #include <fstream>
 #include <iostream>
+#include <range/v3/action/transform.hpp>
+#include <range/v3/core.hpp>
+#include <range/v3/view/cache1.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/drop_while.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>  // TODO: remove
 #include <string_view>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
+class ConsumerContext;
 
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
-      templateArgsToSymbolsMap;
-
-  auto symbols = prog.find_all_symbols();
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
+class CreateTypeGraphConsumer;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
-    if (val.first.empty() || val.second.empty()) {
-      continue;
-    }
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  return strongToWeakSymbols;
-}
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
 
-std::unordered_map<std::string, drgn_qualified_type>
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
+ private:
+  ConsumerContext& ctx;
+};
 
-  std::unordered_map<std::string, drgn_qualified_type> out;
+class ConsumerContext {
+ public:
+  ConsumerContext(const std::vector<std::unique_ptr<ContainerInfo>>& cis)
+      : containerInfos{cis} {
+  }
 
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
-    std::string strongLinkageName;
-    {
-      const char* linkageNameCstr;
-      if (auto err = drgnplusplus::error(
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
-        // throw err;
+  type_graph::TypeGraph typeGraph;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
+  std::optional<bool> pic;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+}  // namespace
+
+int OIGenerator::generate(clang::tooling::CompilationDatabase& db,
+                          const std::vector<std::string>& sourcePaths) {
+  std::map<Feature, bool> featuresMap = {
+      {Feature::TypeGraph, true},
+      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},
+      {Feature::PackStructs, true},
+      {Feature::PruneTypeGraph, true},
+  };
+
+  OICodeGen::Config generatorConfig{};
+  OICompiler::Config compilerConfig{};
+
+  auto features = config::processConfigFiles(
+      configFilePaths, featuresMap, compilerConfig, generatorConfig);
+  if (!features) {
+    LOG(ERROR) << "failed to process config file";
+    return -1;
+  }
+  generatorConfig.features = *features;
+  compilerConfig.features = *features;
+
+  std::vector<std::unique_ptr<ContainerInfo>> containerInfos;
+  containerInfos.reserve(generatorConfig.containerConfigPaths.size());
+  try {
+    for (const auto& path : generatorConfig.containerConfigPaths) {
+      auto info = std::make_unique<ContainerInfo>(path);
+      if (info->requiredFeatures != (*features & info->requiredFeatures)) {
+        VLOG(1) << "Skipping container (feature conflict): " << info->typeName;
         continue;
       }
-      strongLinkageName = linkageNameCstr;
+      containerInfos.emplace_back(std::move(info));
     }
-
-    std::string weakLinkageName;
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return -1;
   }
 
-  return out;
-}
+  ConsumerContext ctx{containerInfos};
+  CreateTypeGraphActionFactory factory{ctx};
 
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
-                                      const OICompiler::Config& compilerConfig,
-                                      const drgn_qualified_type& type,
-                                      const std::string& linkageName,
-                                      SymbolService& symbols) {
-  CodeGen codegen{generatorConfig, symbols};
+  clang::tooling::ClangTool tool{db, sourcePaths};
+  if (auto ret = tool.run(&factory); ret != 0) {
+    return ret;
+  }
+
+  if (ctx.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one type to generate for but we can't currently "
+        "handle this case");
+
+  compilerConfig.usePIC = ctx.pic.value();
+  CodeGen codegen{generatorConfig};
+  for (auto&& ptr : containerInfos)
+    codegen.registerContainer(std::move(ptr));
+  codegen.transform(ctx.typeGraph);
 
   std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
+  codegen.generate(ctx.typeGraph, code, nullptr);
 
   std::string sourcePath = sourceFileDumpPath;
   if (sourceFileDumpPath.empty()) {
@@ -152,78 +172,110 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
   }
 
   OICompiler compiler{{}, compilerConfig};
-
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
-
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
 }
 
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
-  drgnplusplus::program prog;
+namespace {
 
-  {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(
-            drgn_program_load_debug_info(prog.get(),
-                                         std::data(objectPaths),
-                                         std::size(objectPaths),
-                                         false,
-                                         false))) {
-      LOG(ERROR) << "error loading debug info program: " << err;
-      throw err;
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx(ctx_) {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
     }
-  }
 
-  auto oilTypes = findOilTypesAndNames(prog);
-
-  std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
-      {Feature::PruneTypeGraph, true},
-  };
-
-  OICodeGen::Config generatorConfig{};
-  OICompiler::Config compilerConfig{};
-  compilerConfig.usePIC = pic;
-
-  auto features = config::processConfigFiles(
-      configFilePaths, featuresMap, compilerConfig, generatorConfig);
-  if (!features) {
-    LOG(ERROR) << "failed to process config file";
-    return -1;
-  }
-  generatorConfig.features = *features;
-  compilerConfig.features = *features;
-
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
+    auto introspectImpl =
+        std::move(oi_namespaces) |
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
+        ranges::views::transform([](auto* p) {
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
+        }) |
+        ranges::views::filter([](auto* td) {
+          return td != nullptr && td->getName() == "introspectImpl";
+        }) |
+        ranges::views::take(1) | ranges::to<std::vector>();
+    if (introspectImpl.empty()) {
+      LOG(WARNING)
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
+             "you compile with `OIL_AOT_COMPILATION=1`?";
+      return;
     }
-  }
 
-  size_t successes = oilTypes.size() - failures;
-  LOG(INFO) << "object introspection generation complete. " << successes
-            << " successes and " << failures << " failures.";
+    auto nameToClangTypeMap =
+        ranges::views::single(introspectImpl[0]) |
+        ranges::views::for_each(
+            [](auto* td) { return td->specializations(); }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
 
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
-    return -1;
+              assert(fd->getNumParams() == 1);
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, opts};
+
+    auto& Sema = *ctx.sema;
+    ctx.nameToTypeMap =
+        nameToClangTypeMap |
+        ranges::views::transform(
+            [&parser, &Context, &Sema](
+                auto& p) -> std::pair<std::string, type_graph::Type*> {
+              return {p.first, &parser.parse(Context, Sema, *p.second)};
+            }) |
+        ranges::to<std::unordered_map>();
+
+    for (const auto& [name, type] : ctx.nameToTypeMap)
+      ctx.typeGraph.addRoot(*type);
   }
-  return 0;
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  // Compile the output as position independent if any input is position
+  // independent
+  bool pic = CI.getCodeGenOpts().RelocationModel == llvm::Reloc::PIC_;
+  ctx.pic = ctx.pic.value_or(false) || pic;
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
 }
 
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+}  // namespace
 }  // namespace oi::detail
diff --git a/oi/OIGenerator.h b/oi/OIGenerator.h
index dd38384..fac947a 100644
--- a/oi/OIGenerator.h
+++ b/oi/OIGenerator.h
@@ -23,11 +23,19 @@
 #include "oi/OICodeGen.h"
 #include "oi/OICompiler.h"
 
+namespace clang::tooling {
+class CompilationDatabase;
+}
+
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate(clang::tooling::CompilationDatabase&,
+               const std::vector<std::string>&);
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -41,8 +49,8 @@ class OIGenerator {
   void setFailIfNothingGenerated(bool fail) {
     failIfNothingGenerated = fail;
   }
-  void setUsePIC(bool pic_) {
-    pic = pic_;
+  void setClangArgs(std::vector<std::string> args_) {
+    clangArgs = std::move(args_);
   }
 
  private:
@@ -50,20 +58,7 @@ class OIGenerator {
   std::vector<std::filesystem::path> configFilePaths;
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
-  bool pic = false;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
+  std::vector<std::string> clangArgs;
 };
 
 }  // namespace oi::detail
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 190bdfc..6c4da13 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnExporter.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
@@ -27,3 +28,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
new file mode 100644
index 0000000..67f7bfa
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -0,0 +1,416 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Basic/DiagnosticSema.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <iostream>  // TODO: maybe remove
+#include <regex>
+#include <stdexcept>  // TODO: remove
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (VLOG_IS_ON(3)) {
+    std::string fqName = clang::TypeName::getFullyQualifiedName(
+        clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+    VLOG(3) << std::string(depth_ * 2, ' ') << fqName;
+  }
+
+  if (!requireCompleteType(*sema, ty))
+    return makeType<Incomplete>(ty, "incomplete (TODO naming)");
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(llvm::cast<const clang::RecordType>(ty));
+    case clang::Type::LValueReference:
+      return enumerateReference(
+          llvm::cast<const clang::LValueReferenceType>(ty));
+    case clang::Type::Pointer:
+      return enumeratePointer(llvm::cast<const clang::PointerType>(ty));
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          llvm::cast<const clang::SubstTemplateTypeParmType>(ty));
+    case clang::Type::Builtin:
+      return enumeratePrimitive(llvm::cast<const clang::BuiltinType>(ty));
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(
+          llvm::cast<const clang::ElaboratedType>(ty));
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          llvm::cast<const clang::TemplateSpecializationType>(ty));
+    case clang::Type::UnaryTransform:
+      return enumerateUnaryTransformType(
+          llvm::cast<const clang::UnaryTransformType>(ty));
+    case clang::Type::Decltype:
+      return enumerateDecltypeType(llvm::cast<const clang::DecltypeType>(ty));
+    case clang::Type::Typedef:
+      return enumerateTypedef(llvm::cast<const clang::TypedefType>(ty));
+    case clang::Type::Using:
+      return enumerateUsing(llvm::cast<const clang::UsingType>(ty));
+    case clang::Type::ConstantArray:
+      return enumerateArray(llvm::cast<const clang::ConstantArrayType>(ty));
+    case clang::Type::Enum:
+      return enumerateEnum(llvm::cast<const clang::EnumType>(ty));
+
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Type& ClangTypeParser::enumerateDecltypeType(const clang::DecltypeType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Type& ClangTypeParser::enumerateUnaryTransformType(
+    const clang::UnaryTransformType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Typedef& ClangTypeParser::enumerateUsing(const clang::UsingType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+  std::string name = ty.getFoundDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+
+  std::string name = ty.getDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string name = ty.getDecl()->getNameAsString();
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  std::map<int64_t, std::string> enumeratorMap;
+  if (options_.readEnumValues) {
+    for (const auto* enumerator : ty.getDecl()->enumerators()) {
+      enumeratorMap.emplace(enumerator->getInitVal().getExtValue(),
+                            enumerator->getNameAsString());
+    }
+  }
+
+  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+}
+
+Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
+  uint64_t len = ty.getSize().getLimitedValue();
+  auto& t = enumerateType(*ty.getElementType());
+  return makeType<Array>(ty, t, len);
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  if (auto* info = getContainerInfo(fqName)) {
+    auto& c = makeType<Container>(ty, *info, size, nullptr);
+    enumerateClassTemplateParams(ty, c.templateParams);
+    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)));
+    return c;
+  }
+
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+
+  params.reserve(list.size());
+  for (const auto& arg : list.asArray()) {
+    if (auto p = enumerateTemplateParam(arg))
+      params.emplace_back(std::move(p.value()));
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return TemplateParam{ttype, qualifiers};
+    }
+    case clang::TemplateArgument::Integral: {
+      auto& ty = enumerateType(*p.getIntegralType());
+      llvm::SmallString<32> val;
+      p.getAsIntegral().toString(val);
+      return TemplateParam{ty, std::string(val)};
+    }
+    case clang::TemplateArgument::Template: {
+      return enumerateTemplateTemplateParam(p.getAsTemplate());
+    }
+
+#define X(name)                       \
+  case clang::TemplateArgument::name: \
+    throw std::logic_error("unsupported template argument kind: " #name);
+
+      X(Null)
+      X(Declaration)
+      X(NullPtr)
+      X(TemplateExpansion)
+      X(Expression)
+      X(Pack)
+#undef X
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateTemplateParam(
+    const clang::TemplateName& tn) {
+  switch (tn.getKind()) {
+    case clang::TemplateName::Template:
+      return std::nullopt;
+
+#define X(name)                   \
+  case clang::TemplateName::name: \
+    throw std::logic_error("unsupported template name kind: " #name);
+
+      X(OverloadedTemplate)
+      X(AssumedTemplate)
+      X(QualifiedTemplate)
+      X(DependentTemplate)
+      X(SubstTemplateTemplateParm)
+      X(SubstTemplateTemplateParmPack)
+      X(UsingTemplate)
+#undef X
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function references
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Float:
+      return makeType<Primitive>(ty, Primitive::Kind::Float32);
+    case clang::BuiltinType::Double:
+    case clang::BuiltinType::LongDouble:
+      return makeType<Primitive>(ty, Primitive::Kind::Float64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+ContainerInfo* ClangTypeParser::getContainerInfo(
+    const std::string& fqName) const {
+  for (const auto& containerInfo : containers_) {
+    if (std::regex_search(fqName, containerInfo->matcher)) {
+      return containerInfo.get();
+    }
+  }
+  return nullptr;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
+    return true;  // treat as complete
+
+  // TODO: This is a terrible warning.
+  return !sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      clang::diag::warn_nsconsumed_attribute_mismatch);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
new file mode 100644
index 0000000..10eea87
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.h
@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ASTContext;
+class BuiltinType;
+class ConstantArrayType;
+class DecltypeType;
+class ElaboratedType;
+class EnumType;
+class LValueReferenceType;
+class PointerType;
+class RecordType;
+class Sema;
+class SubstTemplateTypeParmType;
+class TemplateArgument;
+class TemplateName;
+class TemplateSpecializationType;
+class Type;
+class TypedefType;
+class UnaryTransformType;
+class UsingType;
+}  // namespace clang
+
+struct ContainerInfo;
+
+namespace oi::detail::type_graph {
+
+class Array;
+class Class;
+class Enum;
+class Member;
+class Primitive;
+class Reference;
+class Type;
+class TypeGraph;
+class Typedef;
+struct TemplateParam;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+  bool readEnumValues = false;
+};
+
+/*
+ * ClangTypeParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given drgn_type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph,
+                  const std::vector<std::unique_ptr<ContainerInfo>>& containers,
+                  ClangTypeParserOptions options)
+      : typeGraph_{typeGraph}, containers_{containers}, options_{options} {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containers_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Type& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+  Typedef& enumerateTypedef(const clang::TypedefType&);
+  Typedef& enumerateUsing(const clang::UsingType&);
+  Type& enumerateUnaryTransformType(const clang::UnaryTransformType&);
+  Type& enumerateDecltypeType(const clang::DecltypeType&);
+
+  Array& enumerateArray(const clang::ConstantArrayType&);
+  Enum& enumerateEnum(const clang::EnumType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  std::optional<TemplateParam> enumerateTemplateParam(
+      const clang::TemplateArgument&);
+  std::optional<TemplateParam> enumerateTemplateTemplateParam(
+      const clang::TemplateName&);
+
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  ContainerInfo* getContainerInfo(const std::string& fqName) const;
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/DrgnExporter.cpp b/oi/type_graph/DrgnExporter.cpp
index 802857e..df9722d 100644
--- a/oi/type_graph/DrgnExporter.cpp
+++ b/oi/type_graph/DrgnExporter.cpp
@@ -200,6 +200,14 @@ drgn_type* DrgnExporter::visit(Pointer& p) {
   return drgnType;
 }
 
+drgn_type* DrgnExporter::visit(Reference& p) {
+  auto* drgnType =
+      makeDrgnType(DRGN_TYPE_POINTER, false, DRGN_NOT_PRIMITIVE_TYPE, p);
+  auto* pointeeType = accept(p.pointeeType());
+  th_.pointerToTypeMap[drgnType] = pointeeType;
+  return drgnType;
+}
+
 drgn_type* DrgnExporter::visit(Dummy& d) {
   return makeDrgnType(DRGN_TYPE_VOID, false, DRGN_C_TYPE_VOID, d);
 }
diff --git a/oi/type_graph/DrgnExporter.h b/oi/type_graph/DrgnExporter.h
index 45f02c6..24f4c59 100644
--- a/oi/type_graph/DrgnExporter.h
+++ b/oi/type_graph/DrgnExporter.h
@@ -54,6 +54,7 @@ class DrgnExporter : public Visitor<drgn_type*> {
   drgn_type* visit(Array&) override;
   drgn_type* visit(Typedef&) override;
   drgn_type* visit(Pointer&) override;
+  drgn_type* visit(Reference&) override;
   drgn_type* visit(Dummy&) override;
   drgn_type* visit(DummyAllocator&) override;
   drgn_type* visit(CaptureKeys&) override;
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index b1d1e8d..f016c13 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -200,6 +200,14 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& p) {
+  RecursiveVisitor::visit(p);
+  p.regenerateName();
+  std::string inputName{p.pointeeType().inputName()};
+  inputName += '*';
+  p.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..e6437d0 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index 0dae0ab..49165ab 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -150,6 +150,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& p) {
+  if (prefix(p))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = p.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(p.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index 8d7138a..e08a943 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& p) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;
diff --git a/oi/type_graph/RemoveTopLevelPointer.cpp b/oi/type_graph/RemoveTopLevelPointer.cpp
index 9ecaaaa..e1909a7 100644
--- a/oi/type_graph/RemoveTopLevelPointer.cpp
+++ b/oi/type_graph/RemoveTopLevelPointer.cpp
@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& p) {
+  topLevelType_ = &p.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/RemoveTopLevelPointer.h b/oi/type_graph/RemoveTopLevelPointer.h
index 611429f..6b70545 100644
--- a/oi/type_graph/RemoveTopLevelPointer.h
+++ b/oi/type_graph/RemoveTopLevelPointer.h
@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
 
  private:
   Type* topLevelType_ = nullptr;
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f0b3fe9..38d53c6 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -734,6 +735,61 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    name_ = pointeeType_.get().name() + "&";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index a7f7eec..f055aea 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -108,6 +108,9 @@ class RecursiveVisitor : public Visitor<void> {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -175,6 +178,10 @@ class RecursiveMutator : public Visitor<Type&> {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }
diff --git a/test/integration/template_template_param.toml b/test/integration/template_template_param.toml
new file mode 100644
index 0000000..c64590d
--- /dev/null
+++ b/test/integration/template_template_param.toml
@@ -0,0 +1,16 @@
+includes = ["vector", "utility", "string"]
+
+definitions = '''
+template <typename K, typename V, template <typename, typename> typename Pair>
+struct bad_map {
+  std::vector<Pair<K, K>> keys;
+  std::vector<Pair<V, V>> values;
+};
+'''
+
+[cases]
+  [cases.int_int_empty]
+    param_types = ["const bad_map<int, std::string, std::pair>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index d1e345d..795389e 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -14,131 +14,81 @@
  * limitations under the License.
  */
 
+#include <clang/Tooling/CommonOptionsParser.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include <cstdlib>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/action/drop_while.hpp>
+#include <range/v3/algorithm/find.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/all.hpp>
+#include <range/v3/view/transform.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
 
-constexpr static OIOpts opts{
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
+static llvm::cl::OptionCategory OilgenCategory("oilgen options");
 
-void usage() {
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
-  std::cerr << opts;
+static llvm::cl::list<std::string> ConfigFiles("config-file",
+                                               llvm::cl::desc(R"(TODO HELP)"),
+                                               llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<int> DebugLevel("debug-level",
+                                     llvm::cl::desc(R"(TODO HELP)"),
+                                     llvm::cl::init(-1),
+                                     llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> DumpJit("dump-jit",
+                                          llvm::cl::desc(R"(TODO HELP)"),
+                                          llvm::cl::init("jit.cpp"),
+                                          llvm::cl::cat(OilgenCategory));
 
-  std::cerr << std::endl
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
-
-int main(int argc, char* argv[]) {
+int main(int argc, const char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
-  std::vector<fs::path> configFilePaths;
-  fs::path sourceFileDumpPath = "";
-  bool exitCode = false;
-  bool pic = false;
-
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
+  auto expectedParser =
+      clang::tooling::CommonOptionsParser::create(argc, argv, OilgenCategory);
+  if (!expectedParser) {
+    llvm::errs() << expectedParser.takeError();
+    return -1;
   }
+  clang::tooling::CommonOptionsParser& options = expectedParser.get();
+  auto& compilations = options.getCompilations();
 
-  if (optind >= argc) {
-    usage();
-    return EXIT_FAILURE;
-  }
-  fs::path primaryObject = argv[optind];
-
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
+  if (DebugLevel.getNumOccurrences()) {
+    google::LogToStderr();
+    google::SetStderrLogging(google::INFO);
+    google::SetVLOGLevel("*", DebugLevel);
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
+    gflags::SetCommandLineOption("minloglevel", "0");
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
-  oigen.setConfigFilePaths(std::move(configFilePaths));
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
-  oigen.setFailIfNothingGenerated(exitCode);
-  oigen.setUsePIC(pic);
+  oigen.setConfigFilePaths(ConfigFiles |
+                           ranges::views::transform([](const auto& p) {
+                             return std::filesystem::path(p);
+                           }) |
+                           ranges::to<std::vector>());
+  if (DumpJit.getNumOccurrences())
+    oigen.setSourceFileDumpPath(DumpJit.getValue());
 
-  SymbolService symbols(primaryObject);
+  oigen.setOutputPath("a.o");  // TODO: make this an opt
 
-  return oigen.generate(primaryObject, symbols);
+  // TODO: can we extract this from the original arguments?
+  // oigen.setUsePIC(args["pic"].as<bool>());
+
+  oigen.setFailIfNothingGenerated(true);
+  // compilations.getAllFiles();
+  // return oigen.generate(compilations,
+  // {"/data/users/jakehillion/fbsource/fbcode/object-introspection/oil/examples/compile-time/VectorOfStrings-Introspect.cpp"});
+  return oigen.generate(compilations, options.getSourcePathList());
 }
diff --git a/types/string_type.toml b/types/string_type.toml
index fa4a2ae..0d2ab0d 100644
--- a/types/string_type.toml
+++ b/types/string_type.toml
@@ -35,3 +35,34 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
     );
 }
 """
+
+traversal_func = """
+    bool sso = ((uintptr_t)container.data() <
+                (uintptr_t)(&container + sizeof(std::__cxx11::basic_string<T0>))) &&
+               ((uintptr_t)container.data() >= (uintptr_t)&container);
+
+    return returnArg.write(container.capacity())
+        .write(sso)
+        .write(container.size());
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+uint64_t capacity = std::get<ParsedData::VarInt>(d.val).value;
+el.container_stats.emplace(result::Element::ContainerStats { .capacity = capacity });
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+bool sso = std::get<ParsedData::VarInt>(d.val).value;
+if (!sso)
+  el.exclusive_size += el.container_stats->capacity * sizeof(T0);
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats->length = std::get<ParsedData::VarInt>(d.val).value;
+"""

From 40c2616bd5d2dea95b9426a223d20fcb4e75dc06 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 19 Dec 2023 15:00:35 +0000
Subject: [PATCH 067/188] codegen: remove reliance on drgn type for top level
 name

Currently we rely on `SymbolService::getTypeName` for getting the hash that's
included in the generated function's name. The value of this must stay the same
to match with the value expected by OIDebugger - changing it causes failure to
relocate when attaching with OID and JIT OIL.

Calculate this name in the `codegenFromDrgn` method and pass it through where
appropriate rather than passing the `drgn_type` itself through.

We don't need to name the type like that when using AoT OIL. Let's
hash the linkage name instead as that is more unique.

Test Plan:
- CI
---
 oi/CodeGen.cpp | 46 ++++++++++++++++++++++++++++++++--------------
 oi/CodeGen.h   | 17 +++++++++++++----
 2 files changed, 45 insertions(+), 18 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index d866df3..74c6f39 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -43,6 +43,9 @@
 #include "type_graph/TypeIdentifier.h"
 #include "type_graph/Types.h"
 
+template <typename T>
+inline constexpr bool always_false_v = false;
+
 namespace oi::detail {
 
 using type_graph::AddChildren;
@@ -1101,11 +1104,17 @@ void CodeGen::addTypeHandlers(const TypeGraph& typeGraph, std::string& code) {
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
                               std::string linkageName,
                               std::string& code) {
-  linkageName_ = std::move(linkageName);
-  return codegenFromDrgn(drgnType, code);
+  return codegenFromDrgn(drgnType, code, ExactName{std::move(linkageName)});
 }
 
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
+  return codegenFromDrgn(
+      drgnType, code, HashedComponent{SymbolService::getTypeName(drgnType)});
+}
+
+bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
+                              std::string& code,
+                              RootFunctionName name) {
   try {
     containerInfos_.reserve(config_.containerConfigPaths.size());
     for (const auto& path : config_.containerConfigPaths) {
@@ -1124,7 +1133,7 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
   }
 
   transform(typeGraph_);
-  generate(typeGraph_, code, drgnType);
+  generate(typeGraph_, code, std::move(name));
   return true;
 }
 
@@ -1213,11 +1222,9 @@ void CodeGen::transform(TypeGraph& typeGraph) {
   };
 }
 
-void CodeGen::generate(
-    TypeGraph& typeGraph,
-    std::string& code,
-    struct drgn_type* drgnType /* TODO: this argument should not be required */
-) {
+void CodeGen::generate(TypeGraph& typeGraph,
+                       std::string& code,
+                       RootFunctionName rootName) {
   code = headers::oi_OITraceCode_cpp;
   if (!config_.features[Feature::Library]) {
     FuncGen::DeclareExterns(code);
@@ -1296,22 +1303,33 @@ void CodeGen::generate(
   code += "\nusing __ROOT_TYPE__ = " + rootType.name() + ";\n";
   code += "} // namespace\n} // namespace OIInternal\n";
 
-  const auto typeName = SymbolService::getTypeName(drgnType);
+  const auto& typeToHash = std::visit(
+      [](const auto& v) -> const std::string& {
+        using T = std::decay_t<decltype(v)>;
+        if constexpr (std::is_same_v<ExactName, T> ||
+                      std::is_same_v<HashedComponent, T>) {
+          return v.name;
+        } else {
+          static_assert(always_false_v<T>, "missing visit");
+        }
+      },
+      rootName);
+
   if (config_.features[Feature::TreeBuilderV2]) {
-    FuncGen::DefineTopLevelIntrospect(code, typeName);
+    FuncGen::DefineTopLevelIntrospect(code, typeToHash);
   } else {
-    FuncGen::DefineTopLevelGetSizeRef(code, typeName, config_.features);
+    FuncGen::DefineTopLevelGetSizeRef(code, typeToHash, config_.features);
   }
 
   if (config_.features[Feature::TreeBuilderV2]) {
     FuncGen::DefineTreeBuilderInstructions(code,
-                                           typeName,
+                                           typeToHash,
                                            calculateExclusiveSize(rootType),
                                            enumerateTypeNames(rootType));
   }
 
-  if (!linkageName_.empty())
-    FuncGen::DefineTopLevelIntrospectNamed(code, typeName, linkageName_);
+  if (auto* n = std::get_if<ExactName>(&rootName))
+    FuncGen::DefineTopLevelIntrospectNamed(code, typeToHash, n->name);
 
   if (VLOG_IS_ON(3)) {
     VLOG(3) << "Generated trace code:\n";
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 2c9b5e1..9c76238 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -45,6 +45,14 @@ class CodeGen {
       : config_(config), symbols_(symbols) {
   }
 
+  struct ExactName {
+    std::string name;
+  };
+  struct HashedComponent {
+    std::string name;
+  };
+  using RootFunctionName = std::variant<ExactName, HashedComponent>;
+
   /*
    * Helper function to perform all the steps required for code generation for a
    * single drgn_type.
@@ -64,9 +72,7 @@ class CodeGen {
   void transform(type_graph::TypeGraph& typeGraph);
   void generate(type_graph::TypeGraph& typeGraph,
                 std::string& code,
-                struct drgn_type*
-                    drgnType /* TODO: this argument should not be required */
-  );
+                RootFunctionName rootName);
 
  private:
   type_graph::TypeGraph typeGraph_;
@@ -76,7 +82,10 @@ class CodeGen {
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
       thriftIssetMembers_;
-  std::string linkageName_;
+
+  bool codegenFromDrgn(struct drgn_type* drgnType,
+                       std::string& code,
+                       RootFunctionName name);
 
   void genDefsThrift(const type_graph::TypeGraph& typeGraph, std::string& code);
   void addGetSizeFuncDefs(const type_graph::TypeGraph& typeGraph,

From 0d45a489940db50584b1ea3b1b028ceec8c7d81e Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 19 Dec 2023 15:22:22 +0000
Subject: [PATCH 068/188] oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD
---
 CMakeLists.txt                                |   3 +-
 oi/CodeGen.cpp                                |  30 +-
 oi/CodeGen.h                                  |   6 +-
 oi/Config.cpp                                 |   9 +-
 oi/ContainerInfo.cpp                          |   2 +-
 oi/OICodeGen.h                                |   2 +-
 oi/OIGenerator.cpp                            | 370 +++++++++-------
 oi/OIGenerator.h                              |  29 +-
 oi/type_graph/CMakeLists.txt                  |   2 +
 oi/type_graph/ClangTypeParser.cpp             | 416 ++++++++++++++++++
 oi/type_graph/ClangTypeParser.h               | 131 ++++++
 oi/type_graph/DrgnExporter.cpp                |   8 +
 oi/type_graph/DrgnExporter.h                  |   1 +
 oi/type_graph/NameGen.cpp                     |   8 +
 oi/type_graph/NameGen.h                       |   1 +
 oi/type_graph/Printer.cpp                     |  11 +
 oi/type_graph/Printer.h                       |   1 +
 oi/type_graph/RemoveTopLevelPointer.cpp       |   4 +
 oi/type_graph/RemoveTopLevelPointer.h         |   1 +
 oi/type_graph/Types.h                         |  56 +++
 oi/type_graph/Visitor.h                       |   7 +
 test/integration/template_template_param.toml |  16 +
 tools/OILGen.cpp                              | 148 +++----
 types/string_type.toml                        |  31 ++
 24 files changed, 1004 insertions(+), 289 deletions(-)
 create mode 100644 oi/type_graph/ClangTypeParser.cpp
 create mode 100644 oi/type_graph/ClangTypeParser.h
 create mode 100644 test/integration/template_template_param.toml

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f9ee7..b6fecd1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -289,7 +289,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -365,6 +365,7 @@ add_executable(oilgen
 target_link_libraries(oilgen
   drgn_utils
   oicore
+  clangTooling
 )
 
 ### Object Introspection cache Printer (OIP)
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 74c6f39..869fc20 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -299,6 +299,10 @@ void genDefsThriftClass(const Class& c, std::string& code) {
 
 }  // namespace
 
+CodeGen::CodeGen(const OICodeGen::Config& config) : config_(config) {
+  DCHECK(!config.features[Feature::PolymorphicInheritance]) << "polymorphic inheritance requires symbol service!";
+}
+
 void CodeGen::genDefsThrift(const TypeGraph& typeGraph, std::string& code) {
   for (const Type& t : typeGraph.finalTypes) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
@@ -542,7 +546,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1115,15 +1119,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
                               std::string& code,
                               RootFunctionName name) {
-  try {
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
+  if (!registerContainers())
     return false;
-  }
 
   try {
     addDrgnRoot(drgnType, typeGraph_);
@@ -1148,6 +1145,19 @@ void CodeGen::exportDrgnTypes(TypeHierarchy& th,
   }
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1189,7 +1199,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(IdentifyContainers::createPass(containerInfos_));
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 9c76238..931aa83 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -41,8 +41,9 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config);
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   struct ExactName {
@@ -65,6 +66,7 @@ class CodeGen {
                        std::list<drgn_type>& drgnTypes,
                        drgn_type** rootType) const;
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -77,7 +79,7 @@ class CodeGen {
  private:
   type_graph::TypeGraph typeGraph_;
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
diff --git a/oi/Config.cpp b/oi/Config.cpp
index 157b279..66798bb 100644
--- a/oi/Config.cpp
+++ b/oi/Config.cpp
@@ -22,6 +22,8 @@
 #include <boost/property_tree/ini_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 #include <filesystem>
+#include <range/v3/action/sort.hpp>
+#include <range/v3/action/unique.hpp>
 
 #include "oi/support/Toml.h"
 
@@ -54,6 +56,9 @@ std::optional<FeatureSet> processConfigFiles(
     enables |= *fs;
   }
 
+  ranges::actions::sort(generatorConfig.containerConfigPaths);
+  ranges::actions::unique(generatorConfig.containerConfigPaths);
+
   // Override anything from the config files with command line options
   for (auto [k, v] : featureMap) {
     enables[k] = v;
@@ -108,8 +113,8 @@ std::optional<FeatureSet> processConfigFile(
           if the right path is absolute, else append the right path to the
           left path.
           */
-          generatorConfig.containerConfigPaths.emplace(configDirectory /
-                                                       el.get());
+          generatorConfig.containerConfigPaths.emplace_back(configDirectory /
+                                                            el.get());
         }
       });
     }
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 4cb20b8..b83b7e4 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -202,7 +202,7 @@ std::regex getMatcher(const std::string& typeName) {
 ContainerInfo::ContainerInfo(const fs::path& path) {
   toml::table container;
   try {
-    container = toml::parse_file(std::string(path));
+    container = toml::parse_file(path.string());
   } catch (const toml::parse_error& err) {
     // Convert into a ContainerInfoError, just to avoid having to include
     // the huge TOML++ header in the caller's file. Use toml::parse_error's
diff --git a/oi/OICodeGen.h b/oi/OICodeGen.h
index a01548f..7226a4e 100644
--- a/oi/OICodeGen.h
+++ b/oi/OICodeGen.h
@@ -66,7 +66,7 @@ class OICodeGen {
     };
 
     FeatureSet features;
-    std::set<std::filesystem::path> containerConfigPaths;
+    std::vector<std::filesystem::path> containerConfigPaths;
     std::set<std::string> defaultHeaders;
     std::set<std::string> defaultNamespaces;
     std::vector<std::pair<std::string, std::string>> membersToStub;
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 1cb2a49..cf7ec44 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -16,130 +16,156 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
 #include <glog/logging.h>
 
 #include <boost/core/demangle.hpp>
 #include <fstream>
 #include <iostream>
+#include <range/v3/action/transform.hpp>
+#include <range/v3/core.hpp>
+#include <range/v3/view/cache1.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/drop_while.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>  // TODO: remove
 #include <string_view>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
+class ConsumerContext;
 
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
-      templateArgsToSymbolsMap;
-
-  auto symbols = prog.find_all_symbols();
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
+class CreateTypeGraphConsumer;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
-    if (val.first.empty() || val.second.empty()) {
-      continue;
-    }
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  return strongToWeakSymbols;
-}
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
 
-std::unordered_map<std::string, drgn_qualified_type>
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
+ private:
+  ConsumerContext& ctx;
+};
 
-  std::unordered_map<std::string, drgn_qualified_type> out;
+class ConsumerContext {
+ public:
+  ConsumerContext(const std::vector<std::unique_ptr<ContainerInfo>>& cis)
+      : containerInfos{cis} {
+  }
 
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
-    std::string strongLinkageName;
-    {
-      const char* linkageNameCstr;
-      if (auto err = drgnplusplus::error(
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
-        // throw err;
+  type_graph::TypeGraph typeGraph;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
+  std::optional<bool> pic;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+}  // namespace
+
+int OIGenerator::generate(clang::tooling::CompilationDatabase& db,
+                          const std::vector<std::string>& sourcePaths) {
+  std::map<Feature, bool> featuresMap = {
+      {Feature::TypeGraph, true},
+      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},
+      {Feature::PackStructs, true},
+      {Feature::PruneTypeGraph, true},
+  };
+
+  OICodeGen::Config generatorConfig{};
+  OICompiler::Config compilerConfig{};
+
+  auto features = config::processConfigFiles(
+      configFilePaths, featuresMap, compilerConfig, generatorConfig);
+  if (!features) {
+    LOG(ERROR) << "failed to process config file";
+    return -1;
+  }
+  generatorConfig.features = *features;
+  compilerConfig.features = *features;
+
+  std::vector<std::unique_ptr<ContainerInfo>> containerInfos;
+  containerInfos.reserve(generatorConfig.containerConfigPaths.size());
+  try {
+    for (const auto& path : generatorConfig.containerConfigPaths) {
+      auto info = std::make_unique<ContainerInfo>(path);
+      if (info->requiredFeatures != (*features & info->requiredFeatures)) {
+        VLOG(1) << "Skipping container (feature conflict): " << info->typeName;
         continue;
       }
-      strongLinkageName = linkageNameCstr;
+      containerInfos.emplace_back(std::move(info));
     }
-
-    std::string weakLinkageName;
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return -1;
   }
 
-  return out;
-}
+  ConsumerContext ctx{containerInfos};
+  CreateTypeGraphActionFactory factory{ctx};
 
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
-                                      const OICompiler::Config& compilerConfig,
-                                      const drgn_qualified_type& type,
-                                      const std::string& linkageName,
-                                      SymbolService& symbols) {
-  CodeGen codegen{generatorConfig, symbols};
+  clang::tooling::ClangTool tool{db, sourcePaths};
+  if (auto ret = tool.run(&factory); ret != 0) {
+    return ret;
+  }
+
+  if (ctx.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one type to generate for but we can't currently "
+        "handle this case");
+
+  if (ctx.nameToTypeMap.empty()) {
+    LOG(ERROR) << "Nothing to generate!";
+    return -1;
+  }
+  const auto& linkageName = ctx.nameToTypeMap.begin()->first;
+
+  compilerConfig.usePIC = ctx.pic.value();
+  CodeGen codegen{generatorConfig};
+  for (auto&& ptr : containerInfos)
+    codegen.registerContainer(std::move(ptr));
+  codegen.transform(ctx.typeGraph);
 
   std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
+  codegen.generate(ctx.typeGraph, code, CodeGen::ExactName{linkageName});
 
   std::string sourcePath = sourceFileDumpPath;
   if (sourceFileDumpPath.empty()) {
@@ -152,78 +178,110 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
   }
 
   OICompiler compiler{{}, compilerConfig};
-
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
-
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
 }
 
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
-  drgnplusplus::program prog;
+namespace {
 
-  {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(
-            drgn_program_load_debug_info(prog.get(),
-                                         std::data(objectPaths),
-                                         std::size(objectPaths),
-                                         false,
-                                         false))) {
-      LOG(ERROR) << "error loading debug info program: " << err;
-      throw err;
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx(ctx_) {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
     }
-  }
 
-  auto oilTypes = findOilTypesAndNames(prog);
-
-  std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
-      {Feature::PruneTypeGraph, true},
-  };
-
-  OICodeGen::Config generatorConfig{};
-  OICompiler::Config compilerConfig{};
-  compilerConfig.usePIC = pic;
-
-  auto features = config::processConfigFiles(
-      configFilePaths, featuresMap, compilerConfig, generatorConfig);
-  if (!features) {
-    LOG(ERROR) << "failed to process config file";
-    return -1;
-  }
-  generatorConfig.features = *features;
-  compilerConfig.features = *features;
-
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
+    auto introspectImpl =
+        std::move(oi_namespaces) |
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
+        ranges::views::transform([](auto* p) {
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
+        }) |
+        ranges::views::filter([](auto* td) {
+          return td != nullptr && td->getName() == "introspectImpl";
+        }) |
+        ranges::views::take(1) | ranges::to<std::vector>();
+    if (introspectImpl.empty()) {
+      LOG(WARNING)
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
+             "you compile with `OIL_AOT_COMPILATION=1`?";
+      return;
     }
-  }
 
-  size_t successes = oilTypes.size() - failures;
-  LOG(INFO) << "object introspection generation complete. " << successes
-            << " successes and " << failures << " failures.";
+    auto nameToClangTypeMap =
+        ranges::views::single(introspectImpl[0]) |
+        ranges::views::for_each(
+            [](auto* td) { return td->specializations(); }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
 
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
-    return -1;
+              assert(fd->getNumParams() == 1);
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, opts};
+
+    auto& Sema = *ctx.sema;
+    ctx.nameToTypeMap =
+        nameToClangTypeMap |
+        ranges::views::transform(
+            [&parser, &Context, &Sema](
+                auto& p) -> std::pair<std::string, type_graph::Type*> {
+              return {p.first, &parser.parse(Context, Sema, *p.second)};
+            }) |
+        ranges::to<std::unordered_map>();
+
+    for (const auto& [name, type] : ctx.nameToTypeMap)
+      ctx.typeGraph.addRoot(*type);
   }
-  return 0;
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  // Compile the output as position independent if any input is position
+  // independent
+  bool pic = CI.getCodeGenOpts().RelocationModel == llvm::Reloc::PIC_;
+  ctx.pic = ctx.pic.value_or(false) || pic;
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
 }
 
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+}  // namespace
 }  // namespace oi::detail
diff --git a/oi/OIGenerator.h b/oi/OIGenerator.h
index dd38384..fac947a 100644
--- a/oi/OIGenerator.h
+++ b/oi/OIGenerator.h
@@ -23,11 +23,19 @@
 #include "oi/OICodeGen.h"
 #include "oi/OICompiler.h"
 
+namespace clang::tooling {
+class CompilationDatabase;
+}
+
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate(clang::tooling::CompilationDatabase&,
+               const std::vector<std::string>&);
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -41,8 +49,8 @@ class OIGenerator {
   void setFailIfNothingGenerated(bool fail) {
     failIfNothingGenerated = fail;
   }
-  void setUsePIC(bool pic_) {
-    pic = pic_;
+  void setClangArgs(std::vector<std::string> args_) {
+    clangArgs = std::move(args_);
   }
 
  private:
@@ -50,20 +58,7 @@ class OIGenerator {
   std::vector<std::filesystem::path> configFilePaths;
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
-  bool pic = false;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
+  std::vector<std::string> clangArgs;
 };
 
 }  // namespace oi::detail
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 190bdfc..6c4da13 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnExporter.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
@@ -27,3 +28,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
new file mode 100644
index 0000000..67f7bfa
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -0,0 +1,416 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Basic/DiagnosticSema.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <iostream>  // TODO: maybe remove
+#include <regex>
+#include <stdexcept>  // TODO: remove
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (VLOG_IS_ON(3)) {
+    std::string fqName = clang::TypeName::getFullyQualifiedName(
+        clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+    VLOG(3) << std::string(depth_ * 2, ' ') << fqName;
+  }
+
+  if (!requireCompleteType(*sema, ty))
+    return makeType<Incomplete>(ty, "incomplete (TODO naming)");
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(llvm::cast<const clang::RecordType>(ty));
+    case clang::Type::LValueReference:
+      return enumerateReference(
+          llvm::cast<const clang::LValueReferenceType>(ty));
+    case clang::Type::Pointer:
+      return enumeratePointer(llvm::cast<const clang::PointerType>(ty));
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          llvm::cast<const clang::SubstTemplateTypeParmType>(ty));
+    case clang::Type::Builtin:
+      return enumeratePrimitive(llvm::cast<const clang::BuiltinType>(ty));
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(
+          llvm::cast<const clang::ElaboratedType>(ty));
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          llvm::cast<const clang::TemplateSpecializationType>(ty));
+    case clang::Type::UnaryTransform:
+      return enumerateUnaryTransformType(
+          llvm::cast<const clang::UnaryTransformType>(ty));
+    case clang::Type::Decltype:
+      return enumerateDecltypeType(llvm::cast<const clang::DecltypeType>(ty));
+    case clang::Type::Typedef:
+      return enumerateTypedef(llvm::cast<const clang::TypedefType>(ty));
+    case clang::Type::Using:
+      return enumerateUsing(llvm::cast<const clang::UsingType>(ty));
+    case clang::Type::ConstantArray:
+      return enumerateArray(llvm::cast<const clang::ConstantArrayType>(ty));
+    case clang::Type::Enum:
+      return enumerateEnum(llvm::cast<const clang::EnumType>(ty));
+
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Type& ClangTypeParser::enumerateDecltypeType(const clang::DecltypeType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Type& ClangTypeParser::enumerateUnaryTransformType(
+    const clang::UnaryTransformType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Typedef& ClangTypeParser::enumerateUsing(const clang::UsingType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+  std::string name = ty.getFoundDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+
+  std::string name = ty.getDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string name = ty.getDecl()->getNameAsString();
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  std::map<int64_t, std::string> enumeratorMap;
+  if (options_.readEnumValues) {
+    for (const auto* enumerator : ty.getDecl()->enumerators()) {
+      enumeratorMap.emplace(enumerator->getInitVal().getExtValue(),
+                            enumerator->getNameAsString());
+    }
+  }
+
+  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+}
+
+Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
+  uint64_t len = ty.getSize().getLimitedValue();
+  auto& t = enumerateType(*ty.getElementType());
+  return makeType<Array>(ty, t, len);
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  if (auto* info = getContainerInfo(fqName)) {
+    auto& c = makeType<Container>(ty, *info, size, nullptr);
+    enumerateClassTemplateParams(ty, c.templateParams);
+    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)));
+    return c;
+  }
+
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+
+  params.reserve(list.size());
+  for (const auto& arg : list.asArray()) {
+    if (auto p = enumerateTemplateParam(arg))
+      params.emplace_back(std::move(p.value()));
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return TemplateParam{ttype, qualifiers};
+    }
+    case clang::TemplateArgument::Integral: {
+      auto& ty = enumerateType(*p.getIntegralType());
+      llvm::SmallString<32> val;
+      p.getAsIntegral().toString(val);
+      return TemplateParam{ty, std::string(val)};
+    }
+    case clang::TemplateArgument::Template: {
+      return enumerateTemplateTemplateParam(p.getAsTemplate());
+    }
+
+#define X(name)                       \
+  case clang::TemplateArgument::name: \
+    throw std::logic_error("unsupported template argument kind: " #name);
+
+      X(Null)
+      X(Declaration)
+      X(NullPtr)
+      X(TemplateExpansion)
+      X(Expression)
+      X(Pack)
+#undef X
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateTemplateParam(
+    const clang::TemplateName& tn) {
+  switch (tn.getKind()) {
+    case clang::TemplateName::Template:
+      return std::nullopt;
+
+#define X(name)                   \
+  case clang::TemplateName::name: \
+    throw std::logic_error("unsupported template name kind: " #name);
+
+      X(OverloadedTemplate)
+      X(AssumedTemplate)
+      X(QualifiedTemplate)
+      X(DependentTemplate)
+      X(SubstTemplateTemplateParm)
+      X(SubstTemplateTemplateParmPack)
+      X(UsingTemplate)
+#undef X
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function references
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Float:
+      return makeType<Primitive>(ty, Primitive::Kind::Float32);
+    case clang::BuiltinType::Double:
+    case clang::BuiltinType::LongDouble:
+      return makeType<Primitive>(ty, Primitive::Kind::Float64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+ContainerInfo* ClangTypeParser::getContainerInfo(
+    const std::string& fqName) const {
+  for (const auto& containerInfo : containers_) {
+    if (std::regex_search(fqName, containerInfo->matcher)) {
+      return containerInfo.get();
+    }
+  }
+  return nullptr;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
+    return true;  // treat as complete
+
+  // TODO: This is a terrible warning.
+  return !sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      clang::diag::warn_nsconsumed_attribute_mismatch);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
new file mode 100644
index 0000000..10eea87
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.h
@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ASTContext;
+class BuiltinType;
+class ConstantArrayType;
+class DecltypeType;
+class ElaboratedType;
+class EnumType;
+class LValueReferenceType;
+class PointerType;
+class RecordType;
+class Sema;
+class SubstTemplateTypeParmType;
+class TemplateArgument;
+class TemplateName;
+class TemplateSpecializationType;
+class Type;
+class TypedefType;
+class UnaryTransformType;
+class UsingType;
+}  // namespace clang
+
+struct ContainerInfo;
+
+namespace oi::detail::type_graph {
+
+class Array;
+class Class;
+class Enum;
+class Member;
+class Primitive;
+class Reference;
+class Type;
+class TypeGraph;
+class Typedef;
+struct TemplateParam;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+  bool readEnumValues = false;
+};
+
+/*
+ * ClangTypeParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given drgn_type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph,
+                  const std::vector<std::unique_ptr<ContainerInfo>>& containers,
+                  ClangTypeParserOptions options)
+      : typeGraph_{typeGraph}, containers_{containers}, options_{options} {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containers_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Type& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+  Typedef& enumerateTypedef(const clang::TypedefType&);
+  Typedef& enumerateUsing(const clang::UsingType&);
+  Type& enumerateUnaryTransformType(const clang::UnaryTransformType&);
+  Type& enumerateDecltypeType(const clang::DecltypeType&);
+
+  Array& enumerateArray(const clang::ConstantArrayType&);
+  Enum& enumerateEnum(const clang::EnumType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  std::optional<TemplateParam> enumerateTemplateParam(
+      const clang::TemplateArgument&);
+  std::optional<TemplateParam> enumerateTemplateTemplateParam(
+      const clang::TemplateName&);
+
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  ContainerInfo* getContainerInfo(const std::string& fqName) const;
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/DrgnExporter.cpp b/oi/type_graph/DrgnExporter.cpp
index 802857e..df9722d 100644
--- a/oi/type_graph/DrgnExporter.cpp
+++ b/oi/type_graph/DrgnExporter.cpp
@@ -200,6 +200,14 @@ drgn_type* DrgnExporter::visit(Pointer& p) {
   return drgnType;
 }
 
+drgn_type* DrgnExporter::visit(Reference& p) {
+  auto* drgnType =
+      makeDrgnType(DRGN_TYPE_POINTER, false, DRGN_NOT_PRIMITIVE_TYPE, p);
+  auto* pointeeType = accept(p.pointeeType());
+  th_.pointerToTypeMap[drgnType] = pointeeType;
+  return drgnType;
+}
+
 drgn_type* DrgnExporter::visit(Dummy& d) {
   return makeDrgnType(DRGN_TYPE_VOID, false, DRGN_C_TYPE_VOID, d);
 }
diff --git a/oi/type_graph/DrgnExporter.h b/oi/type_graph/DrgnExporter.h
index 45f02c6..24f4c59 100644
--- a/oi/type_graph/DrgnExporter.h
+++ b/oi/type_graph/DrgnExporter.h
@@ -54,6 +54,7 @@ class DrgnExporter : public Visitor<drgn_type*> {
   drgn_type* visit(Array&) override;
   drgn_type* visit(Typedef&) override;
   drgn_type* visit(Pointer&) override;
+  drgn_type* visit(Reference&) override;
   drgn_type* visit(Dummy&) override;
   drgn_type* visit(DummyAllocator&) override;
   drgn_type* visit(CaptureKeys&) override;
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index b1d1e8d..f016c13 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -200,6 +200,14 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& p) {
+  RecursiveVisitor::visit(p);
+  p.regenerateName();
+  std::string inputName{p.pointeeType().inputName()};
+  inputName += '*';
+  p.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..e6437d0 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index 0dae0ab..49165ab 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -150,6 +150,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& p) {
+  if (prefix(p))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = p.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(p.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index 8d7138a..e08a943 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& p) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;
diff --git a/oi/type_graph/RemoveTopLevelPointer.cpp b/oi/type_graph/RemoveTopLevelPointer.cpp
index 9ecaaaa..e1909a7 100644
--- a/oi/type_graph/RemoveTopLevelPointer.cpp
+++ b/oi/type_graph/RemoveTopLevelPointer.cpp
@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& p) {
+  topLevelType_ = &p.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/RemoveTopLevelPointer.h b/oi/type_graph/RemoveTopLevelPointer.h
index 611429f..6b70545 100644
--- a/oi/type_graph/RemoveTopLevelPointer.h
+++ b/oi/type_graph/RemoveTopLevelPointer.h
@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
 
  private:
   Type* topLevelType_ = nullptr;
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f0b3fe9..38d53c6 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -734,6 +735,61 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    name_ = pointeeType_.get().name() + "&";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index a7f7eec..f055aea 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -108,6 +108,9 @@ class RecursiveVisitor : public Visitor<void> {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -175,6 +178,10 @@ class RecursiveMutator : public Visitor<Type&> {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }
diff --git a/test/integration/template_template_param.toml b/test/integration/template_template_param.toml
new file mode 100644
index 0000000..c64590d
--- /dev/null
+++ b/test/integration/template_template_param.toml
@@ -0,0 +1,16 @@
+includes = ["vector", "utility", "string"]
+
+definitions = '''
+template <typename K, typename V, template <typename, typename> typename Pair>
+struct bad_map {
+  std::vector<Pair<K, K>> keys;
+  std::vector<Pair<V, V>> values;
+};
+'''
+
+[cases]
+  [cases.int_int_empty]
+    param_types = ["const bad_map<int, std::string, std::pair>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index d1e345d..795389e 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -14,131 +14,81 @@
  * limitations under the License.
  */
 
+#include <clang/Tooling/CommonOptionsParser.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include <cstdlib>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/action/drop_while.hpp>
+#include <range/v3/algorithm/find.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/all.hpp>
+#include <range/v3/view/transform.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
 
-constexpr static OIOpts opts{
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
+static llvm::cl::OptionCategory OilgenCategory("oilgen options");
 
-void usage() {
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
-  std::cerr << opts;
+static llvm::cl::list<std::string> ConfigFiles("config-file",
+                                               llvm::cl::desc(R"(TODO HELP)"),
+                                               llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<int> DebugLevel("debug-level",
+                                     llvm::cl::desc(R"(TODO HELP)"),
+                                     llvm::cl::init(-1),
+                                     llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> DumpJit("dump-jit",
+                                          llvm::cl::desc(R"(TODO HELP)"),
+                                          llvm::cl::init("jit.cpp"),
+                                          llvm::cl::cat(OilgenCategory));
 
-  std::cerr << std::endl
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
-
-int main(int argc, char* argv[]) {
+int main(int argc, const char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
-  std::vector<fs::path> configFilePaths;
-  fs::path sourceFileDumpPath = "";
-  bool exitCode = false;
-  bool pic = false;
-
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
+  auto expectedParser =
+      clang::tooling::CommonOptionsParser::create(argc, argv, OilgenCategory);
+  if (!expectedParser) {
+    llvm::errs() << expectedParser.takeError();
+    return -1;
   }
+  clang::tooling::CommonOptionsParser& options = expectedParser.get();
+  auto& compilations = options.getCompilations();
 
-  if (optind >= argc) {
-    usage();
-    return EXIT_FAILURE;
-  }
-  fs::path primaryObject = argv[optind];
-
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
+  if (DebugLevel.getNumOccurrences()) {
+    google::LogToStderr();
+    google::SetStderrLogging(google::INFO);
+    google::SetVLOGLevel("*", DebugLevel);
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
+    gflags::SetCommandLineOption("minloglevel", "0");
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
-  oigen.setConfigFilePaths(std::move(configFilePaths));
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
-  oigen.setFailIfNothingGenerated(exitCode);
-  oigen.setUsePIC(pic);
+  oigen.setConfigFilePaths(ConfigFiles |
+                           ranges::views::transform([](const auto& p) {
+                             return std::filesystem::path(p);
+                           }) |
+                           ranges::to<std::vector>());
+  if (DumpJit.getNumOccurrences())
+    oigen.setSourceFileDumpPath(DumpJit.getValue());
 
-  SymbolService symbols(primaryObject);
+  oigen.setOutputPath("a.o");  // TODO: make this an opt
 
-  return oigen.generate(primaryObject, symbols);
+  // TODO: can we extract this from the original arguments?
+  // oigen.setUsePIC(args["pic"].as<bool>());
+
+  oigen.setFailIfNothingGenerated(true);
+  // compilations.getAllFiles();
+  // return oigen.generate(compilations,
+  // {"/data/users/jakehillion/fbsource/fbcode/object-introspection/oil/examples/compile-time/VectorOfStrings-Introspect.cpp"});
+  return oigen.generate(compilations, options.getSourcePathList());
 }
diff --git a/types/string_type.toml b/types/string_type.toml
index fa4a2ae..0d2ab0d 100644
--- a/types/string_type.toml
+++ b/types/string_type.toml
@@ -35,3 +35,34 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
     );
 }
 """
+
+traversal_func = """
+    bool sso = ((uintptr_t)container.data() <
+                (uintptr_t)(&container + sizeof(std::__cxx11::basic_string<T0>))) &&
+               ((uintptr_t)container.data() >= (uintptr_t)&container);
+
+    return returnArg.write(container.capacity())
+        .write(sso)
+        .write(container.size());
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+uint64_t capacity = std::get<ParsedData::VarInt>(d.val).value;
+el.container_stats.emplace(result::Element::ContainerStats { .capacity = capacity });
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+bool sso = std::get<ParsedData::VarInt>(d.val).value;
+if (!sso)
+  el.exclusive_size += el.container_stats->capacity * sizeof(T0);
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats->length = std::get<ParsedData::VarInt>(d.val).value;
+"""

From dcc46f7b4509a371573e36957a3fd699f0548f9e Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 19 Dec 2023 15:36:17 +0000
Subject: [PATCH 069/188] oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD
---
 CMakeLists.txt                                |   3 +-
 oi/CodeGen.cpp                                |  30 +-
 oi/CodeGen.h                                  |   6 +-
 oi/Config.cpp                                 |   9 +-
 oi/ContainerInfo.cpp                          |   2 +-
 oi/OICodeGen.h                                |   2 +-
 oi/OIGenerator.cpp                            | 370 +++++++++-------
 oi/OIGenerator.h                              |  29 +-
 oi/type_graph/CMakeLists.txt                  |   2 +
 oi/type_graph/ClangTypeParser.cpp             | 416 ++++++++++++++++++
 oi/type_graph/ClangTypeParser.h               | 131 ++++++
 oi/type_graph/DrgnExporter.cpp                |   8 +
 oi/type_graph/DrgnExporter.h                  |   1 +
 oi/type_graph/NameGen.cpp                     |   8 +
 oi/type_graph/NameGen.h                       |   1 +
 oi/type_graph/Printer.cpp                     |  11 +
 oi/type_graph/Printer.h                       |   1 +
 oi/type_graph/RemoveTopLevelPointer.cpp       |   4 +
 oi/type_graph/RemoveTopLevelPointer.h         |   1 +
 oi/type_graph/Types.h                         |  56 +++
 oi/type_graph/Visitor.h                       |   7 +
 test/integration/template_template_param.toml |  16 +
 tools/OILGen.cpp                              | 148 +++----
 types/string_type.toml                        |  31 ++
 24 files changed, 1004 insertions(+), 289 deletions(-)
 create mode 100644 oi/type_graph/ClangTypeParser.cpp
 create mode 100644 oi/type_graph/ClangTypeParser.h
 create mode 100644 test/integration/template_template_param.toml

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f9ee7..b6fecd1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -289,7 +289,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -365,6 +365,7 @@ add_executable(oilgen
 target_link_libraries(oilgen
   drgn_utils
   oicore
+  clangTooling
 )
 
 ### Object Introspection cache Printer (OIP)
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 74c6f39..869fc20 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -299,6 +299,10 @@ void genDefsThriftClass(const Class& c, std::string& code) {
 
 }  // namespace
 
+CodeGen::CodeGen(const OICodeGen::Config& config) : config_(config) {
+  DCHECK(!config.features[Feature::PolymorphicInheritance]) << "polymorphic inheritance requires symbol service!";
+}
+
 void CodeGen::genDefsThrift(const TypeGraph& typeGraph, std::string& code) {
   for (const Type& t : typeGraph.finalTypes) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
@@ -542,7 +546,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1115,15 +1119,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
                               std::string& code,
                               RootFunctionName name) {
-  try {
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
+  if (!registerContainers())
     return false;
-  }
 
   try {
     addDrgnRoot(drgnType, typeGraph_);
@@ -1148,6 +1145,19 @@ void CodeGen::exportDrgnTypes(TypeHierarchy& th,
   }
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1189,7 +1199,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(IdentifyContainers::createPass(containerInfos_));
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 9c76238..931aa83 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -41,8 +41,9 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config);
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   struct ExactName {
@@ -65,6 +66,7 @@ class CodeGen {
                        std::list<drgn_type>& drgnTypes,
                        drgn_type** rootType) const;
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -77,7 +79,7 @@ class CodeGen {
  private:
   type_graph::TypeGraph typeGraph_;
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
diff --git a/oi/Config.cpp b/oi/Config.cpp
index 157b279..66798bb 100644
--- a/oi/Config.cpp
+++ b/oi/Config.cpp
@@ -22,6 +22,8 @@
 #include <boost/property_tree/ini_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 #include <filesystem>
+#include <range/v3/action/sort.hpp>
+#include <range/v3/action/unique.hpp>
 
 #include "oi/support/Toml.h"
 
@@ -54,6 +56,9 @@ std::optional<FeatureSet> processConfigFiles(
     enables |= *fs;
   }
 
+  ranges::actions::sort(generatorConfig.containerConfigPaths);
+  ranges::actions::unique(generatorConfig.containerConfigPaths);
+
   // Override anything from the config files with command line options
   for (auto [k, v] : featureMap) {
     enables[k] = v;
@@ -108,8 +113,8 @@ std::optional<FeatureSet> processConfigFile(
           if the right path is absolute, else append the right path to the
           left path.
           */
-          generatorConfig.containerConfigPaths.emplace(configDirectory /
-                                                       el.get());
+          generatorConfig.containerConfigPaths.emplace_back(configDirectory /
+                                                            el.get());
         }
       });
     }
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 4cb20b8..b83b7e4 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -202,7 +202,7 @@ std::regex getMatcher(const std::string& typeName) {
 ContainerInfo::ContainerInfo(const fs::path& path) {
   toml::table container;
   try {
-    container = toml::parse_file(std::string(path));
+    container = toml::parse_file(path.string());
   } catch (const toml::parse_error& err) {
     // Convert into a ContainerInfoError, just to avoid having to include
     // the huge TOML++ header in the caller's file. Use toml::parse_error's
diff --git a/oi/OICodeGen.h b/oi/OICodeGen.h
index a01548f..7226a4e 100644
--- a/oi/OICodeGen.h
+++ b/oi/OICodeGen.h
@@ -66,7 +66,7 @@ class OICodeGen {
     };
 
     FeatureSet features;
-    std::set<std::filesystem::path> containerConfigPaths;
+    std::vector<std::filesystem::path> containerConfigPaths;
     std::set<std::string> defaultHeaders;
     std::set<std::string> defaultNamespaces;
     std::vector<std::pair<std::string, std::string>> membersToStub;
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 1cb2a49..cf7ec44 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -16,130 +16,156 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
 #include <glog/logging.h>
 
 #include <boost/core/demangle.hpp>
 #include <fstream>
 #include <iostream>
+#include <range/v3/action/transform.hpp>
+#include <range/v3/core.hpp>
+#include <range/v3/view/cache1.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/drop_while.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>  // TODO: remove
 #include <string_view>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
+class ConsumerContext;
 
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
-      templateArgsToSymbolsMap;
-
-  auto symbols = prog.find_all_symbols();
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
+class CreateTypeGraphConsumer;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
-    if (val.first.empty() || val.second.empty()) {
-      continue;
-    }
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  return strongToWeakSymbols;
-}
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
 
-std::unordered_map<std::string, drgn_qualified_type>
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
+ private:
+  ConsumerContext& ctx;
+};
 
-  std::unordered_map<std::string, drgn_qualified_type> out;
+class ConsumerContext {
+ public:
+  ConsumerContext(const std::vector<std::unique_ptr<ContainerInfo>>& cis)
+      : containerInfos{cis} {
+  }
 
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
-    std::string strongLinkageName;
-    {
-      const char* linkageNameCstr;
-      if (auto err = drgnplusplus::error(
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
-        // throw err;
+  type_graph::TypeGraph typeGraph;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
+  std::optional<bool> pic;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+}  // namespace
+
+int OIGenerator::generate(clang::tooling::CompilationDatabase& db,
+                          const std::vector<std::string>& sourcePaths) {
+  std::map<Feature, bool> featuresMap = {
+      {Feature::TypeGraph, true},
+      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},
+      {Feature::PackStructs, true},
+      {Feature::PruneTypeGraph, true},
+  };
+
+  OICodeGen::Config generatorConfig{};
+  OICompiler::Config compilerConfig{};
+
+  auto features = config::processConfigFiles(
+      configFilePaths, featuresMap, compilerConfig, generatorConfig);
+  if (!features) {
+    LOG(ERROR) << "failed to process config file";
+    return -1;
+  }
+  generatorConfig.features = *features;
+  compilerConfig.features = *features;
+
+  std::vector<std::unique_ptr<ContainerInfo>> containerInfos;
+  containerInfos.reserve(generatorConfig.containerConfigPaths.size());
+  try {
+    for (const auto& path : generatorConfig.containerConfigPaths) {
+      auto info = std::make_unique<ContainerInfo>(path);
+      if (info->requiredFeatures != (*features & info->requiredFeatures)) {
+        VLOG(1) << "Skipping container (feature conflict): " << info->typeName;
         continue;
       }
-      strongLinkageName = linkageNameCstr;
+      containerInfos.emplace_back(std::move(info));
     }
-
-    std::string weakLinkageName;
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return -1;
   }
 
-  return out;
-}
+  ConsumerContext ctx{containerInfos};
+  CreateTypeGraphActionFactory factory{ctx};
 
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
-                                      const OICompiler::Config& compilerConfig,
-                                      const drgn_qualified_type& type,
-                                      const std::string& linkageName,
-                                      SymbolService& symbols) {
-  CodeGen codegen{generatorConfig, symbols};
+  clang::tooling::ClangTool tool{db, sourcePaths};
+  if (auto ret = tool.run(&factory); ret != 0) {
+    return ret;
+  }
+
+  if (ctx.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one type to generate for but we can't currently "
+        "handle this case");
+
+  if (ctx.nameToTypeMap.empty()) {
+    LOG(ERROR) << "Nothing to generate!";
+    return -1;
+  }
+  const auto& linkageName = ctx.nameToTypeMap.begin()->first;
+
+  compilerConfig.usePIC = ctx.pic.value();
+  CodeGen codegen{generatorConfig};
+  for (auto&& ptr : containerInfos)
+    codegen.registerContainer(std::move(ptr));
+  codegen.transform(ctx.typeGraph);
 
   std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
+  codegen.generate(ctx.typeGraph, code, CodeGen::ExactName{linkageName});
 
   std::string sourcePath = sourceFileDumpPath;
   if (sourceFileDumpPath.empty()) {
@@ -152,78 +178,110 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
   }
 
   OICompiler compiler{{}, compilerConfig};
-
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
-
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
 }
 
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
-  drgnplusplus::program prog;
+namespace {
 
-  {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(
-            drgn_program_load_debug_info(prog.get(),
-                                         std::data(objectPaths),
-                                         std::size(objectPaths),
-                                         false,
-                                         false))) {
-      LOG(ERROR) << "error loading debug info program: " << err;
-      throw err;
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx(ctx_) {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
     }
-  }
 
-  auto oilTypes = findOilTypesAndNames(prog);
-
-  std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
-      {Feature::PruneTypeGraph, true},
-  };
-
-  OICodeGen::Config generatorConfig{};
-  OICompiler::Config compilerConfig{};
-  compilerConfig.usePIC = pic;
-
-  auto features = config::processConfigFiles(
-      configFilePaths, featuresMap, compilerConfig, generatorConfig);
-  if (!features) {
-    LOG(ERROR) << "failed to process config file";
-    return -1;
-  }
-  generatorConfig.features = *features;
-  compilerConfig.features = *features;
-
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
+    auto introspectImpl =
+        std::move(oi_namespaces) |
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
+        ranges::views::transform([](auto* p) {
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
+        }) |
+        ranges::views::filter([](auto* td) {
+          return td != nullptr && td->getName() == "introspectImpl";
+        }) |
+        ranges::views::take(1) | ranges::to<std::vector>();
+    if (introspectImpl.empty()) {
+      LOG(WARNING)
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
+             "you compile with `OIL_AOT_COMPILATION=1`?";
+      return;
     }
-  }
 
-  size_t successes = oilTypes.size() - failures;
-  LOG(INFO) << "object introspection generation complete. " << successes
-            << " successes and " << failures << " failures.";
+    auto nameToClangTypeMap =
+        ranges::views::single(introspectImpl[0]) |
+        ranges::views::for_each(
+            [](auto* td) { return td->specializations(); }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
 
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
-    return -1;
+              assert(fd->getNumParams() == 1);
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, opts};
+
+    auto& Sema = *ctx.sema;
+    ctx.nameToTypeMap =
+        nameToClangTypeMap |
+        ranges::views::transform(
+            [&parser, &Context, &Sema](
+                auto& p) -> std::pair<std::string, type_graph::Type*> {
+              return {p.first, &parser.parse(Context, Sema, *p.second)};
+            }) |
+        ranges::to<std::unordered_map>();
+
+    for (const auto& [name, type] : ctx.nameToTypeMap)
+      ctx.typeGraph.addRoot(*type);
   }
-  return 0;
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  // Compile the output as position independent if any input is position
+  // independent
+  bool pic = CI.getCodeGenOpts().RelocationModel == llvm::Reloc::PIC_;
+  ctx.pic = ctx.pic.value_or(false) || pic;
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
 }
 
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+}  // namespace
 }  // namespace oi::detail
diff --git a/oi/OIGenerator.h b/oi/OIGenerator.h
index dd38384..fac947a 100644
--- a/oi/OIGenerator.h
+++ b/oi/OIGenerator.h
@@ -23,11 +23,19 @@
 #include "oi/OICodeGen.h"
 #include "oi/OICompiler.h"
 
+namespace clang::tooling {
+class CompilationDatabase;
+}
+
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate(clang::tooling::CompilationDatabase&,
+               const std::vector<std::string>&);
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -41,8 +49,8 @@ class OIGenerator {
   void setFailIfNothingGenerated(bool fail) {
     failIfNothingGenerated = fail;
   }
-  void setUsePIC(bool pic_) {
-    pic = pic_;
+  void setClangArgs(std::vector<std::string> args_) {
+    clangArgs = std::move(args_);
   }
 
  private:
@@ -50,20 +58,7 @@ class OIGenerator {
   std::vector<std::filesystem::path> configFilePaths;
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
-  bool pic = false;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
+  std::vector<std::string> clangArgs;
 };
 
 }  // namespace oi::detail
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 190bdfc..6c4da13 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnExporter.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
@@ -27,3 +28,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
new file mode 100644
index 0000000..67f7bfa
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -0,0 +1,416 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Basic/DiagnosticSema.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <iostream>  // TODO: maybe remove
+#include <regex>
+#include <stdexcept>  // TODO: remove
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (VLOG_IS_ON(3)) {
+    std::string fqName = clang::TypeName::getFullyQualifiedName(
+        clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+    VLOG(3) << std::string(depth_ * 2, ' ') << fqName;
+  }
+
+  if (!requireCompleteType(*sema, ty))
+    return makeType<Incomplete>(ty, "incomplete (TODO naming)");
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(llvm::cast<const clang::RecordType>(ty));
+    case clang::Type::LValueReference:
+      return enumerateReference(
+          llvm::cast<const clang::LValueReferenceType>(ty));
+    case clang::Type::Pointer:
+      return enumeratePointer(llvm::cast<const clang::PointerType>(ty));
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          llvm::cast<const clang::SubstTemplateTypeParmType>(ty));
+    case clang::Type::Builtin:
+      return enumeratePrimitive(llvm::cast<const clang::BuiltinType>(ty));
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(
+          llvm::cast<const clang::ElaboratedType>(ty));
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          llvm::cast<const clang::TemplateSpecializationType>(ty));
+    case clang::Type::UnaryTransform:
+      return enumerateUnaryTransformType(
+          llvm::cast<const clang::UnaryTransformType>(ty));
+    case clang::Type::Decltype:
+      return enumerateDecltypeType(llvm::cast<const clang::DecltypeType>(ty));
+    case clang::Type::Typedef:
+      return enumerateTypedef(llvm::cast<const clang::TypedefType>(ty));
+    case clang::Type::Using:
+      return enumerateUsing(llvm::cast<const clang::UsingType>(ty));
+    case clang::Type::ConstantArray:
+      return enumerateArray(llvm::cast<const clang::ConstantArrayType>(ty));
+    case clang::Type::Enum:
+      return enumerateEnum(llvm::cast<const clang::EnumType>(ty));
+
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Type& ClangTypeParser::enumerateDecltypeType(const clang::DecltypeType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Type& ClangTypeParser::enumerateUnaryTransformType(
+    const clang::UnaryTransformType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Typedef& ClangTypeParser::enumerateUsing(const clang::UsingType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+  std::string name = ty.getFoundDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+
+  std::string name = ty.getDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string name = ty.getDecl()->getNameAsString();
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  std::map<int64_t, std::string> enumeratorMap;
+  if (options_.readEnumValues) {
+    for (const auto* enumerator : ty.getDecl()->enumerators()) {
+      enumeratorMap.emplace(enumerator->getInitVal().getExtValue(),
+                            enumerator->getNameAsString());
+    }
+  }
+
+  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+}
+
+Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
+  uint64_t len = ty.getSize().getLimitedValue();
+  auto& t = enumerateType(*ty.getElementType());
+  return makeType<Array>(ty, t, len);
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  if (auto* info = getContainerInfo(fqName)) {
+    auto& c = makeType<Container>(ty, *info, size, nullptr);
+    enumerateClassTemplateParams(ty, c.templateParams);
+    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)));
+    return c;
+  }
+
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+
+  params.reserve(list.size());
+  for (const auto& arg : list.asArray()) {
+    if (auto p = enumerateTemplateParam(arg))
+      params.emplace_back(std::move(p.value()));
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return TemplateParam{ttype, qualifiers};
+    }
+    case clang::TemplateArgument::Integral: {
+      auto& ty = enumerateType(*p.getIntegralType());
+      llvm::SmallString<32> val;
+      p.getAsIntegral().toString(val);
+      return TemplateParam{ty, std::string(val)};
+    }
+    case clang::TemplateArgument::Template: {
+      return enumerateTemplateTemplateParam(p.getAsTemplate());
+    }
+
+#define X(name)                       \
+  case clang::TemplateArgument::name: \
+    throw std::logic_error("unsupported template argument kind: " #name);
+
+      X(Null)
+      X(Declaration)
+      X(NullPtr)
+      X(TemplateExpansion)
+      X(Expression)
+      X(Pack)
+#undef X
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateTemplateParam(
+    const clang::TemplateName& tn) {
+  switch (tn.getKind()) {
+    case clang::TemplateName::Template:
+      return std::nullopt;
+
+#define X(name)                   \
+  case clang::TemplateName::name: \
+    throw std::logic_error("unsupported template name kind: " #name);
+
+      X(OverloadedTemplate)
+      X(AssumedTemplate)
+      X(QualifiedTemplate)
+      X(DependentTemplate)
+      X(SubstTemplateTemplateParm)
+      X(SubstTemplateTemplateParmPack)
+      X(UsingTemplate)
+#undef X
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function references
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Float:
+      return makeType<Primitive>(ty, Primitive::Kind::Float32);
+    case clang::BuiltinType::Double:
+    case clang::BuiltinType::LongDouble:
+      return makeType<Primitive>(ty, Primitive::Kind::Float64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+ContainerInfo* ClangTypeParser::getContainerInfo(
+    const std::string& fqName) const {
+  for (const auto& containerInfo : containers_) {
+    if (std::regex_search(fqName, containerInfo->matcher)) {
+      return containerInfo.get();
+    }
+  }
+  return nullptr;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
+    return true;  // treat as complete
+
+  // TODO: This is a terrible warning.
+  return !sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      clang::diag::warn_nsconsumed_attribute_mismatch);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
new file mode 100644
index 0000000..10eea87
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.h
@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ASTContext;
+class BuiltinType;
+class ConstantArrayType;
+class DecltypeType;
+class ElaboratedType;
+class EnumType;
+class LValueReferenceType;
+class PointerType;
+class RecordType;
+class Sema;
+class SubstTemplateTypeParmType;
+class TemplateArgument;
+class TemplateName;
+class TemplateSpecializationType;
+class Type;
+class TypedefType;
+class UnaryTransformType;
+class UsingType;
+}  // namespace clang
+
+struct ContainerInfo;
+
+namespace oi::detail::type_graph {
+
+class Array;
+class Class;
+class Enum;
+class Member;
+class Primitive;
+class Reference;
+class Type;
+class TypeGraph;
+class Typedef;
+struct TemplateParam;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+  bool readEnumValues = false;
+};
+
+/*
+ * ClangTypeParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given drgn_type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph,
+                  const std::vector<std::unique_ptr<ContainerInfo>>& containers,
+                  ClangTypeParserOptions options)
+      : typeGraph_{typeGraph}, containers_{containers}, options_{options} {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containers_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Type& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+  Typedef& enumerateTypedef(const clang::TypedefType&);
+  Typedef& enumerateUsing(const clang::UsingType&);
+  Type& enumerateUnaryTransformType(const clang::UnaryTransformType&);
+  Type& enumerateDecltypeType(const clang::DecltypeType&);
+
+  Array& enumerateArray(const clang::ConstantArrayType&);
+  Enum& enumerateEnum(const clang::EnumType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  std::optional<TemplateParam> enumerateTemplateParam(
+      const clang::TemplateArgument&);
+  std::optional<TemplateParam> enumerateTemplateTemplateParam(
+      const clang::TemplateName&);
+
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  ContainerInfo* getContainerInfo(const std::string& fqName) const;
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/DrgnExporter.cpp b/oi/type_graph/DrgnExporter.cpp
index 802857e..df9722d 100644
--- a/oi/type_graph/DrgnExporter.cpp
+++ b/oi/type_graph/DrgnExporter.cpp
@@ -200,6 +200,14 @@ drgn_type* DrgnExporter::visit(Pointer& p) {
   return drgnType;
 }
 
+drgn_type* DrgnExporter::visit(Reference& p) {
+  auto* drgnType =
+      makeDrgnType(DRGN_TYPE_POINTER, false, DRGN_NOT_PRIMITIVE_TYPE, p);
+  auto* pointeeType = accept(p.pointeeType());
+  th_.pointerToTypeMap[drgnType] = pointeeType;
+  return drgnType;
+}
+
 drgn_type* DrgnExporter::visit(Dummy& d) {
   return makeDrgnType(DRGN_TYPE_VOID, false, DRGN_C_TYPE_VOID, d);
 }
diff --git a/oi/type_graph/DrgnExporter.h b/oi/type_graph/DrgnExporter.h
index 45f02c6..24f4c59 100644
--- a/oi/type_graph/DrgnExporter.h
+++ b/oi/type_graph/DrgnExporter.h
@@ -54,6 +54,7 @@ class DrgnExporter : public Visitor<drgn_type*> {
   drgn_type* visit(Array&) override;
   drgn_type* visit(Typedef&) override;
   drgn_type* visit(Pointer&) override;
+  drgn_type* visit(Reference&) override;
   drgn_type* visit(Dummy&) override;
   drgn_type* visit(DummyAllocator&) override;
   drgn_type* visit(CaptureKeys&) override;
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index b1d1e8d..f016c13 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -200,6 +200,14 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& p) {
+  RecursiveVisitor::visit(p);
+  p.regenerateName();
+  std::string inputName{p.pointeeType().inputName()};
+  inputName += '*';
+  p.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..e6437d0 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index 0dae0ab..49165ab 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -150,6 +150,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& p) {
+  if (prefix(p))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = p.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(p.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index 8d7138a..e08a943 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& p) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;
diff --git a/oi/type_graph/RemoveTopLevelPointer.cpp b/oi/type_graph/RemoveTopLevelPointer.cpp
index 9ecaaaa..e1909a7 100644
--- a/oi/type_graph/RemoveTopLevelPointer.cpp
+++ b/oi/type_graph/RemoveTopLevelPointer.cpp
@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& p) {
+  topLevelType_ = &p.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/RemoveTopLevelPointer.h b/oi/type_graph/RemoveTopLevelPointer.h
index 611429f..6b70545 100644
--- a/oi/type_graph/RemoveTopLevelPointer.h
+++ b/oi/type_graph/RemoveTopLevelPointer.h
@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
 
  private:
   Type* topLevelType_ = nullptr;
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f0b3fe9..38d53c6 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -734,6 +735,61 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    name_ = pointeeType_.get().name() + "&";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index a7f7eec..f055aea 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -108,6 +108,9 @@ class RecursiveVisitor : public Visitor<void> {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -175,6 +178,10 @@ class RecursiveMutator : public Visitor<Type&> {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }
diff --git a/test/integration/template_template_param.toml b/test/integration/template_template_param.toml
new file mode 100644
index 0000000..c64590d
--- /dev/null
+++ b/test/integration/template_template_param.toml
@@ -0,0 +1,16 @@
+includes = ["vector", "utility", "string"]
+
+definitions = '''
+template <typename K, typename V, template <typename, typename> typename Pair>
+struct bad_map {
+  std::vector<Pair<K, K>> keys;
+  std::vector<Pair<V, V>> values;
+};
+'''
+
+[cases]
+  [cases.int_int_empty]
+    param_types = ["const bad_map<int, std::string, std::pair>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index d1e345d..795389e 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -14,131 +14,81 @@
  * limitations under the License.
  */
 
+#include <clang/Tooling/CommonOptionsParser.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include <cstdlib>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/action/drop_while.hpp>
+#include <range/v3/algorithm/find.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/all.hpp>
+#include <range/v3/view/transform.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
 
-constexpr static OIOpts opts{
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
+static llvm::cl::OptionCategory OilgenCategory("oilgen options");
 
-void usage() {
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
-  std::cerr << opts;
+static llvm::cl::list<std::string> ConfigFiles("config-file",
+                                               llvm::cl::desc(R"(TODO HELP)"),
+                                               llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<int> DebugLevel("debug-level",
+                                     llvm::cl::desc(R"(TODO HELP)"),
+                                     llvm::cl::init(-1),
+                                     llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> DumpJit("dump-jit",
+                                          llvm::cl::desc(R"(TODO HELP)"),
+                                          llvm::cl::init("jit.cpp"),
+                                          llvm::cl::cat(OilgenCategory));
 
-  std::cerr << std::endl
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
-
-int main(int argc, char* argv[]) {
+int main(int argc, const char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
-  std::vector<fs::path> configFilePaths;
-  fs::path sourceFileDumpPath = "";
-  bool exitCode = false;
-  bool pic = false;
-
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
+  auto expectedParser =
+      clang::tooling::CommonOptionsParser::create(argc, argv, OilgenCategory);
+  if (!expectedParser) {
+    llvm::errs() << expectedParser.takeError();
+    return -1;
   }
+  clang::tooling::CommonOptionsParser& options = expectedParser.get();
+  auto& compilations = options.getCompilations();
 
-  if (optind >= argc) {
-    usage();
-    return EXIT_FAILURE;
-  }
-  fs::path primaryObject = argv[optind];
-
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
+  if (DebugLevel.getNumOccurrences()) {
+    google::LogToStderr();
+    google::SetStderrLogging(google::INFO);
+    google::SetVLOGLevel("*", DebugLevel);
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
+    gflags::SetCommandLineOption("minloglevel", "0");
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
-  oigen.setConfigFilePaths(std::move(configFilePaths));
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
-  oigen.setFailIfNothingGenerated(exitCode);
-  oigen.setUsePIC(pic);
+  oigen.setConfigFilePaths(ConfigFiles |
+                           ranges::views::transform([](const auto& p) {
+                             return std::filesystem::path(p);
+                           }) |
+                           ranges::to<std::vector>());
+  if (DumpJit.getNumOccurrences())
+    oigen.setSourceFileDumpPath(DumpJit.getValue());
 
-  SymbolService symbols(primaryObject);
+  oigen.setOutputPath("a.o");  // TODO: make this an opt
 
-  return oigen.generate(primaryObject, symbols);
+  // TODO: can we extract this from the original arguments?
+  // oigen.setUsePIC(args["pic"].as<bool>());
+
+  oigen.setFailIfNothingGenerated(true);
+  // compilations.getAllFiles();
+  // return oigen.generate(compilations,
+  // {"/data/users/jakehillion/fbsource/fbcode/object-introspection/oil/examples/compile-time/VectorOfStrings-Introspect.cpp"});
+  return oigen.generate(compilations, options.getSourcePathList());
 }
diff --git a/types/string_type.toml b/types/string_type.toml
index fa4a2ae..0d2ab0d 100644
--- a/types/string_type.toml
+++ b/types/string_type.toml
@@ -35,3 +35,34 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
     );
 }
 """
+
+traversal_func = """
+    bool sso = ((uintptr_t)container.data() <
+                (uintptr_t)(&container + sizeof(std::__cxx11::basic_string<T0>))) &&
+               ((uintptr_t)container.data() >= (uintptr_t)&container);
+
+    return returnArg.write(container.capacity())
+        .write(sso)
+        .write(container.size());
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+uint64_t capacity = std::get<ParsedData::VarInt>(d.val).value;
+el.container_stats.emplace(result::Element::ContainerStats { .capacity = capacity });
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+bool sso = std::get<ParsedData::VarInt>(d.val).value;
+if (!sso)
+  el.exclusive_size += el.container_stats->capacity * sizeof(T0);
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats->length = std::get<ParsedData::VarInt>(d.val).value;
+"""

From 7693a415d2a805b85faf29e84273515df46b0365 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 19 Dec 2023 15:36:17 +0000
Subject: [PATCH 070/188] oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD
---
 CMakeLists.txt                                |   3 +-
 oi/CodeGen.cpp                                |  30 +-
 oi/CodeGen.h                                  |   6 +-
 oi/OIGenerator.cpp                            | 370 +++++++++-------
 oi/OIGenerator.h                              |  29 +-
 oi/type_graph/CMakeLists.txt                  |   2 +
 oi/type_graph/ClangTypeParser.cpp             | 416 ++++++++++++++++++
 oi/type_graph/ClangTypeParser.h               | 131 ++++++
 oi/type_graph/DrgnExporter.cpp                |   8 +
 oi/type_graph/DrgnExporter.h                  |   1 +
 oi/type_graph/NameGen.cpp                     |   8 +
 oi/type_graph/NameGen.h                       |   1 +
 oi/type_graph/Printer.cpp                     |  11 +
 oi/type_graph/Printer.h                       |   1 +
 oi/type_graph/RemoveTopLevelPointer.cpp       |   4 +
 oi/type_graph/RemoveTopLevelPointer.h         |   1 +
 oi/type_graph/Types.h                         |  56 +++
 oi/type_graph/Visitor.h                       |   7 +
 test/integration/template_template_param.toml |  16 +
 tools/OILGen.cpp                              | 148 +++----
 types/string_type.toml                        |  31 ++
 21 files changed, 995 insertions(+), 285 deletions(-)
 create mode 100644 oi/type_graph/ClangTypeParser.cpp
 create mode 100644 oi/type_graph/ClangTypeParser.h
 create mode 100644 test/integration/template_template_param.toml

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f9ee7..b6fecd1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -289,7 +289,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -365,6 +365,7 @@ add_executable(oilgen
 target_link_libraries(oilgen
   drgn_utils
   oicore
+  clangTooling
 )
 
 ### Object Introspection cache Printer (OIP)
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 74c6f39..869fc20 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -299,6 +299,10 @@ void genDefsThriftClass(const Class& c, std::string& code) {
 
 }  // namespace
 
+CodeGen::CodeGen(const OICodeGen::Config& config) : config_(config) {
+  DCHECK(!config.features[Feature::PolymorphicInheritance]) << "polymorphic inheritance requires symbol service!";
+}
+
 void CodeGen::genDefsThrift(const TypeGraph& typeGraph, std::string& code) {
   for (const Type& t : typeGraph.finalTypes) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
@@ -542,7 +546,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1115,15 +1119,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
                               std::string& code,
                               RootFunctionName name) {
-  try {
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
+  if (!registerContainers())
     return false;
-  }
 
   try {
     addDrgnRoot(drgnType, typeGraph_);
@@ -1148,6 +1145,19 @@ void CodeGen::exportDrgnTypes(TypeHierarchy& th,
   }
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1189,7 +1199,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(IdentifyContainers::createPass(containerInfos_));
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 9c76238..8fc5a81 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -41,8 +41,9 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config);
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   struct ExactName {
@@ -65,6 +66,7 @@ class CodeGen {
                        std::list<drgn_type>& drgnTypes,
                        drgn_type** rootType) const;
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -77,7 +79,7 @@ class CodeGen {
  private:
   type_graph::TypeGraph typeGraph_;
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_ = nullptr;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 1cb2a49..cf7ec44 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -16,130 +16,156 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
 #include <glog/logging.h>
 
 #include <boost/core/demangle.hpp>
 #include <fstream>
 #include <iostream>
+#include <range/v3/action/transform.hpp>
+#include <range/v3/core.hpp>
+#include <range/v3/view/cache1.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/drop_while.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>  // TODO: remove
 #include <string_view>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
+class ConsumerContext;
 
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
-      templateArgsToSymbolsMap;
-
-  auto symbols = prog.find_all_symbols();
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
+class CreateTypeGraphConsumer;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
-    if (val.first.empty() || val.second.empty()) {
-      continue;
-    }
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  return strongToWeakSymbols;
-}
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
 
-std::unordered_map<std::string, drgn_qualified_type>
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
+ private:
+  ConsumerContext& ctx;
+};
 
-  std::unordered_map<std::string, drgn_qualified_type> out;
+class ConsumerContext {
+ public:
+  ConsumerContext(const std::vector<std::unique_ptr<ContainerInfo>>& cis)
+      : containerInfos{cis} {
+  }
 
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
-    std::string strongLinkageName;
-    {
-      const char* linkageNameCstr;
-      if (auto err = drgnplusplus::error(
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
-        // throw err;
+  type_graph::TypeGraph typeGraph;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
+  std::optional<bool> pic;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+}  // namespace
+
+int OIGenerator::generate(clang::tooling::CompilationDatabase& db,
+                          const std::vector<std::string>& sourcePaths) {
+  std::map<Feature, bool> featuresMap = {
+      {Feature::TypeGraph, true},
+      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},
+      {Feature::PackStructs, true},
+      {Feature::PruneTypeGraph, true},
+  };
+
+  OICodeGen::Config generatorConfig{};
+  OICompiler::Config compilerConfig{};
+
+  auto features = config::processConfigFiles(
+      configFilePaths, featuresMap, compilerConfig, generatorConfig);
+  if (!features) {
+    LOG(ERROR) << "failed to process config file";
+    return -1;
+  }
+  generatorConfig.features = *features;
+  compilerConfig.features = *features;
+
+  std::vector<std::unique_ptr<ContainerInfo>> containerInfos;
+  containerInfos.reserve(generatorConfig.containerConfigPaths.size());
+  try {
+    for (const auto& path : generatorConfig.containerConfigPaths) {
+      auto info = std::make_unique<ContainerInfo>(path);
+      if (info->requiredFeatures != (*features & info->requiredFeatures)) {
+        VLOG(1) << "Skipping container (feature conflict): " << info->typeName;
         continue;
       }
-      strongLinkageName = linkageNameCstr;
+      containerInfos.emplace_back(std::move(info));
     }
-
-    std::string weakLinkageName;
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return -1;
   }
 
-  return out;
-}
+  ConsumerContext ctx{containerInfos};
+  CreateTypeGraphActionFactory factory{ctx};
 
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
-                                      const OICompiler::Config& compilerConfig,
-                                      const drgn_qualified_type& type,
-                                      const std::string& linkageName,
-                                      SymbolService& symbols) {
-  CodeGen codegen{generatorConfig, symbols};
+  clang::tooling::ClangTool tool{db, sourcePaths};
+  if (auto ret = tool.run(&factory); ret != 0) {
+    return ret;
+  }
+
+  if (ctx.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one type to generate for but we can't currently "
+        "handle this case");
+
+  if (ctx.nameToTypeMap.empty()) {
+    LOG(ERROR) << "Nothing to generate!";
+    return -1;
+  }
+  const auto& linkageName = ctx.nameToTypeMap.begin()->first;
+
+  compilerConfig.usePIC = ctx.pic.value();
+  CodeGen codegen{generatorConfig};
+  for (auto&& ptr : containerInfos)
+    codegen.registerContainer(std::move(ptr));
+  codegen.transform(ctx.typeGraph);
 
   std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
+  codegen.generate(ctx.typeGraph, code, CodeGen::ExactName{linkageName});
 
   std::string sourcePath = sourceFileDumpPath;
   if (sourceFileDumpPath.empty()) {
@@ -152,78 +178,110 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
   }
 
   OICompiler compiler{{}, compilerConfig};
-
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
-
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
 }
 
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
-  drgnplusplus::program prog;
+namespace {
 
-  {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(
-            drgn_program_load_debug_info(prog.get(),
-                                         std::data(objectPaths),
-                                         std::size(objectPaths),
-                                         false,
-                                         false))) {
-      LOG(ERROR) << "error loading debug info program: " << err;
-      throw err;
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx(ctx_) {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
     }
-  }
 
-  auto oilTypes = findOilTypesAndNames(prog);
-
-  std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
-      {Feature::PruneTypeGraph, true},
-  };
-
-  OICodeGen::Config generatorConfig{};
-  OICompiler::Config compilerConfig{};
-  compilerConfig.usePIC = pic;
-
-  auto features = config::processConfigFiles(
-      configFilePaths, featuresMap, compilerConfig, generatorConfig);
-  if (!features) {
-    LOG(ERROR) << "failed to process config file";
-    return -1;
-  }
-  generatorConfig.features = *features;
-  compilerConfig.features = *features;
-
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
+    auto introspectImpl =
+        std::move(oi_namespaces) |
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
+        ranges::views::transform([](auto* p) {
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
+        }) |
+        ranges::views::filter([](auto* td) {
+          return td != nullptr && td->getName() == "introspectImpl";
+        }) |
+        ranges::views::take(1) | ranges::to<std::vector>();
+    if (introspectImpl.empty()) {
+      LOG(WARNING)
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
+             "you compile with `OIL_AOT_COMPILATION=1`?";
+      return;
     }
-  }
 
-  size_t successes = oilTypes.size() - failures;
-  LOG(INFO) << "object introspection generation complete. " << successes
-            << " successes and " << failures << " failures.";
+    auto nameToClangTypeMap =
+        ranges::views::single(introspectImpl[0]) |
+        ranges::views::for_each(
+            [](auto* td) { return td->specializations(); }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
 
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
-    return -1;
+              assert(fd->getNumParams() == 1);
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, opts};
+
+    auto& Sema = *ctx.sema;
+    ctx.nameToTypeMap =
+        nameToClangTypeMap |
+        ranges::views::transform(
+            [&parser, &Context, &Sema](
+                auto& p) -> std::pair<std::string, type_graph::Type*> {
+              return {p.first, &parser.parse(Context, Sema, *p.second)};
+            }) |
+        ranges::to<std::unordered_map>();
+
+    for (const auto& [name, type] : ctx.nameToTypeMap)
+      ctx.typeGraph.addRoot(*type);
   }
-  return 0;
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  // Compile the output as position independent if any input is position
+  // independent
+  bool pic = CI.getCodeGenOpts().RelocationModel == llvm::Reloc::PIC_;
+  ctx.pic = ctx.pic.value_or(false) || pic;
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
 }
 
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+}  // namespace
 }  // namespace oi::detail
diff --git a/oi/OIGenerator.h b/oi/OIGenerator.h
index dd38384..fac947a 100644
--- a/oi/OIGenerator.h
+++ b/oi/OIGenerator.h
@@ -23,11 +23,19 @@
 #include "oi/OICodeGen.h"
 #include "oi/OICompiler.h"
 
+namespace clang::tooling {
+class CompilationDatabase;
+}
+
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate(clang::tooling::CompilationDatabase&,
+               const std::vector<std::string>&);
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -41,8 +49,8 @@ class OIGenerator {
   void setFailIfNothingGenerated(bool fail) {
     failIfNothingGenerated = fail;
   }
-  void setUsePIC(bool pic_) {
-    pic = pic_;
+  void setClangArgs(std::vector<std::string> args_) {
+    clangArgs = std::move(args_);
   }
 
  private:
@@ -50,20 +58,7 @@ class OIGenerator {
   std::vector<std::filesystem::path> configFilePaths;
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
-  bool pic = false;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
+  std::vector<std::string> clangArgs;
 };
 
 }  // namespace oi::detail
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 190bdfc..6c4da13 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnExporter.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
@@ -27,3 +28,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
new file mode 100644
index 0000000..67f7bfa
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -0,0 +1,416 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Basic/DiagnosticSema.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <iostream>  // TODO: maybe remove
+#include <regex>
+#include <stdexcept>  // TODO: remove
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (VLOG_IS_ON(3)) {
+    std::string fqName = clang::TypeName::getFullyQualifiedName(
+        clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+    VLOG(3) << std::string(depth_ * 2, ' ') << fqName;
+  }
+
+  if (!requireCompleteType(*sema, ty))
+    return makeType<Incomplete>(ty, "incomplete (TODO naming)");
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(llvm::cast<const clang::RecordType>(ty));
+    case clang::Type::LValueReference:
+      return enumerateReference(
+          llvm::cast<const clang::LValueReferenceType>(ty));
+    case clang::Type::Pointer:
+      return enumeratePointer(llvm::cast<const clang::PointerType>(ty));
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          llvm::cast<const clang::SubstTemplateTypeParmType>(ty));
+    case clang::Type::Builtin:
+      return enumeratePrimitive(llvm::cast<const clang::BuiltinType>(ty));
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(
+          llvm::cast<const clang::ElaboratedType>(ty));
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          llvm::cast<const clang::TemplateSpecializationType>(ty));
+    case clang::Type::UnaryTransform:
+      return enumerateUnaryTransformType(
+          llvm::cast<const clang::UnaryTransformType>(ty));
+    case clang::Type::Decltype:
+      return enumerateDecltypeType(llvm::cast<const clang::DecltypeType>(ty));
+    case clang::Type::Typedef:
+      return enumerateTypedef(llvm::cast<const clang::TypedefType>(ty));
+    case clang::Type::Using:
+      return enumerateUsing(llvm::cast<const clang::UsingType>(ty));
+    case clang::Type::ConstantArray:
+      return enumerateArray(llvm::cast<const clang::ConstantArrayType>(ty));
+    case clang::Type::Enum:
+      return enumerateEnum(llvm::cast<const clang::EnumType>(ty));
+
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Type& ClangTypeParser::enumerateDecltypeType(const clang::DecltypeType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Type& ClangTypeParser::enumerateUnaryTransformType(
+    const clang::UnaryTransformType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Typedef& ClangTypeParser::enumerateUsing(const clang::UsingType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+  std::string name = ty.getFoundDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+
+  std::string name = ty.getDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string name = ty.getDecl()->getNameAsString();
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  std::map<int64_t, std::string> enumeratorMap;
+  if (options_.readEnumValues) {
+    for (const auto* enumerator : ty.getDecl()->enumerators()) {
+      enumeratorMap.emplace(enumerator->getInitVal().getExtValue(),
+                            enumerator->getNameAsString());
+    }
+  }
+
+  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+}
+
+Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
+  uint64_t len = ty.getSize().getLimitedValue();
+  auto& t = enumerateType(*ty.getElementType());
+  return makeType<Array>(ty, t, len);
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  if (auto* info = getContainerInfo(fqName)) {
+    auto& c = makeType<Container>(ty, *info, size, nullptr);
+    enumerateClassTemplateParams(ty, c.templateParams);
+    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)));
+    return c;
+  }
+
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+
+  params.reserve(list.size());
+  for (const auto& arg : list.asArray()) {
+    if (auto p = enumerateTemplateParam(arg))
+      params.emplace_back(std::move(p.value()));
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return TemplateParam{ttype, qualifiers};
+    }
+    case clang::TemplateArgument::Integral: {
+      auto& ty = enumerateType(*p.getIntegralType());
+      llvm::SmallString<32> val;
+      p.getAsIntegral().toString(val);
+      return TemplateParam{ty, std::string(val)};
+    }
+    case clang::TemplateArgument::Template: {
+      return enumerateTemplateTemplateParam(p.getAsTemplate());
+    }
+
+#define X(name)                       \
+  case clang::TemplateArgument::name: \
+    throw std::logic_error("unsupported template argument kind: " #name);
+
+      X(Null)
+      X(Declaration)
+      X(NullPtr)
+      X(TemplateExpansion)
+      X(Expression)
+      X(Pack)
+#undef X
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateTemplateParam(
+    const clang::TemplateName& tn) {
+  switch (tn.getKind()) {
+    case clang::TemplateName::Template:
+      return std::nullopt;
+
+#define X(name)                   \
+  case clang::TemplateName::name: \
+    throw std::logic_error("unsupported template name kind: " #name);
+
+      X(OverloadedTemplate)
+      X(AssumedTemplate)
+      X(QualifiedTemplate)
+      X(DependentTemplate)
+      X(SubstTemplateTemplateParm)
+      X(SubstTemplateTemplateParmPack)
+      X(UsingTemplate)
+#undef X
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function references
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Float:
+      return makeType<Primitive>(ty, Primitive::Kind::Float32);
+    case clang::BuiltinType::Double:
+    case clang::BuiltinType::LongDouble:
+      return makeType<Primitive>(ty, Primitive::Kind::Float64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+ContainerInfo* ClangTypeParser::getContainerInfo(
+    const std::string& fqName) const {
+  for (const auto& containerInfo : containers_) {
+    if (std::regex_search(fqName, containerInfo->matcher)) {
+      return containerInfo.get();
+    }
+  }
+  return nullptr;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
+    return true;  // treat as complete
+
+  // TODO: This is a terrible warning.
+  return !sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      clang::diag::warn_nsconsumed_attribute_mismatch);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
new file mode 100644
index 0000000..10eea87
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.h
@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ASTContext;
+class BuiltinType;
+class ConstantArrayType;
+class DecltypeType;
+class ElaboratedType;
+class EnumType;
+class LValueReferenceType;
+class PointerType;
+class RecordType;
+class Sema;
+class SubstTemplateTypeParmType;
+class TemplateArgument;
+class TemplateName;
+class TemplateSpecializationType;
+class Type;
+class TypedefType;
+class UnaryTransformType;
+class UsingType;
+}  // namespace clang
+
+struct ContainerInfo;
+
+namespace oi::detail::type_graph {
+
+class Array;
+class Class;
+class Enum;
+class Member;
+class Primitive;
+class Reference;
+class Type;
+class TypeGraph;
+class Typedef;
+struct TemplateParam;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+  bool readEnumValues = false;
+};
+
+/*
+ * ClangTypeParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given drgn_type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph,
+                  const std::vector<std::unique_ptr<ContainerInfo>>& containers,
+                  ClangTypeParserOptions options)
+      : typeGraph_{typeGraph}, containers_{containers}, options_{options} {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containers_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Type& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+  Typedef& enumerateTypedef(const clang::TypedefType&);
+  Typedef& enumerateUsing(const clang::UsingType&);
+  Type& enumerateUnaryTransformType(const clang::UnaryTransformType&);
+  Type& enumerateDecltypeType(const clang::DecltypeType&);
+
+  Array& enumerateArray(const clang::ConstantArrayType&);
+  Enum& enumerateEnum(const clang::EnumType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  std::optional<TemplateParam> enumerateTemplateParam(
+      const clang::TemplateArgument&);
+  std::optional<TemplateParam> enumerateTemplateTemplateParam(
+      const clang::TemplateName&);
+
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  ContainerInfo* getContainerInfo(const std::string& fqName) const;
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/DrgnExporter.cpp b/oi/type_graph/DrgnExporter.cpp
index 802857e..df9722d 100644
--- a/oi/type_graph/DrgnExporter.cpp
+++ b/oi/type_graph/DrgnExporter.cpp
@@ -200,6 +200,14 @@ drgn_type* DrgnExporter::visit(Pointer& p) {
   return drgnType;
 }
 
+drgn_type* DrgnExporter::visit(Reference& p) {
+  auto* drgnType =
+      makeDrgnType(DRGN_TYPE_POINTER, false, DRGN_NOT_PRIMITIVE_TYPE, p);
+  auto* pointeeType = accept(p.pointeeType());
+  th_.pointerToTypeMap[drgnType] = pointeeType;
+  return drgnType;
+}
+
 drgn_type* DrgnExporter::visit(Dummy& d) {
   return makeDrgnType(DRGN_TYPE_VOID, false, DRGN_C_TYPE_VOID, d);
 }
diff --git a/oi/type_graph/DrgnExporter.h b/oi/type_graph/DrgnExporter.h
index 45f02c6..24f4c59 100644
--- a/oi/type_graph/DrgnExporter.h
+++ b/oi/type_graph/DrgnExporter.h
@@ -54,6 +54,7 @@ class DrgnExporter : public Visitor<drgn_type*> {
   drgn_type* visit(Array&) override;
   drgn_type* visit(Typedef&) override;
   drgn_type* visit(Pointer&) override;
+  drgn_type* visit(Reference&) override;
   drgn_type* visit(Dummy&) override;
   drgn_type* visit(DummyAllocator&) override;
   drgn_type* visit(CaptureKeys&) override;
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index b1d1e8d..f016c13 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -200,6 +200,14 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& p) {
+  RecursiveVisitor::visit(p);
+  p.regenerateName();
+  std::string inputName{p.pointeeType().inputName()};
+  inputName += '*';
+  p.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..e6437d0 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index 0dae0ab..49165ab 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -150,6 +150,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& p) {
+  if (prefix(p))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = p.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(p.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index 8d7138a..e08a943 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& p) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;
diff --git a/oi/type_graph/RemoveTopLevelPointer.cpp b/oi/type_graph/RemoveTopLevelPointer.cpp
index 9ecaaaa..e1909a7 100644
--- a/oi/type_graph/RemoveTopLevelPointer.cpp
+++ b/oi/type_graph/RemoveTopLevelPointer.cpp
@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& p) {
+  topLevelType_ = &p.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/RemoveTopLevelPointer.h b/oi/type_graph/RemoveTopLevelPointer.h
index 611429f..6b70545 100644
--- a/oi/type_graph/RemoveTopLevelPointer.h
+++ b/oi/type_graph/RemoveTopLevelPointer.h
@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& p) override;
 
  private:
   Type* topLevelType_ = nullptr;
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f0b3fe9..38d53c6 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -734,6 +735,61 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    name_ = pointeeType_.get().name() + "&";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index a7f7eec..f055aea 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -108,6 +108,9 @@ class RecursiveVisitor : public Visitor<void> {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -175,6 +178,10 @@ class RecursiveMutator : public Visitor<Type&> {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }
diff --git a/test/integration/template_template_param.toml b/test/integration/template_template_param.toml
new file mode 100644
index 0000000..c64590d
--- /dev/null
+++ b/test/integration/template_template_param.toml
@@ -0,0 +1,16 @@
+includes = ["vector", "utility", "string"]
+
+definitions = '''
+template <typename K, typename V, template <typename, typename> typename Pair>
+struct bad_map {
+  std::vector<Pair<K, K>> keys;
+  std::vector<Pair<V, V>> values;
+};
+'''
+
+[cases]
+  [cases.int_int_empty]
+    param_types = ["const bad_map<int, std::string, std::pair>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index d1e345d..795389e 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -14,131 +14,81 @@
  * limitations under the License.
  */
 
+#include <clang/Tooling/CommonOptionsParser.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include <cstdlib>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/action/drop_while.hpp>
+#include <range/v3/algorithm/find.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/all.hpp>
+#include <range/v3/view/transform.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
 
-constexpr static OIOpts opts{
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
+static llvm::cl::OptionCategory OilgenCategory("oilgen options");
 
-void usage() {
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
-  std::cerr << opts;
+static llvm::cl::list<std::string> ConfigFiles("config-file",
+                                               llvm::cl::desc(R"(TODO HELP)"),
+                                               llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<int> DebugLevel("debug-level",
+                                     llvm::cl::desc(R"(TODO HELP)"),
+                                     llvm::cl::init(-1),
+                                     llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> DumpJit("dump-jit",
+                                          llvm::cl::desc(R"(TODO HELP)"),
+                                          llvm::cl::init("jit.cpp"),
+                                          llvm::cl::cat(OilgenCategory));
 
-  std::cerr << std::endl
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
-
-int main(int argc, char* argv[]) {
+int main(int argc, const char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
-  std::vector<fs::path> configFilePaths;
-  fs::path sourceFileDumpPath = "";
-  bool exitCode = false;
-  bool pic = false;
-
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
+  auto expectedParser =
+      clang::tooling::CommonOptionsParser::create(argc, argv, OilgenCategory);
+  if (!expectedParser) {
+    llvm::errs() << expectedParser.takeError();
+    return -1;
   }
+  clang::tooling::CommonOptionsParser& options = expectedParser.get();
+  auto& compilations = options.getCompilations();
 
-  if (optind >= argc) {
-    usage();
-    return EXIT_FAILURE;
-  }
-  fs::path primaryObject = argv[optind];
-
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
+  if (DebugLevel.getNumOccurrences()) {
+    google::LogToStderr();
+    google::SetStderrLogging(google::INFO);
+    google::SetVLOGLevel("*", DebugLevel);
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
+    gflags::SetCommandLineOption("minloglevel", "0");
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
-  oigen.setConfigFilePaths(std::move(configFilePaths));
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
-  oigen.setFailIfNothingGenerated(exitCode);
-  oigen.setUsePIC(pic);
+  oigen.setConfigFilePaths(ConfigFiles |
+                           ranges::views::transform([](const auto& p) {
+                             return std::filesystem::path(p);
+                           }) |
+                           ranges::to<std::vector>());
+  if (DumpJit.getNumOccurrences())
+    oigen.setSourceFileDumpPath(DumpJit.getValue());
 
-  SymbolService symbols(primaryObject);
+  oigen.setOutputPath("a.o");  // TODO: make this an opt
 
-  return oigen.generate(primaryObject, symbols);
+  // TODO: can we extract this from the original arguments?
+  // oigen.setUsePIC(args["pic"].as<bool>());
+
+  oigen.setFailIfNothingGenerated(true);
+  // compilations.getAllFiles();
+  // return oigen.generate(compilations,
+  // {"/data/users/jakehillion/fbsource/fbcode/object-introspection/oil/examples/compile-time/VectorOfStrings-Introspect.cpp"});
+  return oigen.generate(compilations, options.getSourcePathList());
 }
diff --git a/types/string_type.toml b/types/string_type.toml
index fa4a2ae..0d2ab0d 100644
--- a/types/string_type.toml
+++ b/types/string_type.toml
@@ -35,3 +35,34 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
     );
 }
 """
+
+traversal_func = """
+    bool sso = ((uintptr_t)container.data() <
+                (uintptr_t)(&container + sizeof(std::__cxx11::basic_string<T0>))) &&
+               ((uintptr_t)container.data() >= (uintptr_t)&container);
+
+    return returnArg.write(container.capacity())
+        .write(sso)
+        .write(container.size());
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+uint64_t capacity = std::get<ParsedData::VarInt>(d.val).value;
+el.container_stats.emplace(result::Element::ContainerStats { .capacity = capacity });
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+bool sso = std::get<ParsedData::VarInt>(d.val).value;
+if (!sso)
+  el.exclusive_size += el.container_stats->capacity * sizeof(T0);
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats->length = std::get<ParsedData::VarInt>(d.val).value;
+"""

From d80d19212b4d0277079de3791230ad533dcfb53e Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 19 Dec 2023 16:01:24 +0000
Subject: [PATCH 071/188] oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD
---
 CMakeLists.txt                                |   3 +-
 oi/CodeGen.cpp                                |  31 +-
 oi/CodeGen.h                                  |   6 +-
 oi/OIGenerator.cpp                            | 370 +++++++++-------
 oi/OIGenerator.h                              |  29 +-
 oi/type_graph/CMakeLists.txt                  |   2 +
 oi/type_graph/ClangTypeParser.cpp             | 416 ++++++++++++++++++
 oi/type_graph/ClangTypeParser.h               | 131 ++++++
 oi/type_graph/DrgnExporter.cpp                |   8 +
 oi/type_graph/DrgnExporter.h                  |   1 +
 oi/type_graph/NameGen.cpp                     |   9 +
 oi/type_graph/NameGen.h                       |   1 +
 oi/type_graph/Printer.cpp                     |  11 +
 oi/type_graph/Printer.h                       |   1 +
 oi/type_graph/RemoveTopLevelPointer.cpp       |   4 +
 oi/type_graph/RemoveTopLevelPointer.h         |   1 +
 oi/type_graph/Types.h                         |  61 +++
 oi/type_graph/Visitor.h                       |   7 +
 test/integration/template_template_param.toml |  16 +
 tools/OILGen.cpp                              | 150 +++----
 types/string_type.toml                        |  31 ++
 21 files changed, 1004 insertions(+), 285 deletions(-)
 create mode 100644 oi/type_graph/ClangTypeParser.cpp
 create mode 100644 oi/type_graph/ClangTypeParser.h
 create mode 100644 test/integration/template_template_param.toml

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f9ee7..b6fecd1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -289,7 +289,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -365,6 +365,7 @@ add_executable(oilgen
 target_link_libraries(oilgen
   drgn_utils
   oicore
+  clangTooling
 )
 
 ### Object Introspection cache Printer (OIP)
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 74c6f39..9f41c12 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -299,6 +299,11 @@ void genDefsThriftClass(const Class& c, std::string& code) {
 
 }  // namespace
 
+CodeGen::CodeGen(const OICodeGen::Config& config) : config_(config) {
+  DCHECK(!config.features[Feature::PolymorphicInheritance])
+      << "polymorphic inheritance requires symbol service!";
+}
+
 void CodeGen::genDefsThrift(const TypeGraph& typeGraph, std::string& code) {
   for (const Type& t : typeGraph.finalTypes) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
@@ -542,7 +547,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1115,15 +1120,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
                               std::string& code,
                               RootFunctionName name) {
-  try {
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
+  if (!registerContainers())
     return false;
-  }
 
   try {
     addDrgnRoot(drgnType, typeGraph_);
@@ -1148,6 +1146,19 @@ void CodeGen::exportDrgnTypes(TypeHierarchy& th,
   }
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1189,7 +1200,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(IdentifyContainers::createPass(containerInfos_));
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 9c76238..8fc5a81 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -41,8 +41,9 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config);
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   struct ExactName {
@@ -65,6 +66,7 @@ class CodeGen {
                        std::list<drgn_type>& drgnTypes,
                        drgn_type** rootType) const;
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -77,7 +79,7 @@ class CodeGen {
  private:
   type_graph::TypeGraph typeGraph_;
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_ = nullptr;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 1cb2a49..a7c46e4 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -16,130 +16,156 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
 #include <glog/logging.h>
 
 #include <boost/core/demangle.hpp>
 #include <fstream>
 #include <iostream>
+#include <range/v3/action/transform.hpp>
+#include <range/v3/core.hpp>
+#include <range/v3/view/cache1.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/drop_while.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>  // TODO: remove
 #include <string_view>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
+class ConsumerContext;
 
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
-      templateArgsToSymbolsMap;
-
-  auto symbols = prog.find_all_symbols();
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
+class CreateTypeGraphConsumer;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
-    if (val.first.empty() || val.second.empty()) {
-      continue;
-    }
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  return strongToWeakSymbols;
-}
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
 
-std::unordered_map<std::string, drgn_qualified_type>
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
+ private:
+  ConsumerContext& ctx;
+};
 
-  std::unordered_map<std::string, drgn_qualified_type> out;
+class ConsumerContext {
+ public:
+  ConsumerContext(const std::vector<std::unique_ptr<ContainerInfo>>& cis)
+      : containerInfos{cis} {
+  }
 
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
-    std::string strongLinkageName;
-    {
-      const char* linkageNameCstr;
-      if (auto err = drgnplusplus::error(
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
-        // throw err;
+  type_graph::TypeGraph typeGraph;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
+  std::optional<bool> pic;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+}  // namespace
+
+int OIGenerator::generate(clang::tooling::CompilationDatabase& db,
+                          const std::vector<std::string>& sourcePaths) {
+  std::map<Feature, bool> featuresMap = {
+      {Feature::TypeGraph, true},
+      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},
+      {Feature::PackStructs, true},
+      {Feature::PruneTypeGraph, true},
+  };
+
+  OICodeGen::Config generatorConfig{};
+  OICompiler::Config compilerConfig{};
+
+  auto features = config::processConfigFiles(
+      configFilePaths, featuresMap, compilerConfig, generatorConfig);
+  if (!features) {
+    LOG(ERROR) << "failed to process config file";
+    return -1;
+  }
+  generatorConfig.features = *features;
+  compilerConfig.features = *features;
+
+  std::vector<std::unique_ptr<ContainerInfo>> containerInfos;
+  containerInfos.reserve(generatorConfig.containerConfigPaths.size());
+  try {
+    for (const auto& path : generatorConfig.containerConfigPaths) {
+      auto info = std::make_unique<ContainerInfo>(path);
+      if (info->requiredFeatures != (*features & info->requiredFeatures)) {
+        VLOG(1) << "Skipping container (feature conflict): " << info->typeName;
         continue;
       }
-      strongLinkageName = linkageNameCstr;
+      containerInfos.emplace_back(std::move(info));
     }
-
-    std::string weakLinkageName;
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return -1;
   }
 
-  return out;
-}
+  ConsumerContext ctx{containerInfos};
+  CreateTypeGraphActionFactory factory{ctx};
 
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
-                                      const OICompiler::Config& compilerConfig,
-                                      const drgn_qualified_type& type,
-                                      const std::string& linkageName,
-                                      SymbolService& symbols) {
-  CodeGen codegen{generatorConfig, symbols};
+  clang::tooling::ClangTool tool{db, sourcePaths};
+  if (auto ret = tool.run(&factory); ret != 0) {
+    return ret;
+  }
+
+  if (ctx.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one type to generate for but we can't currently "
+        "handle this case");
+
+  if (ctx.nameToTypeMap.empty()) {
+    LOG(ERROR) << "Nothing to generate!";
+    return failIfNothingGenerated ? -1 : 0;
+  }
+  const auto& linkageName = ctx.nameToTypeMap.begin()->first;
+
+  compilerConfig.usePIC = ctx.pic.value();
+  CodeGen codegen{generatorConfig};
+  for (auto&& ptr : containerInfos)
+    codegen.registerContainer(std::move(ptr));
+  codegen.transform(ctx.typeGraph);
 
   std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
+  codegen.generate(ctx.typeGraph, code, CodeGen::ExactName{linkageName});
 
   std::string sourcePath = sourceFileDumpPath;
   if (sourceFileDumpPath.empty()) {
@@ -152,78 +178,110 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
   }
 
   OICompiler compiler{{}, compilerConfig};
-
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
-
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
 }
 
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
-  drgnplusplus::program prog;
+namespace {
 
-  {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(
-            drgn_program_load_debug_info(prog.get(),
-                                         std::data(objectPaths),
-                                         std::size(objectPaths),
-                                         false,
-                                         false))) {
-      LOG(ERROR) << "error loading debug info program: " << err;
-      throw err;
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx(ctx_) {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
     }
-  }
 
-  auto oilTypes = findOilTypesAndNames(prog);
-
-  std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
-      {Feature::PruneTypeGraph, true},
-  };
-
-  OICodeGen::Config generatorConfig{};
-  OICompiler::Config compilerConfig{};
-  compilerConfig.usePIC = pic;
-
-  auto features = config::processConfigFiles(
-      configFilePaths, featuresMap, compilerConfig, generatorConfig);
-  if (!features) {
-    LOG(ERROR) << "failed to process config file";
-    return -1;
-  }
-  generatorConfig.features = *features;
-  compilerConfig.features = *features;
-
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
+    auto introspectImpl =
+        std::move(oi_namespaces) |
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
+        ranges::views::transform([](auto* p) {
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
+        }) |
+        ranges::views::filter([](auto* td) {
+          return td != nullptr && td->getName() == "introspectImpl";
+        }) |
+        ranges::views::take(1) | ranges::to<std::vector>();
+    if (introspectImpl.empty()) {
+      LOG(WARNING)
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
+             "you compile with `OIL_AOT_COMPILATION=1`?";
+      return;
     }
-  }
 
-  size_t successes = oilTypes.size() - failures;
-  LOG(INFO) << "object introspection generation complete. " << successes
-            << " successes and " << failures << " failures.";
+    auto nameToClangTypeMap =
+        ranges::views::single(introspectImpl[0]) |
+        ranges::views::for_each(
+            [](auto* td) { return td->specializations(); }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
 
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
-    return -1;
+              assert(fd->getNumParams() == 1);
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, opts};
+
+    auto& Sema = *ctx.sema;
+    ctx.nameToTypeMap =
+        nameToClangTypeMap |
+        ranges::views::transform(
+            [&parser, &Context, &Sema](
+                auto& p) -> std::pair<std::string, type_graph::Type*> {
+              return {p.first, &parser.parse(Context, Sema, *p.second)};
+            }) |
+        ranges::to<std::unordered_map>();
+
+    for (const auto& [name, type] : ctx.nameToTypeMap)
+      ctx.typeGraph.addRoot(*type);
   }
-  return 0;
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  // Compile the output as position independent if any input is position
+  // independent
+  bool pic = CI.getCodeGenOpts().RelocationModel == llvm::Reloc::PIC_;
+  ctx.pic = ctx.pic.value_or(false) || pic;
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
 }
 
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+}  // namespace
 }  // namespace oi::detail
diff --git a/oi/OIGenerator.h b/oi/OIGenerator.h
index dd38384..fac947a 100644
--- a/oi/OIGenerator.h
+++ b/oi/OIGenerator.h
@@ -23,11 +23,19 @@
 #include "oi/OICodeGen.h"
 #include "oi/OICompiler.h"
 
+namespace clang::tooling {
+class CompilationDatabase;
+}
+
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate(clang::tooling::CompilationDatabase&,
+               const std::vector<std::string>&);
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -41,8 +49,8 @@ class OIGenerator {
   void setFailIfNothingGenerated(bool fail) {
     failIfNothingGenerated = fail;
   }
-  void setUsePIC(bool pic_) {
-    pic = pic_;
+  void setClangArgs(std::vector<std::string> args_) {
+    clangArgs = std::move(args_);
   }
 
  private:
@@ -50,20 +58,7 @@ class OIGenerator {
   std::vector<std::filesystem::path> configFilePaths;
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
-  bool pic = false;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
+  std::vector<std::string> clangArgs;
 };
 
 }  // namespace oi::detail
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 190bdfc..6c4da13 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnExporter.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
@@ -27,3 +28,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
new file mode 100644
index 0000000..67f7bfa
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -0,0 +1,416 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Basic/DiagnosticSema.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <iostream>  // TODO: maybe remove
+#include <regex>
+#include <stdexcept>  // TODO: remove
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (VLOG_IS_ON(3)) {
+    std::string fqName = clang::TypeName::getFullyQualifiedName(
+        clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+    VLOG(3) << std::string(depth_ * 2, ' ') << fqName;
+  }
+
+  if (!requireCompleteType(*sema, ty))
+    return makeType<Incomplete>(ty, "incomplete (TODO naming)");
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(llvm::cast<const clang::RecordType>(ty));
+    case clang::Type::LValueReference:
+      return enumerateReference(
+          llvm::cast<const clang::LValueReferenceType>(ty));
+    case clang::Type::Pointer:
+      return enumeratePointer(llvm::cast<const clang::PointerType>(ty));
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          llvm::cast<const clang::SubstTemplateTypeParmType>(ty));
+    case clang::Type::Builtin:
+      return enumeratePrimitive(llvm::cast<const clang::BuiltinType>(ty));
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(
+          llvm::cast<const clang::ElaboratedType>(ty));
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          llvm::cast<const clang::TemplateSpecializationType>(ty));
+    case clang::Type::UnaryTransform:
+      return enumerateUnaryTransformType(
+          llvm::cast<const clang::UnaryTransformType>(ty));
+    case clang::Type::Decltype:
+      return enumerateDecltypeType(llvm::cast<const clang::DecltypeType>(ty));
+    case clang::Type::Typedef:
+      return enumerateTypedef(llvm::cast<const clang::TypedefType>(ty));
+    case clang::Type::Using:
+      return enumerateUsing(llvm::cast<const clang::UsingType>(ty));
+    case clang::Type::ConstantArray:
+      return enumerateArray(llvm::cast<const clang::ConstantArrayType>(ty));
+    case clang::Type::Enum:
+      return enumerateEnum(llvm::cast<const clang::EnumType>(ty));
+
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Type& ClangTypeParser::enumerateDecltypeType(const clang::DecltypeType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Type& ClangTypeParser::enumerateUnaryTransformType(
+    const clang::UnaryTransformType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Typedef& ClangTypeParser::enumerateUsing(const clang::UsingType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+  std::string name = ty.getFoundDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+
+  std::string name = ty.getDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string name = ty.getDecl()->getNameAsString();
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  std::map<int64_t, std::string> enumeratorMap;
+  if (options_.readEnumValues) {
+    for (const auto* enumerator : ty.getDecl()->enumerators()) {
+      enumeratorMap.emplace(enumerator->getInitVal().getExtValue(),
+                            enumerator->getNameAsString());
+    }
+  }
+
+  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+}
+
+Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
+  uint64_t len = ty.getSize().getLimitedValue();
+  auto& t = enumerateType(*ty.getElementType());
+  return makeType<Array>(ty, t, len);
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  if (auto* info = getContainerInfo(fqName)) {
+    auto& c = makeType<Container>(ty, *info, size, nullptr);
+    enumerateClassTemplateParams(ty, c.templateParams);
+    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)));
+    return c;
+  }
+
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+
+  params.reserve(list.size());
+  for (const auto& arg : list.asArray()) {
+    if (auto p = enumerateTemplateParam(arg))
+      params.emplace_back(std::move(p.value()));
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return TemplateParam{ttype, qualifiers};
+    }
+    case clang::TemplateArgument::Integral: {
+      auto& ty = enumerateType(*p.getIntegralType());
+      llvm::SmallString<32> val;
+      p.getAsIntegral().toString(val);
+      return TemplateParam{ty, std::string(val)};
+    }
+    case clang::TemplateArgument::Template: {
+      return enumerateTemplateTemplateParam(p.getAsTemplate());
+    }
+
+#define X(name)                       \
+  case clang::TemplateArgument::name: \
+    throw std::logic_error("unsupported template argument kind: " #name);
+
+      X(Null)
+      X(Declaration)
+      X(NullPtr)
+      X(TemplateExpansion)
+      X(Expression)
+      X(Pack)
+#undef X
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateTemplateParam(
+    const clang::TemplateName& tn) {
+  switch (tn.getKind()) {
+    case clang::TemplateName::Template:
+      return std::nullopt;
+
+#define X(name)                   \
+  case clang::TemplateName::name: \
+    throw std::logic_error("unsupported template name kind: " #name);
+
+      X(OverloadedTemplate)
+      X(AssumedTemplate)
+      X(QualifiedTemplate)
+      X(DependentTemplate)
+      X(SubstTemplateTemplateParm)
+      X(SubstTemplateTemplateParmPack)
+      X(UsingTemplate)
+#undef X
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function references
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Float:
+      return makeType<Primitive>(ty, Primitive::Kind::Float32);
+    case clang::BuiltinType::Double:
+    case clang::BuiltinType::LongDouble:
+      return makeType<Primitive>(ty, Primitive::Kind::Float64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+ContainerInfo* ClangTypeParser::getContainerInfo(
+    const std::string& fqName) const {
+  for (const auto& containerInfo : containers_) {
+    if (std::regex_search(fqName, containerInfo->matcher)) {
+      return containerInfo.get();
+    }
+  }
+  return nullptr;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
+    return true;  // treat as complete
+
+  // TODO: This is a terrible warning.
+  return !sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      clang::diag::warn_nsconsumed_attribute_mismatch);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
new file mode 100644
index 0000000..9b27d64
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.h
@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ASTContext;
+class BuiltinType;
+class ConstantArrayType;
+class DecltypeType;
+class ElaboratedType;
+class EnumType;
+class LValueReferenceType;
+class PointerType;
+class RecordType;
+class Sema;
+class SubstTemplateTypeParmType;
+class TemplateArgument;
+class TemplateName;
+class TemplateSpecializationType;
+class Type;
+class TypedefType;
+class UnaryTransformType;
+class UsingType;
+}  // namespace clang
+
+struct ContainerInfo;
+
+namespace oi::detail::type_graph {
+
+class Array;
+class Class;
+class Enum;
+class Member;
+class Primitive;
+class Reference;
+class Type;
+class TypeGraph;
+class Typedef;
+struct TemplateParam;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+  bool readEnumValues = false;
+};
+
+/*
+ * ClangTypeParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given clang::Type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph,
+                  const std::vector<std::unique_ptr<ContainerInfo>>& containers,
+                  ClangTypeParserOptions options)
+      : typeGraph_{typeGraph}, containers_{containers}, options_{options} {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containers_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Type& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+  Typedef& enumerateTypedef(const clang::TypedefType&);
+  Typedef& enumerateUsing(const clang::UsingType&);
+  Type& enumerateUnaryTransformType(const clang::UnaryTransformType&);
+  Type& enumerateDecltypeType(const clang::DecltypeType&);
+
+  Array& enumerateArray(const clang::ConstantArrayType&);
+  Enum& enumerateEnum(const clang::EnumType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  std::optional<TemplateParam> enumerateTemplateParam(
+      const clang::TemplateArgument&);
+  std::optional<TemplateParam> enumerateTemplateTemplateParam(
+      const clang::TemplateName&);
+
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  ContainerInfo* getContainerInfo(const std::string& fqName) const;
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/DrgnExporter.cpp b/oi/type_graph/DrgnExporter.cpp
index 802857e..df9722d 100644
--- a/oi/type_graph/DrgnExporter.cpp
+++ b/oi/type_graph/DrgnExporter.cpp
@@ -200,6 +200,14 @@ drgn_type* DrgnExporter::visit(Pointer& p) {
   return drgnType;
 }
 
+drgn_type* DrgnExporter::visit(Reference& p) {
+  auto* drgnType =
+      makeDrgnType(DRGN_TYPE_POINTER, false, DRGN_NOT_PRIMITIVE_TYPE, p);
+  auto* pointeeType = accept(p.pointeeType());
+  th_.pointerToTypeMap[drgnType] = pointeeType;
+  return drgnType;
+}
+
 drgn_type* DrgnExporter::visit(Dummy& d) {
   return makeDrgnType(DRGN_TYPE_VOID, false, DRGN_C_TYPE_VOID, d);
 }
diff --git a/oi/type_graph/DrgnExporter.h b/oi/type_graph/DrgnExporter.h
index 45f02c6..24f4c59 100644
--- a/oi/type_graph/DrgnExporter.h
+++ b/oi/type_graph/DrgnExporter.h
@@ -54,6 +54,7 @@ class DrgnExporter : public Visitor<drgn_type*> {
   drgn_type* visit(Array&) override;
   drgn_type* visit(Typedef&) override;
   drgn_type* visit(Pointer&) override;
+  drgn_type* visit(Reference&) override;
   drgn_type* visit(Dummy&) override;
   drgn_type* visit(DummyAllocator&) override;
   drgn_type* visit(CaptureKeys&) override;
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index b1d1e8d..725ab59 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -200,6 +200,15 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& r) {
+  RecursiveVisitor::visit(r);
+
+  r.regenerateName();
+  std::string inputName{r.pointeeType().inputName()};
+  inputName += '&';
+  r.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..4f1c258 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& r) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index 0dae0ab..f598161 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -150,6 +150,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& r) {
+  if (prefix(r))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = r.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(r.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index 8d7138a..e68f562 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& r) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;
diff --git a/oi/type_graph/RemoveTopLevelPointer.cpp b/oi/type_graph/RemoveTopLevelPointer.cpp
index 9ecaaaa..c2fc91d 100644
--- a/oi/type_graph/RemoveTopLevelPointer.cpp
+++ b/oi/type_graph/RemoveTopLevelPointer.cpp
@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& r) {
+  topLevelType_ = &r.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/RemoveTopLevelPointer.h b/oi/type_graph/RemoveTopLevelPointer.h
index 611429f..3ed2c15 100644
--- a/oi/type_graph/RemoveTopLevelPointer.h
+++ b/oi/type_graph/RemoveTopLevelPointer.h
@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& r) override;
 
  private:
   Type* topLevelType_ = nullptr;
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f0b3fe9..b99d654 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -734,6 +735,66 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    // Following a pointer wouldn't trigger cycle checking, as it would look
+    // like anything else we're sure is there. Generate as a pointer. It will be
+    // followed regardless of `ChaseRawPointers` because that affects whether a
+    // type becomes a `StubbedPointer` and not whether pointers are followed in
+    // the generated code.
+    name_ = pointeeType_.get().name() + "*";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index a7f7eec..f055aea 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -108,6 +108,9 @@ class RecursiveVisitor : public Visitor<void> {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -175,6 +178,10 @@ class RecursiveMutator : public Visitor<Type&> {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }
diff --git a/test/integration/template_template_param.toml b/test/integration/template_template_param.toml
new file mode 100644
index 0000000..c64590d
--- /dev/null
+++ b/test/integration/template_template_param.toml
@@ -0,0 +1,16 @@
+includes = ["vector", "utility", "string"]
+
+definitions = '''
+template <typename K, typename V, template <typename, typename> typename Pair>
+struct bad_map {
+  std::vector<Pair<K, K>> keys;
+  std::vector<Pair<V, V>> values;
+};
+'''
+
+[cases]
+  [cases.int_int_empty]
+    param_types = ["const bad_map<int, std::string, std::pair>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index d1e345d..87739f9 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -14,131 +14,83 @@
  * limitations under the License.
  */
 
+#include <clang/Tooling/CommonOptionsParser.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include <cstdlib>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/action/drop_while.hpp>
+#include <range/v3/algorithm/find.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/all.hpp>
+#include <range/v3/view/transform.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
 
-constexpr static OIOpts opts{
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
+static llvm::cl::OptionCategory OilgenCategory("oilgen options");
 
-void usage() {
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
-  std::cerr << opts;
+static llvm::cl::list<std::string> ConfigFiles(
+    "config-file",
+    llvm::cl::desc(R"(</path/to/oid.toml>)"),
+    llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> OutputFile(
+    "output",
+    llvm::cl::desc(R"(Write output to this file.)"),
+    llvm::cl::init("a.o"),
+    llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<int> DebugLevel(
+    "debug-level",
+    llvm::cl::desc(R"(Verbose level for logging.)"),
+    llvm::cl::init(-1),
+    llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> DumpJit(
+    "dump-jit",
+    llvm::cl::desc(R"(Write the generated code to a file.)"),
+    llvm::cl::init("jit.cpp"),
+    llvm::cl::cat(OilgenCategory));
 
-  std::cerr << std::endl
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
-
-int main(int argc, char* argv[]) {
+int main(int argc, const char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
-  std::vector<fs::path> configFilePaths;
-  fs::path sourceFileDumpPath = "";
-  bool exitCode = false;
-  bool pic = false;
-
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
+  auto expectedParser =
+      clang::tooling::CommonOptionsParser::create(argc, argv, OilgenCategory);
+  if (!expectedParser) {
+    llvm::errs() << expectedParser.takeError();
+    return -1;
   }
+  clang::tooling::CommonOptionsParser& options = expectedParser.get();
+  auto& compilations = options.getCompilations();
 
-  if (optind >= argc) {
-    usage();
-    return EXIT_FAILURE;
-  }
-  fs::path primaryObject = argv[optind];
-
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
+  if (DebugLevel.getNumOccurrences()) {
+    google::LogToStderr();
+    google::SetStderrLogging(google::INFO);
+    google::SetVLOGLevel("*", DebugLevel);
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
+    gflags::SetCommandLineOption("minloglevel", "0");
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
-  oigen.setConfigFilePaths(std::move(configFilePaths));
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
-  oigen.setFailIfNothingGenerated(exitCode);
-  oigen.setUsePIC(pic);
+  oigen.setConfigFilePaths(ConfigFiles |
+                           ranges::views::transform([](const auto& p) {
+                             return std::filesystem::path(p);
+                           }) |
+                           ranges::to<std::vector>());
+  if (DumpJit.getNumOccurrences())
+    oigen.setSourceFileDumpPath(DumpJit.getValue());
 
-  SymbolService symbols(primaryObject);
+  oigen.setOutputPath(OutputFile.getValue());
 
-  return oigen.generate(primaryObject, symbols);
+  oigen.setFailIfNothingGenerated(true);
+  return oigen.generate(compilations, options.getSourcePathList());
 }
diff --git a/types/string_type.toml b/types/string_type.toml
index fa4a2ae..0d2ab0d 100644
--- a/types/string_type.toml
+++ b/types/string_type.toml
@@ -35,3 +35,34 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
     );
 }
 """
+
+traversal_func = """
+    bool sso = ((uintptr_t)container.data() <
+                (uintptr_t)(&container + sizeof(std::__cxx11::basic_string<T0>))) &&
+               ((uintptr_t)container.data() >= (uintptr_t)&container);
+
+    return returnArg.write(container.capacity())
+        .write(sso)
+        .write(container.size());
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+uint64_t capacity = std::get<ParsedData::VarInt>(d.val).value;
+el.container_stats.emplace(result::Element::ContainerStats { .capacity = capacity });
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+bool sso = std::get<ParsedData::VarInt>(d.val).value;
+if (!sso)
+  el.exclusive_size += el.container_stats->capacity * sizeof(T0);
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats->length = std::get<ParsedData::VarInt>(d.val).value;
+"""

From 107e95fd9e4c1b565f3ba823688372d282e22bad Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 19 Dec 2023 16:50:08 +0000
Subject: [PATCH 072/188] oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD
---
 CMakeLists.txt                                |   3 +-
 oi/CodeGen.cpp                                |  31 +-
 oi/CodeGen.h                                  |   6 +-
 oi/OIGenerator.cpp                            | 368 ++++++++-------
 oi/OIGenerator.h                              |  29 +-
 oi/type_graph/CMakeLists.txt                  |   2 +
 oi/type_graph/ClangTypeParser.cpp             | 418 ++++++++++++++++++
 oi/type_graph/ClangTypeParser.h               | 131 ++++++
 oi/type_graph/DrgnExporter.cpp                |   8 +
 oi/type_graph/DrgnExporter.h                  |   1 +
 oi/type_graph/NameGen.cpp                     |   9 +
 oi/type_graph/NameGen.h                       |   1 +
 oi/type_graph/Printer.cpp                     |  11 +
 oi/type_graph/Printer.h                       |   1 +
 oi/type_graph/RemoveTopLevelPointer.cpp       |   4 +
 oi/type_graph/RemoveTopLevelPointer.h         |   1 +
 oi/type_graph/Types.h                         |  61 +++
 oi/type_graph/Visitor.h                       |   7 +
 test/integration/template_template_param.toml |  16 +
 tools/OILGen.cpp                              | 147 ++----
 types/string_type.toml                        |  31 ++
 21 files changed, 998 insertions(+), 288 deletions(-)
 create mode 100644 oi/type_graph/ClangTypeParser.cpp
 create mode 100644 oi/type_graph/ClangTypeParser.h
 create mode 100644 test/integration/template_template_param.toml

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f9ee7..b6fecd1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -289,7 +289,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -365,6 +365,7 @@ add_executable(oilgen
 target_link_libraries(oilgen
   drgn_utils
   oicore
+  clangTooling
 )
 
 ### Object Introspection cache Printer (OIP)
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 74c6f39..9f41c12 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -299,6 +299,11 @@ void genDefsThriftClass(const Class& c, std::string& code) {
 
 }  // namespace
 
+CodeGen::CodeGen(const OICodeGen::Config& config) : config_(config) {
+  DCHECK(!config.features[Feature::PolymorphicInheritance])
+      << "polymorphic inheritance requires symbol service!";
+}
+
 void CodeGen::genDefsThrift(const TypeGraph& typeGraph, std::string& code) {
   for (const Type& t : typeGraph.finalTypes) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
@@ -542,7 +547,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1115,15 +1120,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
                               std::string& code,
                               RootFunctionName name) {
-  try {
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
+  if (!registerContainers())
     return false;
-  }
 
   try {
     addDrgnRoot(drgnType, typeGraph_);
@@ -1148,6 +1146,19 @@ void CodeGen::exportDrgnTypes(TypeHierarchy& th,
   }
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1189,7 +1200,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(IdentifyContainers::createPass(containerInfos_));
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 9c76238..8fc5a81 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -41,8 +41,9 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config);
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   struct ExactName {
@@ -65,6 +66,7 @@ class CodeGen {
                        std::list<drgn_type>& drgnTypes,
                        drgn_type** rootType) const;
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -77,7 +79,7 @@ class CodeGen {
  private:
   type_graph::TypeGraph typeGraph_;
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_ = nullptr;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 1cb2a49..76cf939 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -16,130 +16,149 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
 #include <glog/logging.h>
 
-#include <boost/core/demangle.hpp>
 #include <fstream>
-#include <iostream>
-#include <string_view>
+#include <range/v3/core.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
+class ConsumerContext;
 
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
-      templateArgsToSymbolsMap;
-
-  auto symbols = prog.find_all_symbols();
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
+class CreateTypeGraphConsumer;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
-    if (val.first.empty() || val.second.empty()) {
-      continue;
-    }
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  return strongToWeakSymbols;
-}
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
 
-std::unordered_map<std::string, drgn_qualified_type>
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
+ private:
+  ConsumerContext& ctx;
+};
 
-  std::unordered_map<std::string, drgn_qualified_type> out;
+class ConsumerContext {
+ public:
+  ConsumerContext(const std::vector<std::unique_ptr<ContainerInfo>>& cis)
+      : containerInfos{cis} {
+  }
 
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
-    std::string strongLinkageName;
-    {
-      const char* linkageNameCstr;
-      if (auto err = drgnplusplus::error(
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
-        // throw err;
+  type_graph::TypeGraph typeGraph;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
+  std::optional<bool> pic;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+}  // namespace
+
+int OIGenerator::generate(clang::tooling::CompilationDatabase& db,
+                          const std::vector<std::string>& sourcePaths) {
+  std::map<Feature, bool> featuresMap = {
+      {Feature::TypeGraph, true},
+      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},
+      {Feature::PackStructs, true},
+      {Feature::PruneTypeGraph, true},
+  };
+
+  OICodeGen::Config generatorConfig{};
+  OICompiler::Config compilerConfig{};
+
+  auto features = config::processConfigFiles(
+      configFilePaths, featuresMap, compilerConfig, generatorConfig);
+  if (!features) {
+    LOG(ERROR) << "failed to process config file";
+    return -1;
+  }
+  generatorConfig.features = *features;
+  compilerConfig.features = *features;
+
+  std::vector<std::unique_ptr<ContainerInfo>> containerInfos;
+  containerInfos.reserve(generatorConfig.containerConfigPaths.size());
+  try {
+    for (const auto& path : generatorConfig.containerConfigPaths) {
+      auto info = std::make_unique<ContainerInfo>(path);
+      if (info->requiredFeatures != (*features & info->requiredFeatures)) {
+        VLOG(1) << "Skipping container (feature conflict): " << info->typeName;
         continue;
       }
-      strongLinkageName = linkageNameCstr;
+      containerInfos.emplace_back(std::move(info));
     }
-
-    std::string weakLinkageName;
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return -1;
   }
 
-  return out;
-}
+  ConsumerContext ctx{containerInfos};
+  CreateTypeGraphActionFactory factory{ctx};
 
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
-                                      const OICompiler::Config& compilerConfig,
-                                      const drgn_qualified_type& type,
-                                      const std::string& linkageName,
-                                      SymbolService& symbols) {
-  CodeGen codegen{generatorConfig, symbols};
+  clang::tooling::ClangTool tool{db, sourcePaths};
+  if (auto ret = tool.run(&factory); ret != 0) {
+    return ret;
+  }
+
+  if (ctx.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one site to generate for but we can't currently "
+        "handle this case");
+
+  if (ctx.nameToTypeMap.empty()) {
+    LOG(ERROR) << "Nothing to generate!";
+    return failIfNothingGenerated ? -1 : 0;
+  }
+  const auto& linkageName = ctx.nameToTypeMap.begin()->first;
+
+  compilerConfig.usePIC = ctx.pic.value();
+  CodeGen codegen{generatorConfig};
+  for (auto&& ptr : containerInfos)
+    codegen.registerContainer(std::move(ptr));
+  codegen.transform(ctx.typeGraph);
 
   std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
+  codegen.generate(ctx.typeGraph, code, CodeGen::ExactName{linkageName});
 
   std::string sourcePath = sourceFileDumpPath;
   if (sourceFileDumpPath.empty()) {
@@ -152,78 +171,109 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
   }
 
   OICompiler compiler{{}, compilerConfig};
-
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
-
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
 }
 
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
-  drgnplusplus::program prog;
+namespace {
 
-  {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(
-            drgn_program_load_debug_info(prog.get(),
-                                         std::data(objectPaths),
-                                         std::size(objectPaths),
-                                         false,
-                                         false))) {
-      LOG(ERROR) << "error loading debug info program: " << err;
-      throw err;
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx(ctx_) {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
     }
-  }
 
-  auto oilTypes = findOilTypesAndNames(prog);
-
-  std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
-      {Feature::PruneTypeGraph, true},
-  };
-
-  OICodeGen::Config generatorConfig{};
-  OICompiler::Config compilerConfig{};
-  compilerConfig.usePIC = pic;
-
-  auto features = config::processConfigFiles(
-      configFilePaths, featuresMap, compilerConfig, generatorConfig);
-  if (!features) {
-    LOG(ERROR) << "failed to process config file";
-    return -1;
-  }
-  generatorConfig.features = *features;
-  compilerConfig.features = *features;
-
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
+    auto introspectImpl =
+        std::move(oi_namespaces) |
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
+        ranges::views::transform([](auto* p) {
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
+        }) |
+        ranges::views::filter([](auto* td) {
+          return td != nullptr && td->getName() == "introspectImpl";
+        }) |
+        ranges::views::take(1) | ranges::to<std::vector>();
+    if (introspectImpl.empty()) {
+      LOG(WARNING)
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
+             "you compile with `OIL_AOT_COMPILATION=1`?";
+      return;
     }
-  }
 
-  size_t successes = oilTypes.size() - failures;
-  LOG(INFO) << "object introspection generation complete. " << successes
-            << " successes and " << failures << " failures.";
+    auto nameToClangTypeMap =
+        introspectImpl | ranges::views::for_each([](auto* td) {
+          return td->specializations();
+        }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
 
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
-    return -1;
+              assert(fd->getNumParams() == 1);
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, opts};
+
+    auto& Sema = *ctx.sema;
+    auto els = nameToClangTypeMap |
+               ranges::views::transform(
+                   [&parser, &Context, &Sema](
+                       auto& p) -> std::pair<std::string, type_graph::Type*> {
+                     return {p.first, &parser.parse(Context, Sema, *p.second)};
+                   });
+    ctx.nameToTypeMap.insert(els.begin(), els.end());
+
+    for (const auto& [name, type] : ctx.nameToTypeMap)
+      ctx.typeGraph.addRoot(*type);
   }
-  return 0;
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  // Compile the output as position independent if any input is position
+  // independent
+  bool pic = CI.getCodeGenOpts().RelocationModel == llvm::Reloc::PIC_;
+  ctx.pic = ctx.pic.value_or(false) || pic;
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
 }
 
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+}  // namespace
 }  // namespace oi::detail
diff --git a/oi/OIGenerator.h b/oi/OIGenerator.h
index dd38384..fac947a 100644
--- a/oi/OIGenerator.h
+++ b/oi/OIGenerator.h
@@ -23,11 +23,19 @@
 #include "oi/OICodeGen.h"
 #include "oi/OICompiler.h"
 
+namespace clang::tooling {
+class CompilationDatabase;
+}
+
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate(clang::tooling::CompilationDatabase&,
+               const std::vector<std::string>&);
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -41,8 +49,8 @@ class OIGenerator {
   void setFailIfNothingGenerated(bool fail) {
     failIfNothingGenerated = fail;
   }
-  void setUsePIC(bool pic_) {
-    pic = pic_;
+  void setClangArgs(std::vector<std::string> args_) {
+    clangArgs = std::move(args_);
   }
 
  private:
@@ -50,20 +58,7 @@ class OIGenerator {
   std::vector<std::filesystem::path> configFilePaths;
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
-  bool pic = false;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
+  std::vector<std::string> clangArgs;
 };
 
 }  // namespace oi::detail
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 190bdfc..6c4da13 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnExporter.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
@@ -27,3 +28,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
new file mode 100644
index 0000000..5190478
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -0,0 +1,418 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Basic/DiagnosticSema.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <regex>
+#include <stdexcept>
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (VLOG_IS_ON(3)) {
+    std::string fqName = clang::TypeName::getFullyQualifiedName(
+        clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+    VLOG(3) << std::string(depth_ * 2, ' ') << fqName;
+  }
+
+  // TODO: This check basically doesn't work. The action has failed before it
+  // returns false. Fix this.
+  if (!requireCompleteType(*sema, ty)) {
+    std::string fqName = clang::TypeName::getFullyQualifiedName(
+        clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+    return makeType<Incomplete>(ty, std::move(fqName));
+  }
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(llvm::cast<const clang::RecordType>(ty));
+    case clang::Type::LValueReference:
+      return enumerateReference(
+          llvm::cast<const clang::LValueReferenceType>(ty));
+    case clang::Type::Pointer:
+      return enumeratePointer(llvm::cast<const clang::PointerType>(ty));
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          llvm::cast<const clang::SubstTemplateTypeParmType>(ty));
+    case clang::Type::Builtin:
+      return enumeratePrimitive(llvm::cast<const clang::BuiltinType>(ty));
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(
+          llvm::cast<const clang::ElaboratedType>(ty));
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          llvm::cast<const clang::TemplateSpecializationType>(ty));
+    case clang::Type::UnaryTransform:
+      return enumerateUnaryTransformType(
+          llvm::cast<const clang::UnaryTransformType>(ty));
+    case clang::Type::Decltype:
+      return enumerateDecltypeType(llvm::cast<const clang::DecltypeType>(ty));
+    case clang::Type::Typedef:
+      return enumerateTypedef(llvm::cast<const clang::TypedefType>(ty));
+    case clang::Type::Using:
+      return enumerateUsing(llvm::cast<const clang::UsingType>(ty));
+    case clang::Type::ConstantArray:
+      return enumerateArray(llvm::cast<const clang::ConstantArrayType>(ty));
+    case clang::Type::Enum:
+      return enumerateEnum(llvm::cast<const clang::EnumType>(ty));
+
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Type& ClangTypeParser::enumerateDecltypeType(const clang::DecltypeType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Type& ClangTypeParser::enumerateUnaryTransformType(
+    const clang::UnaryTransformType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Typedef& ClangTypeParser::enumerateUsing(const clang::UsingType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+  std::string name = ty.getFoundDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+
+  std::string name = ty.getDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string name = ty.getDecl()->getNameAsString();
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  std::map<int64_t, std::string> enumeratorMap;
+  if (options_.readEnumValues) {
+    for (const auto* enumerator : ty.getDecl()->enumerators()) {
+      enumeratorMap.emplace(enumerator->getInitVal().getExtValue(),
+                            enumerator->getNameAsString());
+    }
+  }
+
+  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+}
+
+Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
+  uint64_t len = ty.getSize().getLimitedValue();
+  auto& t = enumerateType(*ty.getElementType());
+  return makeType<Array>(ty, t, len);
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  if (auto* info = getContainerInfo(fqName)) {
+    auto& c = makeType<Container>(ty, *info, size, nullptr);
+    enumerateClassTemplateParams(ty, c.templateParams);
+    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)));
+    return c;
+  }
+
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+
+  params.reserve(list.size());
+  for (const auto& arg : list.asArray()) {
+    if (auto p = enumerateTemplateParam(arg))
+      params.emplace_back(std::move(p.value()));
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return TemplateParam{ttype, qualifiers};
+    }
+    case clang::TemplateArgument::Integral: {
+      auto& ty = enumerateType(*p.getIntegralType());
+      llvm::SmallString<32> val;
+      p.getAsIntegral().toString(val);
+      return TemplateParam{ty, std::string(val)};
+    }
+    case clang::TemplateArgument::Template: {
+      return enumerateTemplateTemplateParam(p.getAsTemplate());
+    }
+
+#define X(name)                       \
+  case clang::TemplateArgument::name: \
+    throw std::logic_error("unsupported template argument kind: " #name);
+
+      X(Null)
+      X(Declaration)
+      X(NullPtr)
+      X(TemplateExpansion)
+      X(Expression)
+      X(Pack)
+#undef X
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateTemplateParam(
+    const clang::TemplateName& tn) {
+  switch (tn.getKind()) {
+    case clang::TemplateName::Template:
+      return std::nullopt;
+
+#define X(name)                   \
+  case clang::TemplateName::name: \
+    throw std::logic_error("unsupported template name kind: " #name);
+
+      X(OverloadedTemplate)
+      X(AssumedTemplate)
+      X(QualifiedTemplate)
+      X(DependentTemplate)
+      X(SubstTemplateTemplateParm)
+      X(SubstTemplateTemplateParmPack)
+      X(UsingTemplate)
+#undef X
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function pointers
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Float:
+      return makeType<Primitive>(ty, Primitive::Kind::Float32);
+    case clang::BuiltinType::Double:
+    case clang::BuiltinType::LongDouble:
+      return makeType<Primitive>(ty, Primitive::Kind::Float64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+ContainerInfo* ClangTypeParser::getContainerInfo(
+    const std::string& fqName) const {
+  for (const auto& containerInfo : containers_) {
+    if (std::regex_search(fqName, containerInfo->matcher)) {
+      return containerInfo.get();
+    }
+  }
+  return nullptr;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
+    return true;  // treat as complete
+
+  // TODO: This is a terrible warning.
+  return !sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      clang::diag::warn_nsconsumed_attribute_mismatch);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
new file mode 100644
index 0000000..9b27d64
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.h
@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ASTContext;
+class BuiltinType;
+class ConstantArrayType;
+class DecltypeType;
+class ElaboratedType;
+class EnumType;
+class LValueReferenceType;
+class PointerType;
+class RecordType;
+class Sema;
+class SubstTemplateTypeParmType;
+class TemplateArgument;
+class TemplateName;
+class TemplateSpecializationType;
+class Type;
+class TypedefType;
+class UnaryTransformType;
+class UsingType;
+}  // namespace clang
+
+struct ContainerInfo;
+
+namespace oi::detail::type_graph {
+
+class Array;
+class Class;
+class Enum;
+class Member;
+class Primitive;
+class Reference;
+class Type;
+class TypeGraph;
+class Typedef;
+struct TemplateParam;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+  bool readEnumValues = false;
+};
+
+/*
+ * ClangTypeParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given clang::Type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph,
+                  const std::vector<std::unique_ptr<ContainerInfo>>& containers,
+                  ClangTypeParserOptions options)
+      : typeGraph_{typeGraph}, containers_{containers}, options_{options} {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containers_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Type& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+  Typedef& enumerateTypedef(const clang::TypedefType&);
+  Typedef& enumerateUsing(const clang::UsingType&);
+  Type& enumerateUnaryTransformType(const clang::UnaryTransformType&);
+  Type& enumerateDecltypeType(const clang::DecltypeType&);
+
+  Array& enumerateArray(const clang::ConstantArrayType&);
+  Enum& enumerateEnum(const clang::EnumType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  std::optional<TemplateParam> enumerateTemplateParam(
+      const clang::TemplateArgument&);
+  std::optional<TemplateParam> enumerateTemplateTemplateParam(
+      const clang::TemplateName&);
+
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  ContainerInfo* getContainerInfo(const std::string& fqName) const;
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/DrgnExporter.cpp b/oi/type_graph/DrgnExporter.cpp
index 802857e..df9722d 100644
--- a/oi/type_graph/DrgnExporter.cpp
+++ b/oi/type_graph/DrgnExporter.cpp
@@ -200,6 +200,14 @@ drgn_type* DrgnExporter::visit(Pointer& p) {
   return drgnType;
 }
 
+drgn_type* DrgnExporter::visit(Reference& p) {
+  auto* drgnType =
+      makeDrgnType(DRGN_TYPE_POINTER, false, DRGN_NOT_PRIMITIVE_TYPE, p);
+  auto* pointeeType = accept(p.pointeeType());
+  th_.pointerToTypeMap[drgnType] = pointeeType;
+  return drgnType;
+}
+
 drgn_type* DrgnExporter::visit(Dummy& d) {
   return makeDrgnType(DRGN_TYPE_VOID, false, DRGN_C_TYPE_VOID, d);
 }
diff --git a/oi/type_graph/DrgnExporter.h b/oi/type_graph/DrgnExporter.h
index 45f02c6..24f4c59 100644
--- a/oi/type_graph/DrgnExporter.h
+++ b/oi/type_graph/DrgnExporter.h
@@ -54,6 +54,7 @@ class DrgnExporter : public Visitor<drgn_type*> {
   drgn_type* visit(Array&) override;
   drgn_type* visit(Typedef&) override;
   drgn_type* visit(Pointer&) override;
+  drgn_type* visit(Reference&) override;
   drgn_type* visit(Dummy&) override;
   drgn_type* visit(DummyAllocator&) override;
   drgn_type* visit(CaptureKeys&) override;
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index b1d1e8d..725ab59 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -200,6 +200,15 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& r) {
+  RecursiveVisitor::visit(r);
+
+  r.regenerateName();
+  std::string inputName{r.pointeeType().inputName()};
+  inputName += '&';
+  r.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..4f1c258 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& r) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index 0dae0ab..f598161 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -150,6 +150,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& r) {
+  if (prefix(r))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = r.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(r.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index 8d7138a..e68f562 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& r) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;
diff --git a/oi/type_graph/RemoveTopLevelPointer.cpp b/oi/type_graph/RemoveTopLevelPointer.cpp
index 9ecaaaa..c2fc91d 100644
--- a/oi/type_graph/RemoveTopLevelPointer.cpp
+++ b/oi/type_graph/RemoveTopLevelPointer.cpp
@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& r) {
+  topLevelType_ = &r.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/RemoveTopLevelPointer.h b/oi/type_graph/RemoveTopLevelPointer.h
index 611429f..3ed2c15 100644
--- a/oi/type_graph/RemoveTopLevelPointer.h
+++ b/oi/type_graph/RemoveTopLevelPointer.h
@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& r) override;
 
  private:
   Type* topLevelType_ = nullptr;
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f0b3fe9..1d5dd79 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -734,6 +735,66 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    // Following a reference wouldn't trigger cycle checking, as it would look
+    // like anything else we're sure is there. Generate as a pointer. It will be
+    // followed regardless of `ChaseRawPointers` because that affects whether a
+    // type becomes a `StubbedPointer` and not whether pointers are followed in
+    // the generated code.
+    name_ = pointeeType_.get().name() + "*";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index a7f7eec..f055aea 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -108,6 +108,9 @@ class RecursiveVisitor : public Visitor<void> {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -175,6 +178,10 @@ class RecursiveMutator : public Visitor<Type&> {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }
diff --git a/test/integration/template_template_param.toml b/test/integration/template_template_param.toml
new file mode 100644
index 0000000..c64590d
--- /dev/null
+++ b/test/integration/template_template_param.toml
@@ -0,0 +1,16 @@
+includes = ["vector", "utility", "string"]
+
+definitions = '''
+template <typename K, typename V, template <typename, typename> typename Pair>
+struct bad_map {
+  std::vector<Pair<K, K>> keys;
+  std::vector<Pair<V, V>> values;
+};
+'''
+
+[cases]
+  [cases.int_int_empty]
+    param_types = ["const bad_map<int, std::string, std::pair>&"]
+    setup = "return {};"
+    expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index d1e345d..77c3569 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -14,131 +14,80 @@
  * limitations under the License.
  */
 
+#include <clang/Tooling/CommonOptionsParser.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include <cstdlib>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/transform.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
 
-constexpr static OIOpts opts{
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
+static llvm::cl::OptionCategory OilgenCategory("oilgen options");
 
-void usage() {
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
-  std::cerr << opts;
+static llvm::cl::list<std::string> ConfigFiles(
+    "config-file",
+    llvm::cl::desc(R"(</path/to/oid.toml>)"),
+    llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> OutputFile(
+    "output",
+    llvm::cl::desc(R"(Write output to this file.)"),
+    llvm::cl::init("a.o"),
+    llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<int> DebugLevel(
+    "debug-level",
+    llvm::cl::desc(R"(Verbose level for logging.)"),
+    llvm::cl::init(-1),
+    llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> DumpJit(
+    "dump-jit",
+    llvm::cl::desc(R"(Write the generated code to a file.)"),
+    llvm::cl::init("jit.cpp"),
+    llvm::cl::cat(OilgenCategory));
 
-  std::cerr << std::endl
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
-
-int main(int argc, char* argv[]) {
+int main(int argc, const char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
-  std::vector<fs::path> configFilePaths;
-  fs::path sourceFileDumpPath = "";
-  bool exitCode = false;
-  bool pic = false;
-
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
+  auto expectedParser =
+      clang::tooling::CommonOptionsParser::create(argc, argv, OilgenCategory);
+  if (!expectedParser) {
+    llvm::errs() << expectedParser.takeError();
+    return -1;
   }
+  clang::tooling::CommonOptionsParser& options = expectedParser.get();
+  auto& compilations = options.getCompilations();
 
-  if (optind >= argc) {
-    usage();
-    return EXIT_FAILURE;
-  }
-  fs::path primaryObject = argv[optind];
-
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
+  if (DebugLevel.getNumOccurrences()) {
+    google::LogToStderr();
+    google::SetStderrLogging(google::INFO);
+    google::SetVLOGLevel("*", DebugLevel);
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
+    gflags::SetCommandLineOption("minloglevel", "0");
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
-  oigen.setConfigFilePaths(std::move(configFilePaths));
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
-  oigen.setFailIfNothingGenerated(exitCode);
-  oigen.setUsePIC(pic);
+  oigen.setConfigFilePaths(ConfigFiles |
+                           ranges::views::transform([](const auto& p) {
+                             return std::filesystem::path(p);
+                           }) |
+                           ranges::to<std::vector>());
+  if (DumpJit.getNumOccurrences())
+    oigen.setSourceFileDumpPath(DumpJit.getValue());
 
-  SymbolService symbols(primaryObject);
+  oigen.setOutputPath(OutputFile.getValue());
 
-  return oigen.generate(primaryObject, symbols);
+  oigen.setFailIfNothingGenerated(true);
+  return oigen.generate(compilations, options.getSourcePathList());
 }
diff --git a/types/string_type.toml b/types/string_type.toml
index fa4a2ae..0d2ab0d 100644
--- a/types/string_type.toml
+++ b/types/string_type.toml
@@ -35,3 +35,34 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
     );
 }
 """
+
+traversal_func = """
+    bool sso = ((uintptr_t)container.data() <
+                (uintptr_t)(&container + sizeof(std::__cxx11::basic_string<T0>))) &&
+               ((uintptr_t)container.data() >= (uintptr_t)&container);
+
+    return returnArg.write(container.capacity())
+        .write(sso)
+        .write(container.size());
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+uint64_t capacity = std::get<ParsedData::VarInt>(d.val).value;
+el.container_stats.emplace(result::Element::ContainerStats { .capacity = capacity });
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+bool sso = std::get<ParsedData::VarInt>(d.val).value;
+if (!sso)
+  el.exclusive_size += el.container_stats->capacity * sizeof(T0);
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats->length = std::get<ParsedData::VarInt>(d.val).value;
+"""

From 802042e3f38dfdc58fd2d07f3a7bc94209dbfb2e Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 19 Dec 2023 16:50:08 +0000
Subject: [PATCH 073/188] oilgen: migrate to source parsing

WIP: This change needs to be integrated with the internal build system before
landing.

Using debug information generated from partial source (that is, not the final
binary) has been insufficient to generally generate OIL code.

A particular example is pointers to templates:
```cpp
#include <oi/oi.h>
template <typename T>
struct Foo {
  T t;
};
template <typename T>
struct Bar {
  Foo<T>& f;
};
void foo(const Bar<int>& b) {
  oi::introspect(b);
}
```

The pointer/reference to `Foo<int>` appears in DWARF with
`DW_AT_declaration(true)` because it could be specialised before its usage.
However, with OIL, we are creating an implicit usage site in the
`oi::introspect` call that the compiler is unable to see.

This change reworks OILGen to work from a Clang command line rather than debug
information. We setup and run a compiler on the source, giving us access to an
AST and Semantic Analyser. We then:
- Find the `oi::introspect` template.
- Iterate through each of its callsites for their type.
- Run `ClangTypeParser::parse` on each type.
- Run codegen.
- Compile into an object file.

Having access to the semantic analyser allows us to forcefully complete a type,
as it would be if it was used in the initial code.

Test Plan:
- TBD
---
 CMakeLists.txt                          |   3 +-
 oi/CodeGen.cpp                          |  31 +-
 oi/CodeGen.h                            |   6 +-
 oi/OIGenerator.cpp                      | 368 ++++++++++++---------
 oi/OIGenerator.h                        |  29 +-
 oi/type_graph/CMakeLists.txt            |   2 +
 oi/type_graph/ClangTypeParser.cpp       | 418 ++++++++++++++++++++++++
 oi/type_graph/ClangTypeParser.h         | 131 ++++++++
 oi/type_graph/DrgnExporter.cpp          |   8 +
 oi/type_graph/DrgnExporter.h            |   1 +
 oi/type_graph/NameGen.cpp               |   9 +
 oi/type_graph/NameGen.h                 |   1 +
 oi/type_graph/Printer.cpp               |  11 +
 oi/type_graph/Printer.h                 |   1 +
 oi/type_graph/RemoveTopLevelPointer.cpp |   4 +
 oi/type_graph/RemoveTopLevelPointer.h   |   1 +
 oi/type_graph/Types.h                   |  61 ++++
 oi/type_graph/Visitor.h                 |   7 +
 tools/OILGen.cpp                        | 147 +++------
 types/string_type.toml                  |  31 ++
 20 files changed, 982 insertions(+), 288 deletions(-)
 create mode 100644 oi/type_graph/ClangTypeParser.cpp
 create mode 100644 oi/type_graph/ClangTypeParser.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7f9ee7..b6fecd1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -289,7 +289,7 @@ add_library(oicore
   oi/Serialize.cpp
 )
 add_dependencies(oicore libdrgn)
-target_include_directories(oicore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
+target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
@@ -365,6 +365,7 @@ add_executable(oilgen
 target_link_libraries(oilgen
   drgn_utils
   oicore
+  clangTooling
 )
 
 ### Object Introspection cache Printer (OIP)
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 74c6f39..9f41c12 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -299,6 +299,11 @@ void genDefsThriftClass(const Class& c, std::string& code) {
 
 }  // namespace
 
+CodeGen::CodeGen(const OICodeGen::Config& config) : config_(config) {
+  DCHECK(!config.features[Feature::PolymorphicInheritance])
+      << "polymorphic inheritance requires symbol service!";
+}
+
 void CodeGen::genDefsThrift(const TypeGraph& typeGraph, std::string& code) {
   for (const Type& t : typeGraph.finalTypes) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
@@ -542,7 +547,7 @@ void CodeGen::getClassSizeFuncDef(const Class& c, std::string& code) {
     std::string childVtableName = "vtable for ";
     childVtableName += fqChildName;
 
-    auto optVtableSym = symbols_.locateSymbol(childVtableName, true);
+    auto optVtableSym = symbols_->locateSymbol(childVtableName, true);
     if (!optVtableSym) {
       //        LOG(ERROR) << "Failed to find vtable address for '" <<
       //        childVtableName; LOG(ERROR) << "Falling back to non dynamic
@@ -1115,15 +1120,8 @@ bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType, std::string& code) {
 bool CodeGen::codegenFromDrgn(struct drgn_type* drgnType,
                               std::string& code,
                               RootFunctionName name) {
-  try {
-    containerInfos_.reserve(config_.containerConfigPaths.size());
-    for (const auto& path : config_.containerConfigPaths) {
-      registerContainer(path);
-    }
-  } catch (const ContainerInfoError& err) {
-    LOG(ERROR) << "Error reading container TOML file " << err.what();
+  if (!registerContainers())
     return false;
-  }
 
   try {
     addDrgnRoot(drgnType, typeGraph_);
@@ -1148,6 +1146,19 @@ void CodeGen::exportDrgnTypes(TypeHierarchy& th,
   }
 }
 
+bool CodeGen::registerContainers() {
+  try {
+    containerInfos_.reserve(config_.containerConfigPaths.size());
+    for (const auto& path : config_.containerConfigPaths) {
+      registerContainer(path);
+    }
+    return true;
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return false;
+  }
+}
+
 void CodeGen::registerContainer(std::unique_ptr<ContainerInfo> info) {
   VLOG(1) << "Registered container: " << info->typeName;
   containerInfos_.emplace_back(std::move(info));
@@ -1189,7 +1200,7 @@ void CodeGen::transform(TypeGraph& typeGraph) {
         .chaseRawPointers = config_.features[Feature::ChaseRawPointers],
     };
     DrgnParser drgnParser{typeGraph, options};
-    pm.addPass(AddChildren::createPass(drgnParser, symbols_));
+    pm.addPass(AddChildren::createPass(drgnParser, *symbols_));
 
     // Re-run passes over newly added children
     pm.addPass(IdentifyContainers::createPass(containerInfos_));
diff --git a/oi/CodeGen.h b/oi/CodeGen.h
index 9c76238..8fc5a81 100644
--- a/oi/CodeGen.h
+++ b/oi/CodeGen.h
@@ -41,8 +41,9 @@ namespace oi::detail {
 
 class CodeGen {
  public:
+  CodeGen(const OICodeGen::Config& config);
   CodeGen(const OICodeGen::Config& config, SymbolService& symbols)
-      : config_(config), symbols_(symbols) {
+      : config_(config), symbols_(&symbols) {
   }
 
   struct ExactName {
@@ -65,6 +66,7 @@ class CodeGen {
                        std::list<drgn_type>& drgnTypes,
                        drgn_type** rootType) const;
 
+  bool registerContainers();
   void registerContainer(std::unique_ptr<ContainerInfo> containerInfo);
   void registerContainer(const std::filesystem::path& path);
   void addDrgnRoot(struct drgn_type* drgnType,
@@ -77,7 +79,7 @@ class CodeGen {
  private:
   type_graph::TypeGraph typeGraph_;
   const OICodeGen::Config& config_;
-  SymbolService& symbols_;
+  SymbolService* symbols_ = nullptr;
   std::vector<std::unique_ptr<ContainerInfo>> containerInfos_;
   std::unordered_set<const ContainerInfo*> definedContainers_;
   std::unordered_map<const type_graph::Class*, const type_graph::Member*>
diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp
index 1cb2a49..76cf939 100644
--- a/oi/OIGenerator.cpp
+++ b/oi/OIGenerator.cpp
@@ -16,130 +16,149 @@
 
 #include "oi/OIGenerator.h"
 
+#include <clang/AST/Mangle.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
 #include <glog/logging.h>
 
-#include <boost/core/demangle.hpp>
 #include <fstream>
-#include <iostream>
-#include <string_view>
+#include <range/v3/core.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <stdexcept>
 #include <unordered_map>
 #include <variant>
 
 #include "oi/CodeGen.h"
 #include "oi/Config.h"
-#include "oi/DrgnUtils.h"
 #include "oi/Headers.h"
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/TypeGraph.h"
+#include "oi/type_graph/Types.h"
 
 namespace oi::detail {
+namespace {
 
-std::unordered_map<std::string, std::string>
-OIGenerator::oilStrongToWeakSymbolsMap(drgnplusplus::program& prog) {
-  static constexpr std::string_view strongSymbolPrefix =
-      "oi::IntrospectionResult oi::introspect<";
-  static constexpr std::string_view weakSymbolPrefix =
-      "oi::IntrospectionResult oi::introspectImpl<";
+class ConsumerContext;
 
-  std::unordered_map<std::string, std::pair<std::string, std::string>>
-      templateArgsToSymbolsMap;
-
-  auto symbols = prog.find_all_symbols();
-  for (drgn_symbol* sym : *symbols) {
-    auto symName = drgnplusplus::symbol::name(sym);
-    if (symName == nullptr || *symName == '\0')
-      continue;
-    auto demangled = boost::core::demangle(symName);
-
-    if (demangled.starts_with(strongSymbolPrefix)) {
-      auto& matchedSyms = templateArgsToSymbolsMap[demangled.substr(
-          strongSymbolPrefix.length())];
-      if (!matchedSyms.first.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.first
-                     << "` and `" << symName << '`';
-      }
-      matchedSyms.first = symName;
-    } else if (demangled.starts_with(weakSymbolPrefix)) {
-      auto& matchedSyms =
-          templateArgsToSymbolsMap[demangled.substr(weakSymbolPrefix.length())];
-      if (!matchedSyms.second.empty()) {
-        LOG(WARNING) << "non-unique symbols found: `" << matchedSyms.second
-                     << "` and `" << symName << "`";
-      }
-      matchedSyms.second = symName;
-    }
+class CreateTypeGraphConsumer;
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  std::unordered_map<std::string, std::string> strongToWeakSymbols;
-  for (auto& [_, val] : templateArgsToSymbolsMap) {
-    if (val.first.empty() || val.second.empty()) {
-      continue;
-    }
-    strongToWeakSymbols[std::move(val.first)] = std::move(val.second);
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
   }
 
-  return strongToWeakSymbols;
-}
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
 
-std::unordered_map<std::string, drgn_qualified_type>
-OIGenerator::findOilTypesAndNames(drgnplusplus::program& prog) {
-  auto strongToWeakSymbols = oilStrongToWeakSymbolsMap(prog);
+ private:
+  ConsumerContext& ctx;
+};
 
-  std::unordered_map<std::string, drgn_qualified_type> out;
+class ConsumerContext {
+ public:
+  ConsumerContext(const std::vector<std::unique_ptr<ContainerInfo>>& cis)
+      : containerInfos{cis} {
+  }
 
-  for (drgn_qualified_type& func : drgnplusplus::func_iterator(prog)) {
-    std::string strongLinkageName;
-    {
-      const char* linkageNameCstr;
-      if (auto err = drgnplusplus::error(
-              drgn_type_linkage_name(func.type, &linkageNameCstr))) {
-        // throw err;
+  type_graph::TypeGraph typeGraph;
+  std::unordered_map<std::string, type_graph::Type*> nameToTypeMap;
+  std::optional<bool> pic;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+}  // namespace
+
+int OIGenerator::generate(clang::tooling::CompilationDatabase& db,
+                          const std::vector<std::string>& sourcePaths) {
+  std::map<Feature, bool> featuresMap = {
+      {Feature::TypeGraph, true},
+      {Feature::TreeBuilderV2, true},
+      {Feature::Library, true},
+      {Feature::PackStructs, true},
+      {Feature::PruneTypeGraph, true},
+  };
+
+  OICodeGen::Config generatorConfig{};
+  OICompiler::Config compilerConfig{};
+
+  auto features = config::processConfigFiles(
+      configFilePaths, featuresMap, compilerConfig, generatorConfig);
+  if (!features) {
+    LOG(ERROR) << "failed to process config file";
+    return -1;
+  }
+  generatorConfig.features = *features;
+  compilerConfig.features = *features;
+
+  std::vector<std::unique_ptr<ContainerInfo>> containerInfos;
+  containerInfos.reserve(generatorConfig.containerConfigPaths.size());
+  try {
+    for (const auto& path : generatorConfig.containerConfigPaths) {
+      auto info = std::make_unique<ContainerInfo>(path);
+      if (info->requiredFeatures != (*features & info->requiredFeatures)) {
+        VLOG(1) << "Skipping container (feature conflict): " << info->typeName;
         continue;
       }
-      strongLinkageName = linkageNameCstr;
+      containerInfos.emplace_back(std::move(info));
     }
-
-    std::string weakLinkageName;
-    if (auto search = strongToWeakSymbols.find(strongLinkageName);
-        search != strongToWeakSymbols.end()) {
-      weakLinkageName = search->second;
-    } else {
-      continue;  // not an oil strong symbol
-    }
-
-    // IntrospectionResult (*)(const T&)
-    CHECK(drgn_type_has_parameters(func.type)) << "functions have parameters";
-    CHECK(drgn_type_num_parameters(func.type) == 1)
-        << "introspection func has one parameter";
-
-    auto* params = drgn_type_parameters(func.type);
-    drgn_qualified_type tType;
-    if (auto err =
-            drgnplusplus::error(drgn_parameter_type(&params[0], &tType))) {
-      throw err;
-    }
-
-    if (drgn_type_has_name(tType.type)) {
-      LOG(INFO) << "found OIL type: " << drgn_type_name(tType.type);
-    } else {
-      LOG(INFO) << "found OIL type: (no name)";
-    }
-    out.emplace(std::move(weakLinkageName), tType);
+  } catch (const ContainerInfoError& err) {
+    LOG(ERROR) << "Error reading container TOML file " << err.what();
+    return -1;
   }
 
-  return out;
-}
+  ConsumerContext ctx{containerInfos};
+  CreateTypeGraphActionFactory factory{ctx};
 
-fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
-                                      const OICompiler::Config& compilerConfig,
-                                      const drgn_qualified_type& type,
-                                      const std::string& linkageName,
-                                      SymbolService& symbols) {
-  CodeGen codegen{generatorConfig, symbols};
+  clang::tooling::ClangTool tool{db, sourcePaths};
+  if (auto ret = tool.run(&factory); ret != 0) {
+    return ret;
+  }
+
+  if (ctx.nameToTypeMap.size() > 1)
+    throw std::logic_error(
+        "found more than one site to generate for but we can't currently "
+        "handle this case");
+
+  if (ctx.nameToTypeMap.empty()) {
+    LOG(ERROR) << "Nothing to generate!";
+    return failIfNothingGenerated ? -1 : 0;
+  }
+  const auto& linkageName = ctx.nameToTypeMap.begin()->first;
+
+  compilerConfig.usePIC = ctx.pic.value();
+  CodeGen codegen{generatorConfig};
+  for (auto&& ptr : containerInfos)
+    codegen.registerContainer(std::move(ptr));
+  codegen.transform(ctx.typeGraph);
 
   std::string code;
-  if (!codegen.codegenFromDrgn(type.type, linkageName, code)) {
-    LOG(ERROR) << "codegen failed!";
-    return {};
-  }
+  codegen.generate(ctx.typeGraph, code, CodeGen::ExactName{linkageName});
 
   std::string sourcePath = sourceFileDumpPath;
   if (sourceFileDumpPath.empty()) {
@@ -152,78 +171,109 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
   }
 
   OICompiler compiler{{}, compilerConfig};
-
-  // TODO: Revert to outputPath and remove printing when typegraph is done.
-  fs::path tmpObject = outputPath;
-  tmpObject.replace_extension(
-      "." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
-
-  if (!compiler.compile(code, sourcePath, tmpObject)) {
-    return {};
-  }
-  return tmpObject;
+  return compiler.compile(code, sourcePath, outputPath) ? 0 : -1;
 }
 
-int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
-  drgnplusplus::program prog;
+namespace {
 
-  {
-    std::array<const char*, 1> objectPaths = {{primaryObject.c_str()}};
-    if (auto err = drgnplusplus::error(
-            drgn_program_load_debug_info(prog.get(),
-                                         std::data(objectPaths),
-                                         std::size(objectPaths),
-                                         false,
-                                         false))) {
-      LOG(ERROR) << "error loading debug info program: " << err;
-      throw err;
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx(ctx_) {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    auto* tu_decl = Context.getTranslationUnitDecl();
+    auto decls = tu_decl->decls();
+    auto oi_namespaces = decls | ranges::views::transform([](auto* p) {
+                           return llvm::dyn_cast<clang::NamespaceDecl>(p);
+                         }) |
+                         ranges::views::filter([](auto* ns) {
+                           return ns != nullptr && ns->getName() == "oi";
+                         });
+    if (oi_namespaces.empty()) {
+      LOG(WARNING) << "Failed to find `oi` namespace. Does this input "
+                      "include <oi/oi.h>?";
+      return;
     }
-  }
 
-  auto oilTypes = findOilTypesAndNames(prog);
-
-  std::map<Feature, bool> featuresMap = {
-      {Feature::TypeGraph, true},
-      {Feature::TreeBuilderV2, true},
-      {Feature::Library, true},
-      {Feature::PackStructs, true},
-      {Feature::PruneTypeGraph, true},
-  };
-
-  OICodeGen::Config generatorConfig{};
-  OICompiler::Config compilerConfig{};
-  compilerConfig.usePIC = pic;
-
-  auto features = config::processConfigFiles(
-      configFilePaths, featuresMap, compilerConfig, generatorConfig);
-  if (!features) {
-    LOG(ERROR) << "failed to process config file";
-    return -1;
-  }
-  generatorConfig.features = *features;
-  compilerConfig.features = *features;
-
-  size_t failures = 0;
-  for (const auto& [linkageName, type] : oilTypes) {
-    if (auto obj = generateForType(
-            generatorConfig, compilerConfig, type, linkageName, symbols);
-        !obj.empty()) {
-      std::cout << obj.string() << std::endl;
-    } else {
-      LOG(WARNING) << "failed to generate for symbol `" << linkageName
-                   << "`. this is non-fatal but the call will not work.";
-      failures++;
+    auto introspectImpl =
+        std::move(oi_namespaces) |
+        ranges::views::for_each([](auto* ns) { return ns->decls(); }) |
+        ranges::views::transform([](auto* p) {
+          return llvm::dyn_cast<clang::FunctionTemplateDecl>(p);
+        }) |
+        ranges::views::filter([](auto* td) {
+          return td != nullptr && td->getName() == "introspectImpl";
+        }) |
+        ranges::views::take(1) | ranges::to<std::vector>();
+    if (introspectImpl.empty()) {
+      LOG(WARNING)
+          << "Failed to find `oi::introspect` within the `oi` namespace. Did "
+             "you compile with `OIL_AOT_COMPILATION=1`?";
+      return;
     }
-  }
 
-  size_t successes = oilTypes.size() - failures;
-  LOG(INFO) << "object introspection generation complete. " << successes
-            << " successes and " << failures << " failures.";
+    auto nameToClangTypeMap =
+        introspectImpl | ranges::views::for_each([](auto* td) {
+          return td->specializations();
+        }) |
+        ranges::views::transform(
+            [](auto* p) { return llvm::dyn_cast<clang::FunctionDecl>(p); }) |
+        ranges::views::filter([](auto* p) { return p != nullptr; }) |
+        ranges::views::transform(
+            [](auto* fd) -> std::pair<std::string, const clang::Type*> {
+              clang::ASTContext& Ctx = fd->getASTContext();
+              clang::ASTNameGenerator ASTNameGen(Ctx);
+              std::string name = ASTNameGen.getName(fd);
 
-  if (failures > 0 || (failIfNothingGenerated && successes == 0)) {
-    return -1;
+              assert(fd->getNumParams() == 1);
+              const clang::Type* type =
+                  fd->parameters()[0]->getType().getTypePtr();
+              return {name, type};
+            }) |
+        ranges::to<std::unordered_map>();
+    if (nameToClangTypeMap.empty())
+      return;
+
+    type_graph::ClangTypeParserOptions opts;
+    type_graph::ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, opts};
+
+    auto& Sema = *ctx.sema;
+    auto els = nameToClangTypeMap |
+               ranges::views::transform(
+                   [&parser, &Context, &Sema](
+                       auto& p) -> std::pair<std::string, type_graph::Type*> {
+                     return {p.first, &parser.parse(Context, Sema, *p.second)};
+                   });
+    ctx.nameToTypeMap.insert(els.begin(), els.end());
+
+    for (const auto& [name, type] : ctx.nameToTypeMap)
+      ctx.typeGraph.addRoot(*type);
   }
-  return 0;
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  // Compile the output as position independent if any input is position
+  // independent
+  bool pic = CI.getCodeGenOpts().RelocationModel == llvm::Reloc::PIC_;
+  ctx.pic = ctx.pic.value_or(false) || pic;
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
 }
 
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    clang::CompilerInstance& CI, clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+}  // namespace
 }  // namespace oi::detail
diff --git a/oi/OIGenerator.h b/oi/OIGenerator.h
index dd38384..fac947a 100644
--- a/oi/OIGenerator.h
+++ b/oi/OIGenerator.h
@@ -23,11 +23,19 @@
 #include "oi/OICodeGen.h"
 #include "oi/OICompiler.h"
 
+namespace clang::tooling {
+class CompilationDatabase;
+}
+
 namespace oi::detail {
+namespace type_graph {
+class Type;
+}
 
 class OIGenerator {
  public:
-  int generate(fs::path& primaryObject, SymbolService& symbols);
+  int generate(clang::tooling::CompilationDatabase&,
+               const std::vector<std::string>&);
 
   void setOutputPath(fs::path _outputPath) {
     outputPath = std::move(_outputPath);
@@ -41,8 +49,8 @@ class OIGenerator {
   void setFailIfNothingGenerated(bool fail) {
     failIfNothingGenerated = fail;
   }
-  void setUsePIC(bool pic_) {
-    pic = pic_;
+  void setClangArgs(std::vector<std::string> args_) {
+    clangArgs = std::move(args_);
   }
 
  private:
@@ -50,20 +58,7 @@ class OIGenerator {
   std::vector<std::filesystem::path> configFilePaths;
   std::filesystem::path sourceFileDumpPath;
   bool failIfNothingGenerated = false;
-  bool pic = false;
-
-  std::unordered_map<std::string, std::string> oilStrongToWeakSymbolsMap(
-      drgnplusplus::program& prog);
-
-  std::unordered_map<std::string, drgn_qualified_type> findOilTypesAndNames(
-      drgnplusplus::program& prog);
-
-  std::filesystem::path generateForType(
-      const OICodeGen::Config& generatorConfig,
-      const OICompiler::Config& compilerConfig,
-      const drgn_qualified_type& type,
-      const std::string& linkageName,
-      SymbolService& symbols);
+  std::vector<std::string> clangArgs;
 };
 
 }  // namespace oi::detail
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 190bdfc..6c4da13 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(type_graph
   AddChildren.cpp
   AddPadding.cpp
   AlignmentCalc.cpp
+  ClangTypeParser.cpp
   DrgnExporter.cpp
   DrgnParser.cpp
   EnforceCompatibility.cpp
@@ -27,3 +28,4 @@ target_link_libraries(type_graph
   "-L${DRGN_PATH}/.libs"
   drgn
 )
+target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
new file mode 100644
index 0000000..5190478
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -0,0 +1,418 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ClangTypeParser.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Decl.h>
+#include <clang/AST/DeclTemplate.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/AST/Type.h>
+#include <clang/Basic/DiagnosticSema.h>
+#include <clang/Sema/Sema.h>
+#include <glog/logging.h>
+
+#include <regex>
+#include <stdexcept>
+
+#include "oi/type_graph/Types.h"
+
+namespace oi::detail::type_graph {
+namespace {
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty);
+}
+
+Type& ClangTypeParser::parse(clang::ASTContext& ast_,
+                             clang::Sema& sema_,
+                             const clang::Type& ty) {
+  ast = &ast_;
+  sema = &sema_;
+
+  depth_ = 0;
+  return enumerateType(ty);
+}
+
+Type& ClangTypeParser::enumerateType(const clang::Type& ty) {
+  // Avoid re-enumerating an already-processsed type
+  if (auto it = clang_types_.find(&ty); it != clang_types_.end())
+    return it->second;
+
+  struct DepthTracker {
+    DepthTracker(ClangTypeParser& self_) : self{self_} {
+      self.depth_++;
+    }
+    ~DepthTracker() {
+      self.depth_--;
+    }
+
+   private:
+    ClangTypeParser& self;
+  } d{*this};
+
+  if (VLOG_IS_ON(3)) {
+    std::string fqName = clang::TypeName::getFullyQualifiedName(
+        clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+    VLOG(3) << std::string(depth_ * 2, ' ') << fqName;
+  }
+
+  // TODO: This check basically doesn't work. The action has failed before it
+  // returns false. Fix this.
+  if (!requireCompleteType(*sema, ty)) {
+    std::string fqName = clang::TypeName::getFullyQualifiedName(
+        clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+    return makeType<Incomplete>(ty, std::move(fqName));
+  }
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Record:
+      return enumerateClass(llvm::cast<const clang::RecordType>(ty));
+    case clang::Type::LValueReference:
+      return enumerateReference(
+          llvm::cast<const clang::LValueReferenceType>(ty));
+    case clang::Type::Pointer:
+      return enumeratePointer(llvm::cast<const clang::PointerType>(ty));
+    case clang::Type::SubstTemplateTypeParm:
+      return enumerateSubstTemplateTypeParm(
+          llvm::cast<const clang::SubstTemplateTypeParmType>(ty));
+    case clang::Type::Builtin:
+      return enumeratePrimitive(llvm::cast<const clang::BuiltinType>(ty));
+    case clang::Type::Elaborated:
+      return enumerateElaboratedType(
+          llvm::cast<const clang::ElaboratedType>(ty));
+    case clang::Type::TemplateSpecialization:
+      return enumerateTemplateSpecialization(
+          llvm::cast<const clang::TemplateSpecializationType>(ty));
+    case clang::Type::UnaryTransform:
+      return enumerateUnaryTransformType(
+          llvm::cast<const clang::UnaryTransformType>(ty));
+    case clang::Type::Decltype:
+      return enumerateDecltypeType(llvm::cast<const clang::DecltypeType>(ty));
+    case clang::Type::Typedef:
+      return enumerateTypedef(llvm::cast<const clang::TypedefType>(ty));
+    case clang::Type::Using:
+      return enumerateUsing(llvm::cast<const clang::UsingType>(ty));
+    case clang::Type::ConstantArray:
+      return enumerateArray(llvm::cast<const clang::ConstantArrayType>(ty));
+    case clang::Type::Enum:
+      return enumerateEnum(llvm::cast<const clang::EnumType>(ty));
+
+    default:
+      throw std::logic_error(std::string("unsupported TypeClass `") +
+                             ty.getTypeClassName() + '`');
+  }
+}
+
+Type& ClangTypeParser::enumerateDecltypeType(const clang::DecltypeType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Type& ClangTypeParser::enumerateUnaryTransformType(
+    const clang::UnaryTransformType& ty) {
+  return enumerateType(*ty.getUnderlyingType());
+}
+
+Typedef& ClangTypeParser::enumerateUsing(const clang::UsingType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+  std::string name = ty.getFoundDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
+  auto& inner = enumerateType(*ty.desugar());
+
+  std::string name = ty.getDecl()->getNameAsString();
+  return makeType<Typedef>(ty, std::move(name), inner);
+}
+
+Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string name = ty.getDecl()->getNameAsString();
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  std::map<int64_t, std::string> enumeratorMap;
+  if (options_.readEnumValues) {
+    for (const auto* enumerator : ty.getDecl()->enumerators()) {
+      enumeratorMap.emplace(enumerator->getInitVal().getExtValue(),
+                            enumerator->getNameAsString());
+    }
+  }
+
+  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+}
+
+Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
+  uint64_t len = ty.getSize().getLimitedValue();
+  auto& t = enumerateType(*ty.getElementType());
+  return makeType<Array>(ty, t, len);
+}
+
+Type& ClangTypeParser::enumerateTemplateSpecialization(
+    const clang::TemplateSpecializationType& ty) {
+  if (ty.isSugared())
+    return enumerateType(*ty.desugar());
+
+  LOG(WARNING) << "failed on a TemplateSpecializationType";
+  ty.dump();
+  return makeType<Primitive>(ty, Primitive::Kind::Int32);
+}
+
+Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
+  auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
+
+  if (auto* info = getContainerInfo(fqName)) {
+    auto& c = makeType<Container>(ty, *info, size, nullptr);
+    enumerateClassTemplateParams(ty, c.templateParams);
+    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)));
+    return c;
+  }
+
+  auto* decl = ty.getDecl();
+
+  std::string name = decl->getNameAsString();
+
+  auto kind = Class::Kind::Struct;  // TODO: kind
+
+  int virtuality = 0;
+
+  auto& c = makeType<Class>(
+      ty, kind, std::move(name), std::move(fqName), size, virtuality);
+
+  enumerateClassTemplateParams(ty, c.templateParams);
+  // enumerateClassParents(type, c.parents);
+  enumerateClassMembers(ty, c.members);
+  // enumerateClassFunctions(type, c.functions);
+
+  return c;
+}
+
+void ClangTypeParser::enumerateClassTemplateParams(
+    const clang::RecordType& ty, std::vector<TemplateParam>& params) {
+  assert(params.empty());
+
+  auto* decl = dyn_cast<clang::ClassTemplateSpecializationDecl>(ty.getDecl());
+  if (decl == nullptr)
+    return;
+
+  const auto& list = decl->getTemplateArgs();
+
+  params.reserve(list.size());
+  for (const auto& arg : list.asArray()) {
+    if (auto p = enumerateTemplateParam(arg))
+      params.emplace_back(std::move(p.value()));
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateParam(
+    const clang::TemplateArgument& p) {
+  switch (p.getKind()) {
+    case clang::TemplateArgument::Type: {
+      auto qualType = p.getAsType();
+      QualifierSet qualifiers;
+      qualifiers[Qualifier::Const] = qualType.isConstQualified();
+      Type& ttype = enumerateType(*qualType);
+      return TemplateParam{ttype, qualifiers};
+    }
+    case clang::TemplateArgument::Integral: {
+      auto& ty = enumerateType(*p.getIntegralType());
+      llvm::SmallString<32> val;
+      p.getAsIntegral().toString(val);
+      return TemplateParam{ty, std::string(val)};
+    }
+    case clang::TemplateArgument::Template: {
+      return enumerateTemplateTemplateParam(p.getAsTemplate());
+    }
+
+#define X(name)                       \
+  case clang::TemplateArgument::name: \
+    throw std::logic_error("unsupported template argument kind: " #name);
+
+      X(Null)
+      X(Declaration)
+      X(NullPtr)
+      X(TemplateExpansion)
+      X(Expression)
+      X(Pack)
+#undef X
+  }
+}
+
+std::optional<TemplateParam> ClangTypeParser::enumerateTemplateTemplateParam(
+    const clang::TemplateName& tn) {
+  switch (tn.getKind()) {
+    case clang::TemplateName::Template:
+      return std::nullopt;
+
+#define X(name)                   \
+  case clang::TemplateName::name: \
+    throw std::logic_error("unsupported template name kind: " #name);
+
+      X(OverloadedTemplate)
+      X(AssumedTemplate)
+      X(QualifiedTemplate)
+      X(DependentTemplate)
+      X(SubstTemplateTemplateParm)
+      X(SubstTemplateTemplateParmPack)
+      X(UsingTemplate)
+#undef X
+  }
+}
+
+void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
+                                            std::vector<Member>& members) {
+  assert(members.empty());
+
+  auto* decl = ty.getDecl();
+
+  for (const auto* field : decl->fields()) {
+    clang::QualType qualType = field->getType();
+    std::string member_name = field->getNameAsString();
+
+    size_t size_in_bits = 0;
+    if (field->isBitField()) {
+      size_in_bits = field->getBitWidthValue(*ast);
+    }
+
+    size_t offset_in_bits = decl->getASTContext().getFieldOffset(field);
+
+    auto& mtype = enumerateType(*qualType);
+    Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    members.push_back(m);
+  }
+
+  std::sort(members.begin(), members.end(), [](const auto& a, const auto& b) {
+    return a.bitOffset < b.bitOffset;
+  });
+}
+
+Type& ClangTypeParser::enumerateReference(
+    const clang::LValueReferenceType& ty) {
+  // TODO: function references
+  Type& t = enumerateType(*ty.getPointeeType());
+  if (dynamic_cast<Incomplete*>(&t))
+    return makeType<Pointer>(ty, t);
+
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumeratePointer(const clang::PointerType& ty) {
+  // TODO: function pointers
+  if (!chasePointer())
+    return makeType<Primitive>(ty, Primitive::Kind::StubbedPointer);
+
+  Type& t = enumerateType(*ty.getPointeeType());
+  return makeType<Reference>(ty, t);
+}
+
+Type& ClangTypeParser::enumerateSubstTemplateTypeParm(
+    const clang::SubstTemplateTypeParmType& ty) {
+  // Clang wraps any type that was substituted from e.g. `T` in this type. It
+  // should have no representation in the type graph.
+  return enumerateType(*ty.getReplacementType());
+}
+
+Type& ClangTypeParser::enumerateElaboratedType(
+    const clang::ElaboratedType& ty) {
+  // Clang wraps any type that is name qualified in this type. It should have no
+  // representation in the type graph.
+  return enumerateType(*ty.getNamedType());
+}
+
+Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
+  switch (ty.getKind()) {
+    case clang::BuiltinType::Void:
+      return makeType<Primitive>(ty, Primitive::Kind::Void);
+
+    case clang::BuiltinType::Bool:
+      return makeType<Primitive>(ty, Primitive::Kind::Bool);
+
+    case clang::BuiltinType::Char_U:
+    case clang::BuiltinType::UChar:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt8);
+    case clang::BuiltinType::WChar_U:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+
+    case clang::BuiltinType::Char_S:
+    case clang::BuiltinType::SChar:
+      return makeType<Primitive>(ty, Primitive::Kind::Int8);
+    case clang::BuiltinType::WChar_S:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Char16:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Char32:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+
+    case clang::BuiltinType::UShort:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt16);
+    case clang::BuiltinType::UInt:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt32);
+    case clang::BuiltinType::ULong:
+      return makeType<Primitive>(ty, Primitive::Kind::UInt64);
+    case clang::BuiltinType::ULongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Short:
+      return makeType<Primitive>(ty, Primitive::Kind::Int16);
+    case clang::BuiltinType::Int:
+      return makeType<Primitive>(ty, Primitive::Kind::Int32);
+    case clang::BuiltinType::Long:
+    case clang::BuiltinType::LongLong:
+      return makeType<Primitive>(ty, Primitive::Kind::Int64);
+
+    case clang::BuiltinType::Float:
+      return makeType<Primitive>(ty, Primitive::Kind::Float32);
+    case clang::BuiltinType::Double:
+    case clang::BuiltinType::LongDouble:
+      return makeType<Primitive>(ty, Primitive::Kind::Float64);
+
+    case clang::BuiltinType::UInt128:
+    case clang::BuiltinType::Int128:
+    default:
+      throw std::logic_error(std::string("unsupported BuiltinType::Kind"));
+  }
+}
+
+bool ClangTypeParser::chasePointer() const {
+  // Always chase top-level pointers
+  if (depth_ == 1)
+    return true;
+  return options_.chaseRawPointers;
+}
+
+ContainerInfo* ClangTypeParser::getContainerInfo(
+    const std::string& fqName) const {
+  for (const auto& containerInfo : containers_) {
+    if (std::regex_search(fqName, containerInfo->matcher)) {
+      return containerInfo.get();
+    }
+  }
+  return nullptr;
+}
+
+namespace {
+
+bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
+  if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
+    return true;  // treat as complete
+
+  // TODO: This is a terrible warning.
+  return !sema.RequireCompleteType(
+      sema.getASTContext().getTranslationUnitDecl()->getEndLoc(),
+      clang::QualType{&ty, 0},
+      clang::diag::warn_nsconsumed_attribute_mismatch);
+}
+
+}  // namespace
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
new file mode 100644
index 0000000..9b27d64
--- /dev/null
+++ b/oi/type_graph/ClangTypeParser.h
@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <unordered_map>
+
+#include "oi/type_graph/TypeGraph.h"
+
+namespace clang {
+class ASTContext;
+class BuiltinType;
+class ConstantArrayType;
+class DecltypeType;
+class ElaboratedType;
+class EnumType;
+class LValueReferenceType;
+class PointerType;
+class RecordType;
+class Sema;
+class SubstTemplateTypeParmType;
+class TemplateArgument;
+class TemplateName;
+class TemplateSpecializationType;
+class Type;
+class TypedefType;
+class UnaryTransformType;
+class UsingType;
+}  // namespace clang
+
+struct ContainerInfo;
+
+namespace oi::detail::type_graph {
+
+class Array;
+class Class;
+class Enum;
+class Member;
+class Primitive;
+class Reference;
+class Type;
+class TypeGraph;
+class Typedef;
+struct TemplateParam;
+
+struct ClangTypeParserOptions {
+  bool chaseRawPointers = false;
+  bool readEnumValues = false;
+};
+
+/*
+ * ClangTypeParser
+ *
+ * Reads source information from a source file to build a type graph.
+ * Returns a reference to the Type node corresponding to the given clang::Type.
+ */
+class ClangTypeParser {
+ public:
+  ClangTypeParser(TypeGraph& typeGraph,
+                  const std::vector<std::unique_ptr<ContainerInfo>>& containers,
+                  ClangTypeParserOptions options)
+      : typeGraph_{typeGraph}, containers_{containers}, options_{options} {
+  }
+
+  // Parse from a clang type.
+  Type& parse(clang::ASTContext&, clang::Sema&, const clang::Type&);
+
+ private:
+  TypeGraph& typeGraph_;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containers_;
+  ClangTypeParserOptions options_;
+  clang::ASTContext* ast;
+  clang::Sema* sema;
+
+  uint_fast32_t depth_;
+  std::unordered_map<const clang::Type*, std::reference_wrapper<Type>>
+      clang_types_;
+
+  Type& enumerateType(const clang::Type&);
+  Type& enumerateClass(const clang::RecordType&);
+  Type& enumerateReference(const clang::LValueReferenceType&);
+  Type& enumeratePointer(const clang::PointerType&);
+  Type& enumerateSubstTemplateTypeParm(const clang::SubstTemplateTypeParmType&);
+  Primitive& enumeratePrimitive(const clang::BuiltinType&);
+  Type& enumerateElaboratedType(const clang::ElaboratedType&);
+  Type& enumerateTemplateSpecialization(
+      const clang::TemplateSpecializationType&);
+  Typedef& enumerateTypedef(const clang::TypedefType&);
+  Typedef& enumerateUsing(const clang::UsingType&);
+  Type& enumerateUnaryTransformType(const clang::UnaryTransformType&);
+  Type& enumerateDecltypeType(const clang::DecltypeType&);
+
+  Array& enumerateArray(const clang::ConstantArrayType&);
+  Enum& enumerateEnum(const clang::EnumType&);
+
+  void enumerateClassTemplateParams(const clang::RecordType&,
+                                    std::vector<TemplateParam>&);
+  std::optional<TemplateParam> enumerateTemplateParam(
+      const clang::TemplateArgument&);
+  std::optional<TemplateParam> enumerateTemplateTemplateParam(
+      const clang::TemplateName&);
+
+  void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
+
+  ContainerInfo* getContainerInfo(const std::string& fqName) const;
+
+  template <typename T, typename... Args>
+  T& makeType(const clang::Type& clangType, Args&&... args) {
+    auto& newType = typeGraph_.makeType<T>(std::forward<Args>(args)...);
+    clang_types_.insert({&clangType, newType});
+    return newType;
+  }
+
+  bool chasePointer() const;
+};
+
+}  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/DrgnExporter.cpp b/oi/type_graph/DrgnExporter.cpp
index 802857e..df9722d 100644
--- a/oi/type_graph/DrgnExporter.cpp
+++ b/oi/type_graph/DrgnExporter.cpp
@@ -200,6 +200,14 @@ drgn_type* DrgnExporter::visit(Pointer& p) {
   return drgnType;
 }
 
+drgn_type* DrgnExporter::visit(Reference& p) {
+  auto* drgnType =
+      makeDrgnType(DRGN_TYPE_POINTER, false, DRGN_NOT_PRIMITIVE_TYPE, p);
+  auto* pointeeType = accept(p.pointeeType());
+  th_.pointerToTypeMap[drgnType] = pointeeType;
+  return drgnType;
+}
+
 drgn_type* DrgnExporter::visit(Dummy& d) {
   return makeDrgnType(DRGN_TYPE_VOID, false, DRGN_C_TYPE_VOID, d);
 }
diff --git a/oi/type_graph/DrgnExporter.h b/oi/type_graph/DrgnExporter.h
index 45f02c6..24f4c59 100644
--- a/oi/type_graph/DrgnExporter.h
+++ b/oi/type_graph/DrgnExporter.h
@@ -54,6 +54,7 @@ class DrgnExporter : public Visitor<drgn_type*> {
   drgn_type* visit(Array&) override;
   drgn_type* visit(Typedef&) override;
   drgn_type* visit(Pointer&) override;
+  drgn_type* visit(Reference&) override;
   drgn_type* visit(Dummy&) override;
   drgn_type* visit(DummyAllocator&) override;
   drgn_type* visit(CaptureKeys&) override;
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index b1d1e8d..725ab59 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -200,6 +200,15 @@ void NameGen::visit(Pointer& p) {
   p.setInputName(inputName);
 }
 
+void NameGen::visit(Reference& r) {
+  RecursiveVisitor::visit(r);
+
+  r.regenerateName();
+  std::string inputName{r.pointeeType().inputName()};
+  inputName += '&';
+  r.setInputName(inputName);
+}
+
 void NameGen::visit(DummyAllocator& d) {
   RecursiveVisitor::visit(d);
   d.regenerateName();
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 7b01aef..4f1c258 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -46,6 +46,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Array& a) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& r) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index 0dae0ab..f598161 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -150,6 +150,17 @@ void Printer::visit(const Pointer& p) {
   print(p.pointeeType());
 }
 
+void Printer::visit(const Reference& r) {
+  if (prefix(r))
+    return;
+
+  out_ << "Reference";
+  if (auto inp = r.inputName(); !inp.empty())
+    out_ << " [" << inp << "]";
+  out_ << std::endl;
+  print(r.pointeeType());
+}
+
 void Printer::visit(const Dummy& d) {
   if (prefix(d))
     return;
diff --git a/oi/type_graph/Printer.h b/oi/type_graph/Printer.h
index 8d7138a..e68f562 100644
--- a/oi/type_graph/Printer.h
+++ b/oi/type_graph/Printer.h
@@ -40,6 +40,7 @@ class Printer : public ConstVisitor {
   void visit(const Array& a) override;
   void visit(const Typedef& td) override;
   void visit(const Pointer& p) override;
+  void visit(const Reference& r) override;
   void visit(const Dummy& d) override;
   void visit(const DummyAllocator& d) override;
   void visit(const CaptureKeys& d) override;
diff --git a/oi/type_graph/RemoveTopLevelPointer.cpp b/oi/type_graph/RemoveTopLevelPointer.cpp
index 9ecaaaa..c2fc91d 100644
--- a/oi/type_graph/RemoveTopLevelPointer.cpp
+++ b/oi/type_graph/RemoveTopLevelPointer.cpp
@@ -42,4 +42,8 @@ void RemoveTopLevelPointer::visit(Pointer& p) {
   topLevelType_ = &p.pointeeType();
 }
 
+void RemoveTopLevelPointer::visit(Reference& r) {
+  topLevelType_ = &r.pointeeType();
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/RemoveTopLevelPointer.h b/oi/type_graph/RemoveTopLevelPointer.h
index 611429f..3ed2c15 100644
--- a/oi/type_graph/RemoveTopLevelPointer.h
+++ b/oi/type_graph/RemoveTopLevelPointer.h
@@ -36,6 +36,7 @@ class RemoveTopLevelPointer : public LazyVisitor {
 
   void removeTopLevelPointers(std::vector<std::reference_wrapper<Type>>& types);
   void visit(Pointer& p) override;
+  void visit(Reference& r) override;
 
  private:
   Type* topLevelType_ = nullptr;
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f0b3fe9..1d5dd79 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -49,6 +49,7 @@
   X(Array)           \
   X(Typedef)         \
   X(Pointer)         \
+  X(Reference)       \
   X(Dummy)           \
   X(DummyAllocator)  \
   X(CaptureKeys)
@@ -734,6 +735,66 @@ class Pointer : public Type {
   std::string name_;
 };
 
+class Reference : public Type {
+ public:
+  explicit Reference(NodeId id, Type& pointeeType)
+      : pointeeType_(pointeeType), id_(id) {
+    regenerateName();
+  }
+
+  static inline constexpr bool has_node_id = true;
+
+  DECLARE_ACCEPT
+
+  virtual const std::string& name() const override {
+    return name_;
+  }
+
+  void regenerateName() {
+    // Following a reference wouldn't trigger cycle checking, as it would look
+    // like anything else we're sure is there. Generate as a pointer. It will be
+    // followed regardless of `ChaseRawPointers` because that affects whether a
+    // type becomes a `StubbedPointer` and not whether pointers are followed in
+    // the generated code.
+    name_ = pointeeType_.get().name() + "*";
+  }
+
+  virtual std::string_view inputName() const override {
+    return inputName_;
+  }
+
+  void setInputName(std::string name) {
+    inputName_ = std::move(name);
+  }
+
+  virtual size_t size() const override {
+    return sizeof(uintptr_t);
+  }
+
+  virtual uint64_t align() const override {
+    return size();
+  }
+
+  virtual NodeId id() const override {
+    return id_;
+  }
+
+  Type& pointeeType() const {
+    return pointeeType_;
+  }
+
+  void setPointeeType(Type& type) {
+    pointeeType_ = type;
+  }
+
+ private:
+  std::reference_wrapper<Type> pointeeType_;
+  std::string inputName_;
+  NodeId id_ = -1;
+
+  std::string name_;
+};
+
 /*
  * Dummy
  *
diff --git a/oi/type_graph/Visitor.h b/oi/type_graph/Visitor.h
index a7f7eec..f055aea 100644
--- a/oi/type_graph/Visitor.h
+++ b/oi/type_graph/Visitor.h
@@ -108,6 +108,9 @@ class RecursiveVisitor : public Visitor<void> {
   virtual void visit(Pointer& p) {
     accept(p.pointeeType());
   }
+  virtual void visit(Reference& r) {
+    accept(r.pointeeType());
+  }
   virtual void visit(Dummy&) {
   }
   virtual void visit(DummyAllocator& d) {
@@ -175,6 +178,10 @@ class RecursiveMutator : public Visitor<Type&> {
     p.setPointeeType(mutate(p.pointeeType()));
     return p;
   }
+  virtual Type& visit(Reference& p) {
+    p.setPointeeType(mutate(p.pointeeType()));
+    return p;
+  }
   virtual Type& visit(Dummy& d) {
     return d;
   }
diff --git a/tools/OILGen.cpp b/tools/OILGen.cpp
index d1e345d..77c3569 100644
--- a/tools/OILGen.cpp
+++ b/tools/OILGen.cpp
@@ -14,131 +14,80 @@
  * limitations under the License.
  */
 
+#include <clang/Tooling/CommonOptionsParser.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
 #include <cstdlib>
 #include <filesystem>
 #include <iostream>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/transform.hpp>
 #include <vector>
 
 #include "oi/OICodeGen.h"
 #include "oi/OIGenerator.h"
-#include "oi/OIOpts.h"
 
 namespace fs = std::filesystem;
 using namespace oi::detail;
 
-constexpr static OIOpts opts{
-    OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit."},
-    OIOpt{'o',
-          "output",
-          required_argument,
-          "<file>",
-          "Write output(s) to file(s) with this prefix."},
-    OIOpt{'c',
-          "config-file",
-          required_argument,
-          "<oid.toml>",
-          "Path to OI configuration file."},
-    OIOpt{'d',
-          "debug-level",
-          required_argument,
-          "<level>",
-          "Verbose level for logging"},
-    OIOpt{'j',
-          "dump-jit",
-          optional_argument,
-          "<jit.cpp>",
-          "Write generated code to a file (for debugging)."},
-    OIOpt{'e',
-          "exit-code",
-          no_argument,
-          nullptr,
-          "Return a bad exit code if nothing is generated."},
-    OIOpt{'p',
-          "pic",
-          no_argument,
-          nullptr,
-          "Generate position independent code."},
-};
+static llvm::cl::OptionCategory OilgenCategory("oilgen options");
 
-void usage() {
-  std::cerr << "usage: oilgen ARGS INPUT_OBJECT" << std::endl;
-  std::cerr << opts;
+static llvm::cl::list<std::string> ConfigFiles(
+    "config-file",
+    llvm::cl::desc(R"(</path/to/oid.toml>)"),
+    llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> OutputFile(
+    "output",
+    llvm::cl::desc(R"(Write output to this file.)"),
+    llvm::cl::init("a.o"),
+    llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<int> DebugLevel(
+    "debug-level",
+    llvm::cl::desc(R"(Verbose level for logging.)"),
+    llvm::cl::init(-1),
+    llvm::cl::cat(OilgenCategory));
+static llvm::cl::opt<std::string> DumpJit(
+    "dump-jit",
+    llvm::cl::desc(R"(Write the generated code to a file.)"),
+    llvm::cl::init("jit.cpp"),
+    llvm::cl::cat(OilgenCategory));
 
-  std::cerr << std::endl
-            << "You probably shouldn't be calling this application directly. "
-               "It's meant to be"
-            << std::endl
-            << "called by a clang plugin automatically with BUCK." << std::endl;
-}
-
-int main(int argc, char* argv[]) {
+int main(int argc, const char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   FLAGS_minloglevel = 0;
   FLAGS_stderrthreshold = 0;
 
-  fs::path outputPath = "a.o";
-  std::vector<fs::path> configFilePaths;
-  fs::path sourceFileDumpPath = "";
-  bool exitCode = false;
-  bool pic = false;
-
-  int c;
-  while ((c = getopt_long(
-              argc, argv, opts.shortOpts(), opts.longOpts(), nullptr)) != -1) {
-    switch (c) {
-      case 'h':
-        usage();
-        return EXIT_SUCCESS;
-      case 'o':
-        outputPath = optarg;
-        break;
-      case 'c':
-        configFilePaths.emplace_back(optarg);
-        break;
-      case 'd':
-        google::LogToStderr();
-        google::SetStderrLogging(google::INFO);
-        google::SetVLOGLevel("*", atoi(optarg));
-        // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
-        // but internally it's defined as 1 https://fburl.com/code/9fwams75
-        gflags::SetCommandLineOption("minloglevel", "0");
-        break;
-      case 'j':
-        sourceFileDumpPath = optarg != nullptr ? optarg : "jit.cpp";
-        break;
-      case 'e':
-        exitCode = true;
-        break;
-      case 'p':
-        pic = true;
-        break;
-    }
+  auto expectedParser =
+      clang::tooling::CommonOptionsParser::create(argc, argv, OilgenCategory);
+  if (!expectedParser) {
+    llvm::errs() << expectedParser.takeError();
+    return -1;
   }
+  clang::tooling::CommonOptionsParser& options = expectedParser.get();
+  auto& compilations = options.getCompilations();
 
-  if (optind >= argc) {
-    usage();
-    return EXIT_FAILURE;
-  }
-  fs::path primaryObject = argv[optind];
-
-  if ((setenv("DRGN_ENABLE_TYPE_ITERATOR", "1", 1)) < 0) {
-    LOG(ERROR) << "Failed to set environment variable\
-       DRGN_ENABLE_TYPE_ITERATOR\n";
-    exit(EXIT_FAILURE);
+  if (DebugLevel.getNumOccurrences()) {
+    google::LogToStderr();
+    google::SetStderrLogging(google::INFO);
+    google::SetVLOGLevel("*", DebugLevel);
+    // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0,
+    // but internally it's defined as 1 https://fburl.com/code/9fwams75
+    gflags::SetCommandLineOption("minloglevel", "0");
   }
 
   OIGenerator oigen;
 
-  oigen.setOutputPath(std::move(outputPath));
-  oigen.setConfigFilePaths(std::move(configFilePaths));
-  oigen.setSourceFileDumpPath(sourceFileDumpPath);
-  oigen.setFailIfNothingGenerated(exitCode);
-  oigen.setUsePIC(pic);
+  oigen.setConfigFilePaths(ConfigFiles |
+                           ranges::views::transform([](const auto& p) {
+                             return std::filesystem::path(p);
+                           }) |
+                           ranges::to<std::vector>());
+  if (DumpJit.getNumOccurrences())
+    oigen.setSourceFileDumpPath(DumpJit.getValue());
 
-  SymbolService symbols(primaryObject);
+  oigen.setOutputPath(OutputFile.getValue());
 
-  return oigen.generate(primaryObject, symbols);
+  oigen.setFailIfNothingGenerated(true);
+  return oigen.generate(compilations, options.getSourcePathList());
 }
diff --git a/types/string_type.toml b/types/string_type.toml
index fa4a2ae..0d2ab0d 100644
--- a/types/string_type.toml
+++ b/types/string_type.toml
@@ -35,3 +35,34 @@ void getSizeType(const %1%<T, Traits, Allocator> &container, size_t& returnArg)
     );
 }
 """
+
+traversal_func = """
+    bool sso = ((uintptr_t)container.data() <
+                (uintptr_t)(&container + sizeof(std::__cxx11::basic_string<T0>))) &&
+               ((uintptr_t)container.data() >= (uintptr_t)&container);
+
+    return returnArg.write(container.capacity())
+        .write(sso)
+        .write(container.size());
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+uint64_t capacity = std::get<ParsedData::VarInt>(d.val).value;
+el.container_stats.emplace(result::Element::ContainerStats { .capacity = capacity });
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+bool sso = std::get<ParsedData::VarInt>(d.val).value;
+if (!sso)
+  el.exclusive_size += el.container_stats->capacity * sizeof(T0);
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats->length = std::get<ParsedData::VarInt>(d.val).value;
+"""

From e431c00c9f61ec577b0b98ad1c620a1a9ef64536 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 20 Dec 2023 15:46:51 +0000
Subject: [PATCH 074/188] tbv2: implement folly::IOBuf

folly::IOBuf does not have TreeBuilder v2 container support. Add it.

The implementation is a direct clone of v1. It still lacks tests.

Test Plan:
- It codegens on a prod type.
- No runtime testing... Bad form, I know.
- Issue created to add integration tests: https://github.com/facebookexperimental/object-introspection/issues/436
---
 types/folly_iobuf_type.toml | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/types/folly_iobuf_type.toml b/types/folly_iobuf_type.toml
index c7e1043..6b88d5c 100644
--- a/types/folly_iobuf_type.toml
+++ b/types/folly_iobuf_type.toml
@@ -39,3 +39,34 @@ void getSizeType(const %1% &container, size_t& returnArg)
     SAVE_SIZE(fullCapacity);
 }
 """
+
+traversal_func = """
+// We calculate the length of all IOBufs in the chain manually.
+// IOBuf has built-in computeChainCapacity()/computeChainLength()
+// functions which do this for us. But dead code optimization in TAO
+// caused these functions to be removed which causes relocation
+// errors.
+
+std::size_t fullLength = container.length();
+std::size_t fullCapacity = container.capacity();
+for (const folly::IOBuf* current = container.next(); current != &container;
+    current = current->next()) {
+  fullLength += current->length();
+  fullCapacity += current->capacity();
+}
+
+return returnArg.write(fullCapacity).write(fullLength);
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats.emplace(result::Element::ContainerStats{ .capacity = std::get<ParsedData::VarInt>(d.val).value });
+el.exclusive_size += el.container_stats->capacity;
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats->length = std::get<ParsedData::VarInt>(d.val).value;
+"""

From 851901d6903073b9987a2ebc6c1a73081e0dbd2f Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 20 Dec 2023 15:51:42 +0000
Subject: [PATCH 075/188] tbv2: account for duplicate types when emitting name
 providers

ClangTypeParser has emitted a duplicate type for `std::allocatr<int8_t>`.
Rather than fixing this, add the same check the compiler will do for the
duplicate templates that `addNames` emits. That is, `template<>
NameProvider<Foo>` will collide if `Foo` is used twice. We can do this by
adding a set of these strings for now. If this shows up regularly it will
likely make sense to deduplicate the type graph with a deduplication pass.

Test plan:
- Fixes the issue in prod. This change is quite logical.
---
 oi/CodeGen.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 9f41c12..9614a1f 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -213,9 +213,13 @@ template <typename T>
 struct NameProvider {};
 )";
 
+  // TODO: stop types being duplicated at this point and remove this check
+  std::unordered_set<std::string_view> emittedTypes;
   for (const Type& t : typeGraph.finalTypes) {
     if (dynamic_cast<const Typedef*>(&t))
       continue;
+    if (!emittedTypes.emplace(t.name()).second)
+      continue;
 
     code += "template <> struct NameProvider<";
     code += t.name();

From 6c192f7e58f49a1354fb2fc184681d83afadffdf Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 20 Dec 2023 15:54:50 +0000
Subject: [PATCH 076/188] clangparser: provide correct kind for classes/unions

Previously ClangTypeParser assumed all RecordTypes were structs. This is fine
for structs and classes but completely incorrect for unions. Check which type
it is and give type graph the correct one.

Test plan:
- Unions static assert without this change because their size/alignment is
  wrong.
---
 oi/type_graph/ClangTypeParser.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index 5190478..2025c84 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -183,7 +183,12 @@ Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
 
   std::string name = decl->getNameAsString();
 
-  auto kind = Class::Kind::Struct;  // TODO: kind
+  auto kind = Class::Kind::Struct;
+  if (ty.isUnionType()) {
+    kind = Class::Kind::Union;
+  } else if (ty.isClassType()) {
+    kind = Class::Kind::Class;
+  }
 
   int virtuality = 0;
 

From 976e319e14ed7977d17a9931cc6f49ee02bc6054 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 20 Dec 2023 15:56:08 +0000
Subject: [PATCH 077/188] clangparser: provide alignment info for members

Unlike DWARF, the Clang AST is capable of correctly calculating the alignment
for each member. If we do this then AlignmentCalc doesn't traverse into the
member to attempt to calculate the alignment.

This check might be wrong if the field has explicit alignment. That case can be
covered when we have proper integration testing and a repro.

Test plan:
- Without this lots of static asserts occur. With this it's okay.
---
 oi/type_graph/ClangTypeParser.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index 2025c84..62b3cc2 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -294,6 +294,7 @@ void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
 
     auto& mtype = enumerateType(*qualType);
     Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
+    m.align = decl->getASTContext().getTypeAlign(qualType) / 8;
     members.push_back(m);
   }
 

From 6fe0bf0d635926673eae9ef53cba21112bd4e3b8 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 20 Dec 2023 15:57:34 +0000
Subject: [PATCH 078/188] clangparser: mark incomplete arrays as incomplete
 without failing

Attempting to complete a type which can't be completed currently fails oilgen.
For incomplete arrays, which we know are not possible to complete, return false
deliberately.

`requireCompleteType` likely needs to not fail in all cases in the future. For
now this works.

Test plan:
- `std::unique_ptr<long[]>` used to fail the generation. Now it can
  successfully codegen.
---
 oi/type_graph/ClangTypeParser.cpp | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index 62b3cc2..e665447 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -410,8 +410,16 @@ ContainerInfo* ClangTypeParser::getContainerInfo(
 namespace {
 
 bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
-  if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
-    return true;  // treat as complete
+
+  switch (ty.getTypeClass()) {
+    case clang::Type::Builtin: {
+      if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
+        return true;  // treat as complete
+      break;
+    }
+    case clang::Type::IncompleteArray:
+        return false;  // would fail completion
+  }
 
   // TODO: This is a terrible warning.
   return !sema.RequireCompleteType(

From 37eb9bc98e9017994c6ea6f7f76160443955083b Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 20 Dec 2023 15:57:34 +0000
Subject: [PATCH 079/188] clangparser: mark incomplete arrays as incomplete
 without failing

Attempting to complete a type which can't be completed currently fails oilgen.
For incomplete arrays, which we know are not possible to complete, return false
deliberately.

`requireCompleteType` likely needs to not fail in all cases in the future. For
now this works.

Test plan:
- `std::unique_ptr<long[]>` used to fail the generation. Now it can
  successfully codegen.
---
 oi/type_graph/ClangTypeParser.cpp | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index 62b3cc2..eec86af 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -410,8 +410,15 @@ ContainerInfo* ClangTypeParser::getContainerInfo(
 namespace {
 
 bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
-  if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
-    return true;  // treat as complete
+  switch (ty.getTypeClass()) {
+    case clang::Type::Builtin: {
+      if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
+        return true;  // treat as complete
+      break;
+    }
+    case clang::Type::IncompleteArray:
+      return false;  // would fail completion
+  }
 
   // TODO: This is a terrible warning.
   return !sema.RequireCompleteType(

From f09fd937eb1ea6821e252f854d7106cbfd357870 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 20 Dec 2023 16:17:02 +0000
Subject: [PATCH 080/188] clangparser: mark incomplete arrays as incomplete
 without failing

Attempting to complete a type which can't be completed currently fails oilgen.
For incomplete arrays, which we know are not possible to complete, return false
deliberately.

`requireCompleteType` likely needs to not fail in all cases in the future. For
now this works.

Test plan:
- `std::unique_ptr<long[]>` used to fail the generation. Now it can
  successfully codegen.
---
 oi/type_graph/ClangTypeParser.cpp | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index 62b3cc2..24d5835 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -410,8 +410,17 @@ ContainerInfo* ClangTypeParser::getContainerInfo(
 namespace {
 
 bool requireCompleteType(clang::Sema& sema, const clang::Type& ty) {
-  if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
-    return true;  // treat as complete
+  switch (ty.getTypeClass()) {
+    case clang::Type::Builtin: {
+      if (ty.isSpecificBuiltinType(clang::BuiltinType::Void))
+        return true;  // treat as complete
+      break;
+    }
+    case clang::Type::IncompleteArray:
+      return false;  // would fail completion
+    default:
+      break;
+  }
 
   // TODO: This is a terrible warning.
   return !sema.RequireCompleteType(

From 2a9fc0f00a71045fbffb9034aa812acfb199c567 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 22 Dec 2023 12:59:02 +0000
Subject: [PATCH 081/188] tbv2: use std::decay_t in make_field

CodeGen v2 permits template parameters to be qualified. This means that if we
call `make_field` with a template parameter it will be qualified. However, we
don't qualify the types when generating meta functions such as `NameProvider`
and `TypeHandler`. This means these qualified types don't match up with the
expected type.

Use `std::decay_t` when forwarding the type to `NameProvider` and `TypeHandler`
so they're always the base type that they were generated with.

Test Plan:
- CI
- Added a test for `std::unique_ptr<const Foo>` to exercise this. Failed
  before, passes after.
---
 oi/CodeGen.cpp                      |  6 ++--
 test/integration/std_smart_ptr.toml | 49 +++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 3 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 9614a1f..d131715 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -1072,9 +1072,9 @@ constexpr inst::Field make_field(std::string_view name) {
     sizeof(T),
     ExclusiveSizeProvider<T>::size,
     name,
-    NameProvider<T>::names,
-    TypeHandler<Ctx, T>::fields,
-    TypeHandler<Ctx, T>::processors,
+    NameProvider<std::decay_t<T>>::names,
+    TypeHandler<Ctx, std::decay_t<T>>::fields,
+    TypeHandler<Ctx, std::decay_t<T>>::processors,
   };
 }
 )";
diff --git a/test/integration/std_smart_ptr.toml b/test/integration/std_smart_ptr.toml
index 8fc66bf..b08160c 100644
--- a/test/integration/std_smart_ptr.toml
+++ b/test/integration/std_smart_ptr.toml
@@ -31,6 +31,31 @@ definitions = '''
       }
     ]
     '''
+  [cases.unique_ptr_const_uint64_empty]
+    param_types = ["std::unique_ptr<const std::uint64_t>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 8,
+        "dynamicSize": 0,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 8
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 8,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.unique_ptr_uint64_present]
     param_types = ["std::unique_ptr<std::uint64_t>&"]
     setup = "return {std::make_unique<std::uint64_t>(64)};"
@@ -181,6 +206,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.shared_ptr_const_uint64_empty]
+    param_types = ["std::shared_ptr<const std::uint64_t>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 16,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 8
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 16,
+        "exclusiveSize": 16,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.shared_ptr_uint64_present]
     param_types = ["std::shared_ptr<std::uint64_t>&"]
     setup = "return std::make_shared<std::uint64_t>(64);"

From 5984e599610a3d56895a42ffbd46195e825a2670 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 22 Dec 2023 12:59:02 +0000
Subject: [PATCH 082/188] tbv2: use std::decay_t in make_field

CodeGen v2 permits template parameters to be qualified. This means that if we
call `make_field` with a template parameter it will be qualified. However, we
don't qualify the types when generating meta functions such as `NameProvider`
and `TypeHandler`. This means these qualified types don't match up with the
expected type.

Use `std::decay_t` when forwarding the type to `NameProvider` and `TypeHandler`
so they're always the base type that they were generated with.

Test Plan:
- CI
- Added a test for `std::unique_ptr<const Foo>` to exercise this. Failed
  before, passes after.
---
 oi/CodeGen.cpp                      |  8 +--
 test/integration/std_smart_ptr.toml | 97 +++++++++++++++++++++++++++++
 types/shrd_ptr_type.toml            |  2 +-
 types/uniq_ptr_type.toml            |  2 +-
 4 files changed, 103 insertions(+), 6 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 9614a1f..72631dd 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -1060,7 +1060,7 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
     getSizeType(Ctx& ctx, const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
       JLOG("obj @");
       JLOGPTR(&t);
-      return TypeHandler<Ctx, T>::getSizeType(ctx, t, returnArg);
+      return TypeHandler<Ctx, std::decay_t<T>>::getSizeType(ctx, t, returnArg);
     }
 )";
 
@@ -1072,9 +1072,9 @@ constexpr inst::Field make_field(std::string_view name) {
     sizeof(T),
     ExclusiveSizeProvider<T>::size,
     name,
-    NameProvider<T>::names,
-    TypeHandler<Ctx, T>::fields,
-    TypeHandler<Ctx, T>::processors,
+    NameProvider<std::decay_t<T>>::names,
+    TypeHandler<Ctx, std::decay_t<T>>::fields,
+    TypeHandler<Ctx, std::decay_t<T>>::processors,
   };
 }
 )";
diff --git a/test/integration/std_smart_ptr.toml b/test/integration/std_smart_ptr.toml
index 8fc66bf..e14e398 100644
--- a/test/integration/std_smart_ptr.toml
+++ b/test/integration/std_smart_ptr.toml
@@ -31,6 +31,31 @@ definitions = '''
       }
     ]
     '''
+  [cases.unique_ptr_const_uint64_empty]
+    param_types = ["std::unique_ptr<const std::uint64_t>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 8,
+        "dynamicSize": 0,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 8
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 8,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.unique_ptr_uint64_present]
     param_types = ["std::unique_ptr<std::uint64_t>&"]
     setup = "return {std::make_unique<std::uint64_t>(64)};"
@@ -79,6 +104,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.unique_ptr_const_vector_empty]
+    param_types = ["std::unique_ptr<const std::vector<std::uint64_t>>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 8,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 24
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 8,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.unique_ptr_vector_present]
     param_types = ["std::unique_ptr<std::vector<std::uint64_t>>&"]
     setup = "return {std::make_unique<std::vector<std::uint64_t>>(std::initializer_list<std::uint64_t>({1,2,3,4,5}))};"
@@ -181,6 +230,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.shared_ptr_const_uint64_empty]
+    param_types = ["std::shared_ptr<const std::uint64_t>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 16,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 8
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 16,
+        "exclusiveSize": 16,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.shared_ptr_uint64_present]
     param_types = ["std::shared_ptr<std::uint64_t>&"]
     setup = "return std::make_shared<std::uint64_t>(64);"
@@ -232,6 +305,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.shared_ptr_const_vector_empty]
+    param_types = ["std::shared_ptr<const std::vector<std::uint64_t>>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 16,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 24
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 16,
+        "exclusiveSize": 16,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.shared_ptr_vector_present]
     param_types = ["std::shared_ptr<std::vector<std::uint64_t>>&"]
     setup = "return std::make_shared<std::vector<std::uint64_t>>(std::initializer_list<std::uint64_t>({1,2,3,4,5}));"
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index bb3a622..a7095ce 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -60,7 +60,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<Ctx, T0>::type>
+  typename TypeHandler<Ctx, std::decay_t<T0>>::type>
 """
 func = """
 #ifdef __GLIBCXX__
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index f723f80..68d20e7 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -61,7 +61,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<Ctx, T0>::type>
+  typename TypeHandler<Ctx, std::decay_t<T0>>::type>
 """
 func = """
 auto sum = std::get<ParsedData::Sum>(d.val);

From 8ab3041df9345b734edb5ea8ab887901d67b8e3b Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 22 Dec 2023 16:12:36 +0000
Subject: [PATCH 083/188] tbv2: use std::decay_t with smart pointers

CodeGen v2 permits template parameters to be qualified. This means that if we
call `make_field` with a template parameter it will be qualified. However, we
don't qualify the types when generating meta functions such as `NameProvider`
and `TypeHandler`. This means these qualified types don't match up with the
expected type.

Use `std::decay_t` when forwarding the type to `NameProvider` and `TypeHandler`
so they're always the base type that they were generated with. Most of this is
covered by `make_field`, but there are direct references to `TypeHandler<Ctx,
T>` in a lot of `TypeHandler::type` fields. Fix the problematic types manually
for now, there may need to be a better solution with meta functions for this in
the future.

Test Plan:
- CI
- Added a test for `std::unique_ptr<const uint64_t>` to exercise this. Failed
  before, passes after.
- Added a test for `std::unique_ptr<const std::
---
 oi/CodeGen.cpp                      |  8 +--
 test/integration/std_smart_ptr.toml | 97 +++++++++++++++++++++++++++++
 types/shrd_ptr_type.toml            |  2 +-
 types/uniq_ptr_type.toml            |  2 +-
 4 files changed, 103 insertions(+), 6 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 9614a1f..72631dd 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -1060,7 +1060,7 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
     getSizeType(Ctx& ctx, const T &t, typename TypeHandler<Ctx, T>::type returnArg) {
       JLOG("obj @");
       JLOGPTR(&t);
-      return TypeHandler<Ctx, T>::getSizeType(ctx, t, returnArg);
+      return TypeHandler<Ctx, std::decay_t<T>>::getSizeType(ctx, t, returnArg);
     }
 )";
 
@@ -1072,9 +1072,9 @@ constexpr inst::Field make_field(std::string_view name) {
     sizeof(T),
     ExclusiveSizeProvider<T>::size,
     name,
-    NameProvider<T>::names,
-    TypeHandler<Ctx, T>::fields,
-    TypeHandler<Ctx, T>::processors,
+    NameProvider<std::decay_t<T>>::names,
+    TypeHandler<Ctx, std::decay_t<T>>::fields,
+    TypeHandler<Ctx, std::decay_t<T>>::processors,
   };
 }
 )";
diff --git a/test/integration/std_smart_ptr.toml b/test/integration/std_smart_ptr.toml
index 8fc66bf..e14e398 100644
--- a/test/integration/std_smart_ptr.toml
+++ b/test/integration/std_smart_ptr.toml
@@ -31,6 +31,31 @@ definitions = '''
       }
     ]
     '''
+  [cases.unique_ptr_const_uint64_empty]
+    param_types = ["std::unique_ptr<const std::uint64_t>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 8,
+        "dynamicSize": 0,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 8
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 8,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.unique_ptr_uint64_present]
     param_types = ["std::unique_ptr<std::uint64_t>&"]
     setup = "return {std::make_unique<std::uint64_t>(64)};"
@@ -79,6 +104,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.unique_ptr_const_vector_empty]
+    param_types = ["std::unique_ptr<const std::vector<std::uint64_t>>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 8,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 24
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 8,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.unique_ptr_vector_present]
     param_types = ["std::unique_ptr<std::vector<std::uint64_t>>&"]
     setup = "return {std::make_unique<std::vector<std::uint64_t>>(std::initializer_list<std::uint64_t>({1,2,3,4,5}))};"
@@ -181,6 +230,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.shared_ptr_const_uint64_empty]
+    param_types = ["std::shared_ptr<const std::uint64_t>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 16,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 8
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 16,
+        "exclusiveSize": 16,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.shared_ptr_uint64_present]
     param_types = ["std::shared_ptr<std::uint64_t>&"]
     setup = "return std::make_shared<std::uint64_t>(64);"
@@ -232,6 +305,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.shared_ptr_const_vector_empty]
+    param_types = ["std::shared_ptr<const std::vector<std::uint64_t>>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 16,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 24
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 16,
+        "exclusiveSize": 16,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.shared_ptr_vector_present]
     param_types = ["std::shared_ptr<std::vector<std::uint64_t>>&"]
     setup = "return std::make_shared<std::vector<std::uint64_t>>(std::initializer_list<std::uint64_t>({1,2,3,4,5}));"
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index bb3a622..a7095ce 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -60,7 +60,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<Ctx, T0>::type>
+  typename TypeHandler<Ctx, std::decay_t<T0>>::type>
 """
 func = """
 #ifdef __GLIBCXX__
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index f723f80..68d20e7 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -61,7 +61,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<Ctx, T0>::type>
+  typename TypeHandler<Ctx, std::decay_t<T0>>::type>
 """
 func = """
 auto sum = std::get<ParsedData::Sum>(d.val);

From 2b49e6aa3bc8f31fb2953c02283cf0e2187ec104 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 22 Dec 2023 16:12:36 +0000
Subject: [PATCH 084/188] tbv2: use std::decay_t with smart pointers

CodeGen v2 permits template parameters to be qualified. This means that if we
call `make_field` with a template parameter it will be qualified. However, we
don't qualify the types when generating meta functions such as `NameProvider`
and `TypeHandler`. This means these qualified types don't match up with the
expected type.

Use `std::decay_t` when forwarding the type to `NameProvider` and `TypeHandler`
so they're always the base type that they were generated with. Most of this is
covered by `make_field`, but there are direct references to `TypeHandler<Ctx,
T>` in a lot of `TypeHandler::type` fields. Fix the problematic types manually
for now, there may need to be a better solution with meta functions for this in
the future.

Test Plan:
- CI
- Added a test for `std::unique_ptr<const uint64_t>` to exercise this. Failed
  before, passes after.
- Added a test for `std::unique_ptr<const std::
---
 oi/CodeGen.cpp                      |  6 +-
 test/integration/std_smart_ptr.toml | 97 +++++++++++++++++++++++++++++
 types/shrd_ptr_type.toml            |  2 +-
 types/uniq_ptr_type.toml            |  2 +-
 4 files changed, 102 insertions(+), 5 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 9614a1f..d131715 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -1072,9 +1072,9 @@ constexpr inst::Field make_field(std::string_view name) {
     sizeof(T),
     ExclusiveSizeProvider<T>::size,
     name,
-    NameProvider<T>::names,
-    TypeHandler<Ctx, T>::fields,
-    TypeHandler<Ctx, T>::processors,
+    NameProvider<std::decay_t<T>>::names,
+    TypeHandler<Ctx, std::decay_t<T>>::fields,
+    TypeHandler<Ctx, std::decay_t<T>>::processors,
   };
 }
 )";
diff --git a/test/integration/std_smart_ptr.toml b/test/integration/std_smart_ptr.toml
index 8fc66bf..e14e398 100644
--- a/test/integration/std_smart_ptr.toml
+++ b/test/integration/std_smart_ptr.toml
@@ -31,6 +31,31 @@ definitions = '''
       }
     ]
     '''
+  [cases.unique_ptr_const_uint64_empty]
+    param_types = ["std::unique_ptr<const std::uint64_t>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 8,
+        "dynamicSize": 0,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 8
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 8,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.unique_ptr_uint64_present]
     param_types = ["std::unique_ptr<std::uint64_t>&"]
     setup = "return {std::make_unique<std::uint64_t>(64)};"
@@ -79,6 +104,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.unique_ptr_const_vector_empty]
+    param_types = ["std::unique_ptr<const std::vector<std::uint64_t>>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 8,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 24
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 8,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.unique_ptr_vector_present]
     param_types = ["std::unique_ptr<std::vector<std::uint64_t>>&"]
     setup = "return {std::make_unique<std::vector<std::uint64_t>>(std::initializer_list<std::uint64_t>({1,2,3,4,5}))};"
@@ -181,6 +230,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.shared_ptr_const_uint64_empty]
+    param_types = ["std::shared_ptr<const std::uint64_t>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 16,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 8
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 16,
+        "exclusiveSize": 16,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.shared_ptr_uint64_present]
     param_types = ["std::shared_ptr<std::uint64_t>&"]
     setup = "return std::make_shared<std::uint64_t>(64);"
@@ -232,6 +305,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.shared_ptr_const_vector_empty]
+    param_types = ["std::shared_ptr<const std::vector<std::uint64_t>>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 16,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 24
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 16,
+        "exclusiveSize": 16,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.shared_ptr_vector_present]
     param_types = ["std::shared_ptr<std::vector<std::uint64_t>>&"]
     setup = "return std::make_shared<std::vector<std::uint64_t>>(std::initializer_list<std::uint64_t>({1,2,3,4,5}));"
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index bb3a622..a7095ce 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -60,7 +60,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<Ctx, T0>::type>
+  typename TypeHandler<Ctx, std::decay_t<T0>>::type>
 """
 func = """
 #ifdef __GLIBCXX__
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index f723f80..68d20e7 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -61,7 +61,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<Ctx, T0>::type>
+  typename TypeHandler<Ctx, std::decay_t<T0>>::type>
 """
 func = """
 auto sum = std::get<ParsedData::Sum>(d.val);

From 5887b3a8dcdbf779fe1fbc312ab3d21533c32222 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 22 Dec 2023 16:32:13 +0000
Subject: [PATCH 085/188] type_graph: avoid overwriting explicitly set
 alignment

Previously AlignmentCalc calculates the alignment and sets packing for every
type except a member with explicit alignment. Change this to check whether an
alignment has been previously set for a type before calculating it. Use this in
ClangTypeParser where the full alignment of the type is available.

Remove explicitly aligning members by the type because that was previously
reserved for members with explicit alignment. AlignmentCalc will correctly
align a member to the underlying type without this. Explicit member alignment
is still missing, as before this change.

Test plan:
- CI
- Too little. Gets further into a production type than without this change.
---
 oi/type_graph/AlignmentCalc.cpp   | 30 ++++++++++++++++--------------
 oi/type_graph/ClangTypeParser.cpp |  4 ++--
 2 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/oi/type_graph/AlignmentCalc.cpp b/oi/type_graph/AlignmentCalc.cpp
index cf23eff..9abad71 100644
--- a/oi/type_graph/AlignmentCalc.cpp
+++ b/oi/type_graph/AlignmentCalc.cpp
@@ -50,24 +50,26 @@ void AlignmentCalc::accept(Type& type) {
 void AlignmentCalc::visit(Class& c) {
   RecursiveVisitor::visit(c);
 
-  uint64_t alignment = 1;
-  for (auto& member : c.members) {
-    if (member.align == 0) {
-      // If the member does not have an explicit alignment, calculate it from
-      // the member's type.
-      accept(member.type());
-      member.align = member.type().align();
-    }
-    alignment = std::max(alignment, member.align);
+  if (c.align() == 0) {
+    uint64_t alignment = 1;
+    for (auto& member : c.members) {
+      if (member.align == 0) {
+        // If the member does not have an explicit alignment, calculate it from
+        // the member's type.
+        accept(member.type());
+        member.align = member.type().align();
+      }
+      alignment = std::max(alignment, member.align);
 
-    if (member.align != 0 && (member.bitOffset / 8) % member.align != 0) {
-      // Mark as packed if members are not aligned
-      c.setPacked();
+      if (member.align != 0 && (member.bitOffset / 8) % member.align != 0) {
+        // Mark as packed if members are not aligned
+        c.setPacked();
+      }
     }
+
+    c.setAlign(alignment);
   }
 
-  c.setAlign(alignment);
-
   if (c.size() % c.align() != 0) {
     // Mark as packed if there is no tail padding
     c.setPacked();
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index 24d5835..3a13903 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -175,7 +175,7 @@ Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
   if (auto* info = getContainerInfo(fqName)) {
     auto& c = makeType<Container>(ty, *info, size, nullptr);
     enumerateClassTemplateParams(ty, c.templateParams);
-    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)));
+    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)) / 8);
     return c;
   }
 
@@ -194,6 +194,7 @@ Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
 
   auto& c = makeType<Class>(
       ty, kind, std::move(name), std::move(fqName), size, virtuality);
+  c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)) / 8);
 
   enumerateClassTemplateParams(ty, c.templateParams);
   // enumerateClassParents(type, c.parents);
@@ -294,7 +295,6 @@ void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
 
     auto& mtype = enumerateType(*qualType);
     Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
-    m.align = decl->getASTContext().getTypeAlign(qualType) / 8;
     members.push_back(m);
   }
 

From b5756d2311d4c3b27c58366a6d711c32e33636bd Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 22 Dec 2023 17:14:58 +0000
Subject: [PATCH 086/188] tbv2: use std::decay_t with smart pointers

CodeGen v2 permits template parameters to be qualified. This means that if we
call `make_field` with a template parameter it will be qualified. However, we
don't qualify the types when generating meta functions such as `NameProvider`
and `TypeHandler`. This means these qualified types don't match up with the
expected type.

Use `std::decay_t` when forwarding the type to `NameProvider` and `TypeHandler`
so they're always the base type that they were generated with. Most of this is
covered by `make_field`, but there are direct references to `TypeHandler<Ctx,
T>` in a lot of `TypeHandler::type` fields. Fix the problematic types manually
for now, there may need to be a better solution with meta functions for this in
the future.

Test Plan:
- CI
- Added a test for `std::unique_ptr<const uint64_t>` to exercise this. Failed
  before, passes after.
- Added a test for `std::unique_ptr<const std::vector<uint64_t>>` to test a
  non-primitive type. Failed before, passes after.
---
 oi/CodeGen.cpp                      |  6 +-
 test/integration/std_smart_ptr.toml | 97 +++++++++++++++++++++++++++++
 types/shrd_ptr_type.toml            |  2 +-
 types/uniq_ptr_type.toml            |  2 +-
 4 files changed, 102 insertions(+), 5 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 9614a1f..d131715 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -1072,9 +1072,9 @@ constexpr inst::Field make_field(std::string_view name) {
     sizeof(T),
     ExclusiveSizeProvider<T>::size,
     name,
-    NameProvider<T>::names,
-    TypeHandler<Ctx, T>::fields,
-    TypeHandler<Ctx, T>::processors,
+    NameProvider<std::decay_t<T>>::names,
+    TypeHandler<Ctx, std::decay_t<T>>::fields,
+    TypeHandler<Ctx, std::decay_t<T>>::processors,
   };
 }
 )";
diff --git a/test/integration/std_smart_ptr.toml b/test/integration/std_smart_ptr.toml
index 8fc66bf..e14e398 100644
--- a/test/integration/std_smart_ptr.toml
+++ b/test/integration/std_smart_ptr.toml
@@ -31,6 +31,31 @@ definitions = '''
       }
     ]
     '''
+  [cases.unique_ptr_const_uint64_empty]
+    param_types = ["std::unique_ptr<const std::uint64_t>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 8,
+        "dynamicSize": 0,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 8
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 8,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.unique_ptr_uint64_present]
     param_types = ["std::unique_ptr<std::uint64_t>&"]
     setup = "return {std::make_unique<std::uint64_t>(64)};"
@@ -79,6 +104,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.unique_ptr_const_vector_empty]
+    param_types = ["std::unique_ptr<const std::vector<std::uint64_t>>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 8,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 24
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 8,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.unique_ptr_vector_present]
     param_types = ["std::unique_ptr<std::vector<std::uint64_t>>&"]
     setup = "return {std::make_unique<std::vector<std::uint64_t>>(std::initializer_list<std::uint64_t>({1,2,3,4,5}))};"
@@ -181,6 +230,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.shared_ptr_const_uint64_empty]
+    param_types = ["std::shared_ptr<const std::uint64_t>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 16,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 8
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 16,
+        "exclusiveSize": 16,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.shared_ptr_uint64_present]
     param_types = ["std::shared_ptr<std::uint64_t>&"]
     setup = "return std::make_shared<std::uint64_t>(64);"
@@ -232,6 +305,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.shared_ptr_const_vector_empty]
+    param_types = ["std::shared_ptr<const std::vector<std::uint64_t>>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 16,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 24
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 16,
+        "exclusiveSize": 16,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.shared_ptr_vector_present]
     param_types = ["std::shared_ptr<std::vector<std::uint64_t>>&"]
     setup = "return std::make_shared<std::vector<std::uint64_t>>(std::initializer_list<std::uint64_t>({1,2,3,4,5}));"
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index bb3a622..a7095ce 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -60,7 +60,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<Ctx, T0>::type>
+  typename TypeHandler<Ctx, std::decay_t<T0>>::type>
 """
 func = """
 #ifdef __GLIBCXX__
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index f723f80..68d20e7 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -61,7 +61,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<Ctx, T0>::type>
+  typename TypeHandler<Ctx, std::decay_t<T0>>::type>
 """
 func = """
 auto sum = std::get<ParsedData::Sum>(d.val);

From 460799bf74af49d8cd6e147611746f6e6fd5006b Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 22 Dec 2023 18:10:23 +0000
Subject: [PATCH 087/188] clangparser: add support for parents

TODO: Check the assumption that a "base" always has a Type that can be cast to
RecordType.
TODO: Check the assumption that a "base" always has a Decl that can be cast to
CXXRecordDecl.

Add basic support for class parents. Focus purely on bases for now and ignore
vbases.

Test Plan:
- Tested with a simple example. Base containing a long and a class containing a
  float. Both fields appear in the final flattened code.
---
 oi/type_graph/ClangTypeParser.cpp | 32 ++++++++++++++++++++++++++++++-
 oi/type_graph/ClangTypeParser.h   |  2 ++
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index 24d5835..5919ac2 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -17,8 +17,10 @@
 
 #include <clang/AST/ASTContext.h>
 #include <clang/AST/Decl.h>
+#include <clang/AST/DeclCXX.h>
 #include <clang/AST/DeclTemplate.h>
 #include <clang/AST/QualTypeNames.h>
+#include <clang/AST/RecordLayout.h>
 #include <clang/AST/Type.h>
 #include <clang/Basic/DiagnosticSema.h>
 #include <clang/Sema/Sema.h>
@@ -196,7 +198,7 @@ Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
       ty, kind, std::move(name), std::move(fqName), size, virtuality);
 
   enumerateClassTemplateParams(ty, c.templateParams);
-  // enumerateClassParents(type, c.parents);
+  enumerateClassParents(ty, c.parents);
   enumerateClassMembers(ty, c.members);
   // enumerateClassFunctions(type, c.functions);
 
@@ -275,6 +277,34 @@ std::optional<TemplateParam> ClangTypeParser::enumerateTemplateTemplateParam(
   }
 }
 
+void ClangTypeParser::enumerateClassParents(const clang::RecordType& ty,
+                                            std::vector<Parent>& parents) {
+  assert(parents.empty());
+
+  auto* decl = ty.getDecl();
+  auto* cxxDecl = llvm::dyn_cast<clang::CXXRecordDecl>(decl);
+  if (cxxDecl == nullptr)
+    return;
+
+  const auto& layout = decl->getASTContext().getASTRecordLayout(decl);
+  for (const auto& base : cxxDecl->bases()) {
+    auto baseType = base.getType();
+    const auto* baseRecordType = llvm::dyn_cast<clang::RecordType>(&*baseType);
+    if (baseRecordType == nullptr)
+      continue;
+
+    auto* baseDecl = baseRecordType->getDecl();
+    auto* baseCxxDecl = llvm::dyn_cast<clang::CXXRecordDecl>(baseDecl);
+    if (baseCxxDecl == nullptr)
+      continue;
+
+    auto offset = layout.getBaseClassOffset(baseCxxDecl).getQuantity();
+    auto& ptype = enumerateType(*baseType);
+    Parent p{ptype, static_cast<uint64_t>(offset)};
+    parents.push_back(p);
+  }
+}
+
 void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
                                             std::vector<Member>& members) {
   assert(members.empty());
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
index 9b27d64..98f0f67 100644
--- a/oi/type_graph/ClangTypeParser.h
+++ b/oi/type_graph/ClangTypeParser.h
@@ -50,6 +50,7 @@ class Array;
 class Class;
 class Enum;
 class Member;
+struct Parent;
 class Primitive;
 class Reference;
 class Type;
@@ -114,6 +115,7 @@ class ClangTypeParser {
   std::optional<TemplateParam> enumerateTemplateTemplateParam(
       const clang::TemplateName&);
 
+  void enumerateClassParents(const clang::RecordType&, std::vector<Parent>&);
   void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
 
   ContainerInfo* getContainerInfo(const std::string& fqName) const;

From 25490e493917a955f3c246e7f9f7a39245120892 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 28 Dec 2023 16:47:19 +0000
Subject: [PATCH 088/188] oilgen: add to integration test framework

TODO: Replace the references to local paths.

oilgen (the basis of Ahead Of Time compilation for OIL) has never been passed
through our large test suite and has instead had more focused testing and large
examples. When consuming DWARF information in a similar fashion to JIT OIL this
was okay, but with the new Clang AST based mechanism it means we have very
little coverage.

This change adds an oilgen test for every test case that has an oil test.
Relying on the build system to create the test target as before would make it
difficult to have failing tests, so we move the build into the integration test
runner. This involves:
1. Writing the input source to a file.
2. Consuming it with oilgen to get the implementation object file.
3. Compiling the input source and linking it with this file.
4. Running the newly created target.

This approach can give the full error message at any stage that fails and will
fail the test appropriately. The downside is the build system integration is
more difficult, as we need the correct compiler flags for the target and to use
the correct compiler. It would be very tricky to replicate this in a build
system that's not CMake, so we will likely only run these tests in open source.

Test plan:
- CI
---
 test/integration/CMakeLists.txt    |   5 +-
 test/integration/gen_tests.py      | 149 ++++++++++++++++++++++------
 test/integration/runner_common.cpp | 150 +++++++++++++++++++++++++++++
 test/integration/runner_common.h   |  12 +++
 4 files changed, 283 insertions(+), 33 deletions(-)

diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 12cb239..42346e0 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -63,7 +63,10 @@ target_link_libraries(integration_test_runner PRIVATE
 target_compile_definitions(integration_test_runner PRIVATE
   TARGET_EXE_PATH="${CMAKE_CURRENT_BINARY_DIR}/integration_test_target"
   OID_EXE_PATH="$<TARGET_FILE:oid>"
-  CONFIG_FILE_PATH="${CMAKE_BINARY_DIR}/testing.oid.toml")
+  OILGEN_EXE_PATH="$<TARGET_FILE:oilgen>"
+  CONFIG_FILE_PATH="${CMAKE_BINARY_DIR}/testing.oid.toml"
+  CXX="${CMAKE_CXX_COMPILER}"
+)
 
 if (${THRIFT_FOUND})
   foreach(THRIFT_TEST IN LISTS THRIFT_TESTS)
diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py
index 2dbc3ff..d366a48 100644
--- a/test/integration/gen_tests.py
+++ b/test/integration/gen_tests.py
@@ -50,6 +50,41 @@ def add_headers(f, custom_headers, thrift_headers):
         f.write(f'#include "{header}"\n')
 
 
+def add_test_getters(f, case_name, case):
+    param_types = ", ".join(
+        f"std::remove_cvref_t<{param}>" for param in case["param_types"]
+    )
+    if "arg_types" in case:
+        arg_types = ", ".join(case["arg_types"])
+    else:
+        arg_types = param_types
+
+    f.write(
+        f"\n"
+        f"  std::tuple<{arg_types}> get_{case_name}() {{\n"
+        f'{case["setup"]}\n'
+        f"  }}\n"
+    )
+
+
+def get_param_str(param, i):
+    if "]" in param:
+        # Array param
+
+        if ")" in param:
+            # "int(&)[5]" ->  "int (&a0)[5]"
+            start, end = param.split(")")
+            return f"{start}a{i}){end}"
+
+        # "int[5]" -> "int a0[5]"
+        # "int[5][10]" -> "int a0[5][10]"
+        type_name, array_size = param.split("[", 1)
+        return f"{type_name} a{i}[{array_size}"
+
+    # Non-array param, e.g. "int&" -> "int& a0"
+    return f"{param} a{i}"
+
+
 def add_test_setup(f, config):
     ns = get_namespace(config["suite"])
     # fmt: off
@@ -64,23 +99,6 @@ def add_test_setup(f, config):
     )
     # fmt: on
 
-    def get_param_str(param, i):
-        if "]" in param:
-            # Array param
-
-            if ")" in param:
-                # "int(&)[5]" ->  "int (&a0)[5]"
-                start, end = param.split(")")
-                return f"{start}a{i}){end}"
-
-            # "int[5]" -> "int a0[5]"
-            # "int[5][10]" -> "int a0[5][10]"
-            type_name, array_size = param.split("[", 1)
-            return f"{type_name} a{i}[{array_size}"
-
-        # Non-array param, e.g. "int&" -> "int& a0"
-        return f"{param} a{i}"
-
     def define_traceable_func(name, params, body):
         return (
             f"\n"
@@ -98,21 +116,7 @@ def add_test_setup(f, config):
             # target func for it
             continue
 
-        # generate getter for an object of this type
-        param_types = ", ".join(
-            f"std::remove_cvref_t<{param}>" for param in case["param_types"]
-        )
-        if "arg_types" in case:
-            arg_types = ", ".join(case["arg_types"])
-        else:
-            arg_types = param_types
-
-        f.write(
-            f"\n"
-            f"  std::tuple<{arg_types}> get_{case_name}() {{\n"
-            f'{case["setup"]}\n'
-            f"  }}\n"
-        )
+        add_test_getters(f, case_name, case)
 
         # generate oid and oil targets
         params_str = ", ".join(
@@ -265,6 +269,7 @@ def add_tests(f, config):
     for case_name, case in config["cases"].items():
         add_oid_integration_test(f, config, case_name, case)
         add_oil_integration_test(f, config, case_name, case)
+        add_oilgen_integration_test(f, config, case_name, case)
 
 
 def add_oid_integration_test(f, config, case_name, case):
@@ -399,6 +404,86 @@ def add_oil_integration_test(f, config, case_name, case):
     f.write(f"}}\n")
 
 
+def add_oilgen_integration_test(f, config, case_name, case):
+    case_str = get_case_name(config["suite"], case_name)
+    exit_code = case.get("expect_oil_exit_code", 0)
+
+    if "oil_disable" in case or "target_function" in case:
+        return
+
+    config_prefix = case.get("config_prefix", "")
+    config_suffix = case.get("config_suffix", "")
+
+    f.write(
+        f"\n"
+        f"TEST_F(OilgenIntegration, {case_str}) {{\n"
+        f"{generate_skip(case, 'oil')}"
+    )
+
+    f.write('  constexpr std::string_view targetSrc = R"--(')
+    headers = set(config.get("includes", []))
+    add_headers(f, sorted(headers), [])
+
+    f.write(
+        f"\n"
+        f'{config.get("raw_definitions", "")}\n'
+        f"#pragma clang diagnostic push\n"
+        f"#pragma clang diagnostic ignored \"-Wunused-private-field\"\n"
+        f'{config.get("definitions", "")}\n'
+        f"#pragma clang diagnostic pop\n"
+    )
+    add_test_getters(f, case_name, case)
+
+    main = "int main() {\n"
+    main += "  auto pr = oi::exporters::Json(std::cout);\n"
+    main += "  pr.setPretty(true);\n"
+    main += f"  auto val = get_{case_name}();\n"
+    for i in range(len(case["param_types"])):
+        main += f"  auto ret{i} = oi::introspect(std::get<{i}>(val));\n"
+        main += f"  pr.print(ret{i});\n"
+    main += "}\n"
+
+    f.write(main)
+    f.write(')--";\n')
+
+    f.write(
+        f'  std::string configPrefix = R"--({config_prefix})--";\n'
+        f'  std::string configSuffix = R"--({config_suffix})--";\n'
+        f"  ba::io_context ctx;\n"
+        f"  auto target = runOilgenTarget({{\n"
+        f"    .ctx = ctx,\n"
+        f"    .targetSrc = targetSrc,\n"
+        f"  }}, std::move(configPrefix), std::move(configSuffix));\n\n"
+        f"  ASSERT_EQ(exit_code(target), {exit_code});\n"
+    )
+
+    key = "expect_json"
+    if "expect_json_v2" in case:
+        key = "expect_json_v2"
+    if key in case:
+        try:
+            json.loads(case[key])
+        except json.decoder.JSONDecodeError as error:
+            print(
+                f"\x1b[31m`expect_json` value for test case {config['suite']}.{case_name} was invalid JSON: {error}\x1b[0m",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+        f.write(
+            f"\n"
+            f"  std::stringstream expected_json_ss;\n"
+            f'  expected_json_ss << R"--({case[key]})--";\n'
+            f"  auto result_json_ss = std::stringstream(stdout_);\n"
+            f"  bpt::ptree expected_json, actual_json;\n"
+            f"  bpt::read_json(expected_json_ss, expected_json);\n"
+            f"  bpt::read_json(result_json_ss, actual_json);\n"
+            f"  compare_json(expected_json, actual_json);\n"
+        )
+
+    f.write(f"}}\n")
+
+
 def generate_skip(case, specific):
     possibly_skip = ""
     skip_reason = case.get("skip", False)
diff --git a/test/integration/runner_common.cpp b/test/integration/runner_common.cpp
index c86bd59..9a73158 100644
--- a/test/integration/runner_common.cpp
+++ b/test/integration/runner_common.cpp
@@ -445,3 +445,153 @@ Proc OilIntegration::runOilTarget(OilOpts opts,
               std::move(std_out),
               std::move(std_err)};
 }
+
+std::string OilgenIntegration::TmpDirStr() {
+  return std::string("/tmp/oilgen-integration-XXXXXX");
+}
+
+Proc OilgenIntegration::runOilgenTarget(OilgenOpts opts,
+                                  std::string configPrefix,
+                                  std::string configSuffix) {
+  // Run an oilgen test in three stages.
+  // 1. Generate the OIL implementation .o from the input source.
+  // 2. Compile the input source and link to the implementation.
+  // 3. Run the now complete target.
+
+  auto path = workingDir / "input_src.cpp";
+  {
+    std::ofstream file{path, std::ios_base::app};
+    file << opts.targetSrc;
+  }
+
+  std::string generatorExe = std::string(OILGEN_EXE_PATH) + " " + path.native() + " ";
+  if (auto prefix = writeCustomConfig("prefix", configPrefix)) {
+    generatorExe += "--config-file ";
+    generatorExe += *prefix;
+    generatorExe += " ";
+  }
+  generatorExe += "--config-file ";
+  generatorExe += configFile;
+  if (auto suffix = writeCustomConfig("suffix", configSuffix)) {
+    generatorExe += " ";
+    generatorExe += "--config-file ";
+    generatorExe += *suffix;
+  }
+
+  // TODO: get this from the CMake arguments for integration_test_target.cpp somehow
+  std::vector<std::string> clangArgs{
+      "-DOIL_AOT_COMPILATION=1",
+      "-isystem",
+      "/data/users/jakehillion/object-introspection-sl/include",
+      "--std=c++20",
+      "-resource-dir",
+      "/usr/lib64/clang/15.0.7",
+      "-fpic",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../include/c++/12",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../include/c++/12/x86_64-redhat-linux",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../include/c++/12/backward",
+      "-isystem",
+      "/usr/lib64/clang/15.0.7/include",
+      "-isystem",
+      "/usr/local/include",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../x86_64-redhat-linux/include",
+      "-isystem",
+      "/include",
+      "-isystem",
+      "/usr/include",
+      "-fdebug-compilation-dir=/data/users/jakehillion/Downloads/libclang-oilgen-testing",
+      "-fgnuc-version=4.2.1",
+      "-fcxx-exceptions",
+      "-fexceptions",
+      "-faddrsig",
+      "-D__GCC_HAVE_DWARF2_CFI_ASM=1",
+      "-x",
+      "c++",
+  };
+  for (const auto& arg : clangArgs) {
+    generatorExe += " --extra-arg=";
+    generatorExe += arg;
+  }
+
+  if (verbose) {
+    std::cerr << "Running: " << generatorExe << std::endl;
+  }
+
+  bp::child generatorProc{generatorExe, opts.ctx};
+  generatorProc.wait();
+  if (generatorProc.exit_code() != 0)
+    throw std::runtime_error("generation failed!");
+
+  std::string compilerExe = std::string(CXX) + " a.o input_src.cpp /data/users/jakehillion/object-introspection-sl/oi/exporters/Json.cpp /data/users/jakehillion/object-introspection-sl/oi/IntrospectionResult.cpp /data/users/jakehillion/object-introspection-sl/oi/exporters/ParsedData.cpp";
+  for (const auto& arg : clangArgs) {
+    compilerExe += ' ';
+    compilerExe += arg;
+  }
+
+  if (verbose) {
+    std::cerr << "Running: " << compilerExe << std::endl;
+  }
+
+  bp::child compilerProc{compilerExe, opts.ctx};
+  compilerProc.wait();
+  if (compilerProc.exit_code() != 0)
+    throw std::runtime_error("compilation failed");
+
+  std::string targetExe = "./a.out";
+  if (verbose) {
+    std::cerr << "Running: " << targetExe << std::endl;
+  }
+
+  // Use tee to write the output to files. If verbose is on, also redirect the
+  // output to stderr.
+  bp::async_pipe std_out_pipe(opts.ctx), std_err_pipe(opts.ctx);
+  bp::child std_out, std_err;
+  if (verbose) {
+    // clang-format off
+    std_out = bp::child(bp::search_path("tee"),
+                        (workingDir / "stdout").string(),
+                        bp::std_in < std_out_pipe,
+                        bp::std_out > stderr,
+                        opts.ctx);
+    std_err = bp::child(bp::search_path("tee"),
+                        (workingDir / "stderr").string(),
+                        bp::std_in < std_err_pipe,
+                        bp::std_out > stderr,
+                        opts.ctx);
+    // clang-format on
+  } else {
+    // clang-format off
+    std_out = bp::child(bp::search_path("tee"),
+                        (workingDir / "stdout").string(),
+                        bp::std_in < std_out_pipe,
+                        bp::std_out > bp::null,
+                        opts.ctx);
+    std_err = bp::child(bp::search_path("tee"),
+                        (workingDir / "stderr").string(),
+                        bp::std_in < std_err_pipe,
+                        bp::std_out > bp::null,
+                        opts.ctx);
+    // clang-format on
+  }
+
+  /* Spawn `oid` with tracing on and IOs redirected */
+  // clang-format off
+  bp::child targetProc(
+      targetExe,
+      bp::std_in  < bp::null,
+      bp::std_out > std_out_pipe,
+      bp::std_err > std_err_pipe,
+      opts.ctx);
+  // clang-format on
+
+  return Proc{
+    opts.ctx,
+    std::move(targetProc),
+    std::move(std_out),
+    std::move(std_err),
+  };
+}
diff --git a/test/integration/runner_common.h b/test/integration/runner_common.h
index d2b5e57..0da42d4 100644
--- a/test/integration/runner_common.h
+++ b/test/integration/runner_common.h
@@ -22,6 +22,11 @@ struct OilOpts {
   std::string targetArgs;
 };
 
+struct OilgenOpts {
+  boost::asio::io_context& ctx;
+  std::string_view targetSrc;
+};
+
 struct Proc {
   boost::asio::io_context& ctx;
   boost::process::child proc;
@@ -81,3 +86,10 @@ class OilIntegration : public IntegrationBase {
                     std::string configPrefix,
                     std::string configSuffix);
 };
+
+class OilgenIntegration : public IntegrationBase {
+ protected:
+  std::string TmpDirStr() override;
+
+  Proc runOilgenTarget(OilgenOpts opts, std::string configPrefix, std::string configSuffix);
+};

From d7b1d4954eb47098e11df5297e8b3cab4dec3a5d Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 29 Dec 2023 10:30:53 +0000
Subject: [PATCH 089/188] circleci: run oid in legacy tests

Currently the CodeGen v1 CI tests are filtered to only run OIL tests. This is
incorrect, as OIL always runs with CodeGen v2. Change it so only OID tests are
run as legacy.

Also restructured the CI file a bit to group the build steps together and the
test steps together. I can revert this if it's inconvenient but I think it
makes it clearer.

Test plan:
- Checked the CI and it's running OID tests now
---
 .circleci/config.yml | 50 ++++++++++++++++++++++++--------------------
 1 file changed, 27 insertions(+), 23 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 0a75d99..f3ec2be 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -5,23 +5,48 @@ workflows:
     jobs:
       - lint
 
+      # Build
       - build:
           name: build-gcc
           cc: /usr/bin/gcc
           cxx: /usr/bin/g++
           warnings_as_errors: "OFF"
+      - build:
+          name: build-clang
+          cc: /usr/bin/clang-12
+          cxx: /usr/bin/clang++-12
+          warnings_as_errors: "ON"
+
+      # Legacy tests
       - test:
           name: test-codegenv1-gcc
           requires:
             - build-gcc
           oid_test_args: "-Ftype-graph"
-          tests_regex: "OilIntegration\\..*"
+          tests_regex: "OidIntegration\\..*"
+      - test:
+          name: test-codegenv1-clang
+          requires:
+            - build-clang
+          oid_test_args: "-Ftype-graph"
+          tests_regex: "OidIntegration\\..*"
+          # Tests disabled due to bad DWARF generated by the old clang compiler in CI
+          exclude_regex: "OilIntegration.fbstring_.*|OilIntegration.capture_keys_string|OilIntegration.capture_keys_multi_level"
+
+      # Current tests
       - test:
           name: test-gcc
           requires:
             - build-gcc
-          tests_regex: "OidIntegration\\..*"
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
+      - test:
+          name: test-clang
+          requires:
+            - build-clang
+          # Tests disabled due to bad DWARF generated by the old clang compiler in CI
+          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_a"
+
+      # Coverage
       - coverage:
           name: coverage
           requires:
@@ -31,27 +56,6 @@ workflows:
           requires:
             - test-codegenv1-gcc
 
-      - build:
-          name: build-clang
-          cc: /usr/bin/clang-12
-          cxx: /usr/bin/clang++-12
-          warnings_as_errors: "ON"
-      - test:
-          name: test-codegenv1-clang
-          requires:
-            - build-clang
-          oid_test_args: "-Ftype-graph"
-          tests_regex: "OilIntegration\\..*"
-          # Tests disabled due to bad DWARF generated by the old clang compiler in CI
-          exclude_regex: "OilIntegration.fbstring_.*|OilIntegration.capture_keys_string|OilIntegration.capture_keys_multi_level"
-      - test:
-          name: test-clang
-          requires:
-            - build-clang
-          tests_regex: "OidIntegration\\..*"
-          # Tests disabled due to bad DWARF generated by the old clang compiler in CI
-          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_a"
-
 executors:
   nix-docker:
     docker:

From 1726349e08c119fc6530851245625a6cc58f10f2 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 29 Dec 2023 10:36:10 +0000
Subject: [PATCH 090/188] oilgen: add to integration test framework

TODO: Replace the references to local paths.

oilgen (the basis of Ahead Of Time compilation for OIL) has never been passed
through our large test suite and has instead had more focused testing and large
examples. When consuming DWARF information in a similar fashion to JIT OIL this
was okay, but with the new Clang AST based mechanism it means we have very
little coverage.

This change adds an oilgen test for every test case that has an oil test.
Relying on the build system to create the test target as before would make it
difficult to have failing tests, so we move the build into the integration test
runner. This involves:
1. Writing the input source to a file.
2. Consuming it with oilgen to get the implementation object file.
3. Compiling the input source and linking it with this file.
4. Running the newly created target.

This approach can give the full error message at any stage that fails and will
fail the test appropriately. The downside is the build system integration is
more difficult, as we need the correct compiler flags for the target and to use
the correct compiler. It would be very tricky to replicate this in a build
system that's not CMake, so we will likely only run these tests in open source.

Test plan:
- CI
---
 test/integration/CMakeLists.txt    |   5 +-
 test/integration/gen_tests.py      | 149 ++++++++++++++++++++------
 test/integration/runner_common.cpp | 161 +++++++++++++++++++++++++++++
 test/integration/runner_common.h   |  14 +++
 4 files changed, 296 insertions(+), 33 deletions(-)

diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 12cb239..42346e0 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -63,7 +63,10 @@ target_link_libraries(integration_test_runner PRIVATE
 target_compile_definitions(integration_test_runner PRIVATE
   TARGET_EXE_PATH="${CMAKE_CURRENT_BINARY_DIR}/integration_test_target"
   OID_EXE_PATH="$<TARGET_FILE:oid>"
-  CONFIG_FILE_PATH="${CMAKE_BINARY_DIR}/testing.oid.toml")
+  OILGEN_EXE_PATH="$<TARGET_FILE:oilgen>"
+  CONFIG_FILE_PATH="${CMAKE_BINARY_DIR}/testing.oid.toml"
+  CXX="${CMAKE_CXX_COMPILER}"
+)
 
 if (${THRIFT_FOUND})
   foreach(THRIFT_TEST IN LISTS THRIFT_TESTS)
diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py
index 2dbc3ff..d366a48 100644
--- a/test/integration/gen_tests.py
+++ b/test/integration/gen_tests.py
@@ -50,6 +50,41 @@ def add_headers(f, custom_headers, thrift_headers):
         f.write(f'#include "{header}"\n')
 
 
+def add_test_getters(f, case_name, case):
+    param_types = ", ".join(
+        f"std::remove_cvref_t<{param}>" for param in case["param_types"]
+    )
+    if "arg_types" in case:
+        arg_types = ", ".join(case["arg_types"])
+    else:
+        arg_types = param_types
+
+    f.write(
+        f"\n"
+        f"  std::tuple<{arg_types}> get_{case_name}() {{\n"
+        f'{case["setup"]}\n'
+        f"  }}\n"
+    )
+
+
+def get_param_str(param, i):
+    if "]" in param:
+        # Array param
+
+        if ")" in param:
+            # "int(&)[5]" ->  "int (&a0)[5]"
+            start, end = param.split(")")
+            return f"{start}a{i}){end}"
+
+        # "int[5]" -> "int a0[5]"
+        # "int[5][10]" -> "int a0[5][10]"
+        type_name, array_size = param.split("[", 1)
+        return f"{type_name} a{i}[{array_size}"
+
+    # Non-array param, e.g. "int&" -> "int& a0"
+    return f"{param} a{i}"
+
+
 def add_test_setup(f, config):
     ns = get_namespace(config["suite"])
     # fmt: off
@@ -64,23 +99,6 @@ def add_test_setup(f, config):
     )
     # fmt: on
 
-    def get_param_str(param, i):
-        if "]" in param:
-            # Array param
-
-            if ")" in param:
-                # "int(&)[5]" ->  "int (&a0)[5]"
-                start, end = param.split(")")
-                return f"{start}a{i}){end}"
-
-            # "int[5]" -> "int a0[5]"
-            # "int[5][10]" -> "int a0[5][10]"
-            type_name, array_size = param.split("[", 1)
-            return f"{type_name} a{i}[{array_size}"
-
-        # Non-array param, e.g. "int&" -> "int& a0"
-        return f"{param} a{i}"
-
     def define_traceable_func(name, params, body):
         return (
             f"\n"
@@ -98,21 +116,7 @@ def add_test_setup(f, config):
             # target func for it
             continue
 
-        # generate getter for an object of this type
-        param_types = ", ".join(
-            f"std::remove_cvref_t<{param}>" for param in case["param_types"]
-        )
-        if "arg_types" in case:
-            arg_types = ", ".join(case["arg_types"])
-        else:
-            arg_types = param_types
-
-        f.write(
-            f"\n"
-            f"  std::tuple<{arg_types}> get_{case_name}() {{\n"
-            f'{case["setup"]}\n'
-            f"  }}\n"
-        )
+        add_test_getters(f, case_name, case)
 
         # generate oid and oil targets
         params_str = ", ".join(
@@ -265,6 +269,7 @@ def add_tests(f, config):
     for case_name, case in config["cases"].items():
         add_oid_integration_test(f, config, case_name, case)
         add_oil_integration_test(f, config, case_name, case)
+        add_oilgen_integration_test(f, config, case_name, case)
 
 
 def add_oid_integration_test(f, config, case_name, case):
@@ -399,6 +404,86 @@ def add_oil_integration_test(f, config, case_name, case):
     f.write(f"}}\n")
 
 
+def add_oilgen_integration_test(f, config, case_name, case):
+    case_str = get_case_name(config["suite"], case_name)
+    exit_code = case.get("expect_oil_exit_code", 0)
+
+    if "oil_disable" in case or "target_function" in case:
+        return
+
+    config_prefix = case.get("config_prefix", "")
+    config_suffix = case.get("config_suffix", "")
+
+    f.write(
+        f"\n"
+        f"TEST_F(OilgenIntegration, {case_str}) {{\n"
+        f"{generate_skip(case, 'oil')}"
+    )
+
+    f.write('  constexpr std::string_view targetSrc = R"--(')
+    headers = set(config.get("includes", []))
+    add_headers(f, sorted(headers), [])
+
+    f.write(
+        f"\n"
+        f'{config.get("raw_definitions", "")}\n'
+        f"#pragma clang diagnostic push\n"
+        f"#pragma clang diagnostic ignored \"-Wunused-private-field\"\n"
+        f'{config.get("definitions", "")}\n'
+        f"#pragma clang diagnostic pop\n"
+    )
+    add_test_getters(f, case_name, case)
+
+    main = "int main() {\n"
+    main += "  auto pr = oi::exporters::Json(std::cout);\n"
+    main += "  pr.setPretty(true);\n"
+    main += f"  auto val = get_{case_name}();\n"
+    for i in range(len(case["param_types"])):
+        main += f"  auto ret{i} = oi::introspect(std::get<{i}>(val));\n"
+        main += f"  pr.print(ret{i});\n"
+    main += "}\n"
+
+    f.write(main)
+    f.write(')--";\n')
+
+    f.write(
+        f'  std::string configPrefix = R"--({config_prefix})--";\n'
+        f'  std::string configSuffix = R"--({config_suffix})--";\n'
+        f"  ba::io_context ctx;\n"
+        f"  auto target = runOilgenTarget({{\n"
+        f"    .ctx = ctx,\n"
+        f"    .targetSrc = targetSrc,\n"
+        f"  }}, std::move(configPrefix), std::move(configSuffix));\n\n"
+        f"  ASSERT_EQ(exit_code(target), {exit_code});\n"
+    )
+
+    key = "expect_json"
+    if "expect_json_v2" in case:
+        key = "expect_json_v2"
+    if key in case:
+        try:
+            json.loads(case[key])
+        except json.decoder.JSONDecodeError as error:
+            print(
+                f"\x1b[31m`expect_json` value for test case {config['suite']}.{case_name} was invalid JSON: {error}\x1b[0m",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+        f.write(
+            f"\n"
+            f"  std::stringstream expected_json_ss;\n"
+            f'  expected_json_ss << R"--({case[key]})--";\n'
+            f"  auto result_json_ss = std::stringstream(stdout_);\n"
+            f"  bpt::ptree expected_json, actual_json;\n"
+            f"  bpt::read_json(expected_json_ss, expected_json);\n"
+            f"  bpt::read_json(result_json_ss, actual_json);\n"
+            f"  compare_json(expected_json, actual_json);\n"
+        )
+
+    f.write(f"}}\n")
+
+
 def generate_skip(case, specific):
     possibly_skip = ""
     skip_reason = case.get("skip", False)
diff --git a/test/integration/runner_common.cpp b/test/integration/runner_common.cpp
index c86bd59..a0d8d4d 100644
--- a/test/integration/runner_common.cpp
+++ b/test/integration/runner_common.cpp
@@ -445,3 +445,164 @@ Proc OilIntegration::runOilTarget(OilOpts opts,
               std::move(std_out),
               std::move(std_err)};
 }
+
+std::string OilgenIntegration::TmpDirStr() {
+  return std::string("/tmp/oilgen-integration-XXXXXX");
+}
+
+Proc OilgenIntegration::runOilgenTarget(OilgenOpts opts,
+                                        std::string configPrefix,
+                                        std::string configSuffix) {
+  // Run an oilgen test in three stages.
+  // 1. Generate the OIL implementation .o from the input source.
+  // 2. Compile the input source and link to the implementation.
+  // 3. Run the now complete target.
+
+  auto path = workingDir / "input_src.cpp";
+  {
+    std::ofstream file{path, std::ios_base::app};
+    file << opts.targetSrc;
+  }
+
+  std::string generatorExe =
+      std::string(OILGEN_EXE_PATH) + " " + path.native() + " ";
+  if (auto prefix = writeCustomConfig("prefix", configPrefix)) {
+    generatorExe += "--config-file ";
+    generatorExe += *prefix;
+    generatorExe += " ";
+  }
+  generatorExe += "--config-file ";
+  generatorExe += configFile;
+  if (auto suffix = writeCustomConfig("suffix", configSuffix)) {
+    generatorExe += " ";
+    generatorExe += "--config-file ";
+    generatorExe += *suffix;
+  }
+
+  // TODO: get this from the CMake arguments for integration_test_target.cpp
+  // somehow
+  // clang-format off
+  std::vector<std::string> clangArgs{
+      "-DOIL_AOT_COMPILATION=1",
+      "-isystem",
+      "/data/users/jakehillion/object-introspection-sl/include",
+      "--std=c++20",
+      "-resource-dir",
+      "/usr/lib64/clang/15.0.7",
+      "-fpic",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../include/c++/12",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../include/c++/12/x86_64-redhat-linux",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../include/c++/12/backward",
+      "-isystem",
+      "/usr/lib64/clang/15.0.7/include",
+      "-isystem",
+      "/usr/local/include",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../x86_64-redhat-linux/include",
+      "-isystem",
+      "/include",
+      "-isystem",
+      "/usr/include",
+      "-fdebug-compilation-dir=/data/users/jakehillion/Downloads/libclang-oilgen-testing",
+      "-fgnuc-version=4.2.1",
+      "-fcxx-exceptions",
+      "-fexceptions",
+      "-faddrsig",
+      "-D__GCC_HAVE_DWARF2_CFI_ASM=1",
+      "-x",
+      "c++",
+  };
+  // clang-format on
+  for (const auto& arg : clangArgs) {
+    generatorExe += " --extra-arg=";
+    generatorExe += arg;
+  }
+
+  if (verbose) {
+    std::cerr << "Running: " << generatorExe << std::endl;
+  }
+
+  bp::child generatorProc{generatorExe, opts.ctx};
+  generatorProc.wait();
+  if (generatorProc.exit_code() != 0)
+    throw std::runtime_error("generation failed!");
+
+  std::string compilerExe =
+      std::string(CXX) +
+      " a.o input_src.cpp "
+      "/data/users/jakehillion/object-introspection-sl/oi/exporters/Json.cpp "
+      "/data/users/jakehillion/object-introspection-sl/oi/"
+      "IntrospectionResult.cpp "
+      "/data/users/jakehillion/object-introspection-sl/oi/exporters/"
+      "ParsedData.cpp";
+  for (const auto& arg : clangArgs) {
+    compilerExe += ' ';
+    compilerExe += arg;
+  }
+
+  if (verbose) {
+    std::cerr << "Running: " << compilerExe << std::endl;
+  }
+
+  bp::child compilerProc{compilerExe, opts.ctx};
+  compilerProc.wait();
+  if (compilerProc.exit_code() != 0)
+    throw std::runtime_error("compilation failed");
+
+  std::string targetExe = "./a.out";
+  if (verbose) {
+    std::cerr << "Running: " << targetExe << std::endl;
+  }
+
+  // Use tee to write the output to files. If verbose is on, also redirect the
+  // output to stderr.
+  bp::async_pipe std_out_pipe(opts.ctx), std_err_pipe(opts.ctx);
+  bp::child std_out, std_err;
+  if (verbose) {
+    // clang-format off
+    std_out = bp::child(bp::search_path("tee"),
+                        (workingDir / "stdout").string(),
+                        bp::std_in < std_out_pipe,
+                        bp::std_out > stderr,
+                        opts.ctx);
+    std_err = bp::child(bp::search_path("tee"),
+                        (workingDir / "stderr").string(),
+                        bp::std_in < std_err_pipe,
+                        bp::std_out > stderr,
+                        opts.ctx);
+    // clang-format on
+  } else {
+    // clang-format off
+    std_out = bp::child(bp::search_path("tee"),
+                        (workingDir / "stdout").string(),
+                        bp::std_in < std_out_pipe,
+                        bp::std_out > bp::null,
+                        opts.ctx);
+    std_err = bp::child(bp::search_path("tee"),
+                        (workingDir / "stderr").string(),
+                        bp::std_in < std_err_pipe,
+                        bp::std_out > bp::null,
+                        opts.ctx);
+    // clang-format on
+  }
+
+  /* Spawn `oid` with tracing on and IOs redirected */
+  // clang-format off
+  bp::child targetProc(
+      targetExe,
+      bp::std_in  < bp::null,
+      bp::std_out > std_out_pipe,
+      bp::std_err > std_err_pipe,
+      opts.ctx);
+  // clang-format on
+
+  return Proc{
+      opts.ctx,
+      std::move(targetProc),
+      std::move(std_out),
+      std::move(std_err),
+  };
+}
diff --git a/test/integration/runner_common.h b/test/integration/runner_common.h
index d2b5e57..8c6c8eb 100644
--- a/test/integration/runner_common.h
+++ b/test/integration/runner_common.h
@@ -22,6 +22,11 @@ struct OilOpts {
   std::string targetArgs;
 };
 
+struct OilgenOpts {
+  boost::asio::io_context& ctx;
+  std::string_view targetSrc;
+};
+
 struct Proc {
   boost::asio::io_context& ctx;
   boost::process::child proc;
@@ -81,3 +86,12 @@ class OilIntegration : public IntegrationBase {
                     std::string configPrefix,
                     std::string configSuffix);
 };
+
+class OilgenIntegration : public IntegrationBase {
+ protected:
+  std::string TmpDirStr() override;
+
+  Proc runOilgenTarget(OilgenOpts opts,
+                       std::string configPrefix,
+                       std::string configSuffix);
+};

From 79a124443098acbf2f99db56f38fb6b86d4a4d12 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 29 Dec 2023 10:39:10 +0000
Subject: [PATCH 091/188] oilgen: add to integration test framework

TODO: Replace the references to local paths.

oilgen (the basis of Ahead Of Time compilation for OIL) has never been passed
through our large test suite and has instead had more focused testing and large
examples. When consuming DWARF information in a similar fashion to JIT OIL this
was okay, but with the new Clang AST based mechanism it means we have very
little coverage.

This change adds an oilgen test for every test case that has an oil test.
Relying on the build system to create the test target as before would make it
difficult to have failing tests, so we move the build into the integration test
runner. This involves:
1. Writing the input source to a file.
2. Consuming it with oilgen to get the implementation object file.
3. Compiling the input source and linking it with this file.
4. Running the newly created target.

This approach can give the full error message at any stage that fails and will
fail the test appropriately. The downside is the build system integration is
more difficult, as we need the correct compiler flags for the target and to use
the correct compiler. It would be very tricky to replicate this in a build
system that's not CMake, so we will likely only run these tests in open source.

Test plan:
- CI
---
 test/integration/CMakeLists.txt    |   5 +-
 test/integration/gen_tests.py      | 149 ++++++++++++++++++++------
 test/integration/runner_common.cpp | 161 +++++++++++++++++++++++++++++
 test/integration/runner_common.h   |  14 +++
 4 files changed, 296 insertions(+), 33 deletions(-)

diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 12cb239..42346e0 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -63,7 +63,10 @@ target_link_libraries(integration_test_runner PRIVATE
 target_compile_definitions(integration_test_runner PRIVATE
   TARGET_EXE_PATH="${CMAKE_CURRENT_BINARY_DIR}/integration_test_target"
   OID_EXE_PATH="$<TARGET_FILE:oid>"
-  CONFIG_FILE_PATH="${CMAKE_BINARY_DIR}/testing.oid.toml")
+  OILGEN_EXE_PATH="$<TARGET_FILE:oilgen>"
+  CONFIG_FILE_PATH="${CMAKE_BINARY_DIR}/testing.oid.toml"
+  CXX="${CMAKE_CXX_COMPILER}"
+)
 
 if (${THRIFT_FOUND})
   foreach(THRIFT_TEST IN LISTS THRIFT_TESTS)
diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py
index 2dbc3ff..c1b4bc7 100644
--- a/test/integration/gen_tests.py
+++ b/test/integration/gen_tests.py
@@ -50,6 +50,41 @@ def add_headers(f, custom_headers, thrift_headers):
         f.write(f'#include "{header}"\n')
 
 
+def add_test_getters(f, case_name, case):
+    param_types = ", ".join(
+        f"std::remove_cvref_t<{param}>" for param in case["param_types"]
+    )
+    if "arg_types" in case:
+        arg_types = ", ".join(case["arg_types"])
+    else:
+        arg_types = param_types
+
+    f.write(
+        f"\n"
+        f"  std::tuple<{arg_types}> get_{case_name}() {{\n"
+        f'{case["setup"]}\n'
+        f"  }}\n"
+    )
+
+
+def get_param_str(param, i):
+    if "]" in param:
+        # Array param
+
+        if ")" in param:
+            # "int(&)[5]" ->  "int (&a0)[5]"
+            start, end = param.split(")")
+            return f"{start}a{i}){end}"
+
+        # "int[5]" -> "int a0[5]"
+        # "int[5][10]" -> "int a0[5][10]"
+        type_name, array_size = param.split("[", 1)
+        return f"{type_name} a{i}[{array_size}"
+
+    # Non-array param, e.g. "int&" -> "int& a0"
+    return f"{param} a{i}"
+
+
 def add_test_setup(f, config):
     ns = get_namespace(config["suite"])
     # fmt: off
@@ -64,23 +99,6 @@ def add_test_setup(f, config):
     )
     # fmt: on
 
-    def get_param_str(param, i):
-        if "]" in param:
-            # Array param
-
-            if ")" in param:
-                # "int(&)[5]" ->  "int (&a0)[5]"
-                start, end = param.split(")")
-                return f"{start}a{i}){end}"
-
-            # "int[5]" -> "int a0[5]"
-            # "int[5][10]" -> "int a0[5][10]"
-            type_name, array_size = param.split("[", 1)
-            return f"{type_name} a{i}[{array_size}"
-
-        # Non-array param, e.g. "int&" -> "int& a0"
-        return f"{param} a{i}"
-
     def define_traceable_func(name, params, body):
         return (
             f"\n"
@@ -98,21 +116,7 @@ def add_test_setup(f, config):
             # target func for it
             continue
 
-        # generate getter for an object of this type
-        param_types = ", ".join(
-            f"std::remove_cvref_t<{param}>" for param in case["param_types"]
-        )
-        if "arg_types" in case:
-            arg_types = ", ".join(case["arg_types"])
-        else:
-            arg_types = param_types
-
-        f.write(
-            f"\n"
-            f"  std::tuple<{arg_types}> get_{case_name}() {{\n"
-            f'{case["setup"]}\n'
-            f"  }}\n"
-        )
+        add_test_getters(f, case_name, case)
 
         # generate oid and oil targets
         params_str = ", ".join(
@@ -265,6 +269,7 @@ def add_tests(f, config):
     for case_name, case in config["cases"].items():
         add_oid_integration_test(f, config, case_name, case)
         add_oil_integration_test(f, config, case_name, case)
+        add_oilgen_integration_test(f, config, case_name, case)
 
 
 def add_oid_integration_test(f, config, case_name, case):
@@ -399,6 +404,86 @@ def add_oil_integration_test(f, config, case_name, case):
     f.write(f"}}\n")
 
 
+def add_oilgen_integration_test(f, config, case_name, case):
+    case_str = get_case_name(config["suite"], case_name)
+    exit_code = case.get("expect_oil_exit_code", 0)
+
+    if "oil_disable" in case or "target_function" in case:
+        return
+
+    config_prefix = case.get("config_prefix", "")
+    config_suffix = case.get("config_suffix", "")
+
+    f.write(
+        f"\n"
+        f"TEST_F(OilgenIntegration, {case_str}) {{\n"
+        f"{generate_skip(case, 'oil')}"
+    )
+
+    f.write('  constexpr std::string_view targetSrc = R"--(')
+    headers = set(config.get("includes", []))
+    add_headers(f, sorted(headers), [])
+
+    f.write(
+        f"\n"
+        f'{config.get("raw_definitions", "")}\n'
+        f"#pragma clang diagnostic push\n"
+        f'#pragma clang diagnostic ignored "-Wunused-private-field"\n'
+        f'{config.get("definitions", "")}\n'
+        f"#pragma clang diagnostic pop\n"
+    )
+    add_test_getters(f, case_name, case)
+
+    main = "int main() {\n"
+    main += "  auto pr = oi::exporters::Json(std::cout);\n"
+    main += "  pr.setPretty(true);\n"
+    main += f"  auto val = get_{case_name}();\n"
+    for i in range(len(case["param_types"])):
+        main += f"  auto ret{i} = oi::introspect(std::get<{i}>(val));\n"
+        main += f"  pr.print(ret{i});\n"
+    main += "}\n"
+
+    f.write(main)
+    f.write(')--";\n')
+
+    f.write(
+        f'  std::string configPrefix = R"--({config_prefix})--";\n'
+        f'  std::string configSuffix = R"--({config_suffix})--";\n'
+        f"  ba::io_context ctx;\n"
+        f"  auto target = runOilgenTarget({{\n"
+        f"    .ctx = ctx,\n"
+        f"    .targetSrc = targetSrc,\n"
+        f"  }}, std::move(configPrefix), std::move(configSuffix));\n\n"
+        f"  ASSERT_EQ(exit_code(target), {exit_code});\n"
+    )
+
+    key = "expect_json"
+    if "expect_json_v2" in case:
+        key = "expect_json_v2"
+    if key in case:
+        try:
+            json.loads(case[key])
+        except json.decoder.JSONDecodeError as error:
+            print(
+                f"\x1b[31m`expect_json` value for test case {config['suite']}.{case_name} was invalid JSON: {error}\x1b[0m",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+        f.write(
+            f"\n"
+            f"  std::stringstream expected_json_ss;\n"
+            f'  expected_json_ss << R"--({case[key]})--";\n'
+            f"  auto result_json_ss = std::stringstream(stdout_);\n"
+            f"  bpt::ptree expected_json, actual_json;\n"
+            f"  bpt::read_json(expected_json_ss, expected_json);\n"
+            f"  bpt::read_json(result_json_ss, actual_json);\n"
+            f"  compare_json(expected_json, actual_json);\n"
+        )
+
+    f.write(f"}}\n")
+
+
 def generate_skip(case, specific):
     possibly_skip = ""
     skip_reason = case.get("skip", False)
diff --git a/test/integration/runner_common.cpp b/test/integration/runner_common.cpp
index c86bd59..a0d8d4d 100644
--- a/test/integration/runner_common.cpp
+++ b/test/integration/runner_common.cpp
@@ -445,3 +445,164 @@ Proc OilIntegration::runOilTarget(OilOpts opts,
               std::move(std_out),
               std::move(std_err)};
 }
+
+std::string OilgenIntegration::TmpDirStr() {
+  return std::string("/tmp/oilgen-integration-XXXXXX");
+}
+
+Proc OilgenIntegration::runOilgenTarget(OilgenOpts opts,
+                                        std::string configPrefix,
+                                        std::string configSuffix) {
+  // Run an oilgen test in three stages.
+  // 1. Generate the OIL implementation .o from the input source.
+  // 2. Compile the input source and link to the implementation.
+  // 3. Run the now complete target.
+
+  auto path = workingDir / "input_src.cpp";
+  {
+    std::ofstream file{path, std::ios_base::app};
+    file << opts.targetSrc;
+  }
+
+  std::string generatorExe =
+      std::string(OILGEN_EXE_PATH) + " " + path.native() + " ";
+  if (auto prefix = writeCustomConfig("prefix", configPrefix)) {
+    generatorExe += "--config-file ";
+    generatorExe += *prefix;
+    generatorExe += " ";
+  }
+  generatorExe += "--config-file ";
+  generatorExe += configFile;
+  if (auto suffix = writeCustomConfig("suffix", configSuffix)) {
+    generatorExe += " ";
+    generatorExe += "--config-file ";
+    generatorExe += *suffix;
+  }
+
+  // TODO: get this from the CMake arguments for integration_test_target.cpp
+  // somehow
+  // clang-format off
+  std::vector<std::string> clangArgs{
+      "-DOIL_AOT_COMPILATION=1",
+      "-isystem",
+      "/data/users/jakehillion/object-introspection-sl/include",
+      "--std=c++20",
+      "-resource-dir",
+      "/usr/lib64/clang/15.0.7",
+      "-fpic",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../include/c++/12",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../include/c++/12/x86_64-redhat-linux",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../include/c++/12/backward",
+      "-isystem",
+      "/usr/lib64/clang/15.0.7/include",
+      "-isystem",
+      "/usr/local/include",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../x86_64-redhat-linux/include",
+      "-isystem",
+      "/include",
+      "-isystem",
+      "/usr/include",
+      "-fdebug-compilation-dir=/data/users/jakehillion/Downloads/libclang-oilgen-testing",
+      "-fgnuc-version=4.2.1",
+      "-fcxx-exceptions",
+      "-fexceptions",
+      "-faddrsig",
+      "-D__GCC_HAVE_DWARF2_CFI_ASM=1",
+      "-x",
+      "c++",
+  };
+  // clang-format on
+  for (const auto& arg : clangArgs) {
+    generatorExe += " --extra-arg=";
+    generatorExe += arg;
+  }
+
+  if (verbose) {
+    std::cerr << "Running: " << generatorExe << std::endl;
+  }
+
+  bp::child generatorProc{generatorExe, opts.ctx};
+  generatorProc.wait();
+  if (generatorProc.exit_code() != 0)
+    throw std::runtime_error("generation failed!");
+
+  std::string compilerExe =
+      std::string(CXX) +
+      " a.o input_src.cpp "
+      "/data/users/jakehillion/object-introspection-sl/oi/exporters/Json.cpp "
+      "/data/users/jakehillion/object-introspection-sl/oi/"
+      "IntrospectionResult.cpp "
+      "/data/users/jakehillion/object-introspection-sl/oi/exporters/"
+      "ParsedData.cpp";
+  for (const auto& arg : clangArgs) {
+    compilerExe += ' ';
+    compilerExe += arg;
+  }
+
+  if (verbose) {
+    std::cerr << "Running: " << compilerExe << std::endl;
+  }
+
+  bp::child compilerProc{compilerExe, opts.ctx};
+  compilerProc.wait();
+  if (compilerProc.exit_code() != 0)
+    throw std::runtime_error("compilation failed");
+
+  std::string targetExe = "./a.out";
+  if (verbose) {
+    std::cerr << "Running: " << targetExe << std::endl;
+  }
+
+  // Use tee to write the output to files. If verbose is on, also redirect the
+  // output to stderr.
+  bp::async_pipe std_out_pipe(opts.ctx), std_err_pipe(opts.ctx);
+  bp::child std_out, std_err;
+  if (verbose) {
+    // clang-format off
+    std_out = bp::child(bp::search_path("tee"),
+                        (workingDir / "stdout").string(),
+                        bp::std_in < std_out_pipe,
+                        bp::std_out > stderr,
+                        opts.ctx);
+    std_err = bp::child(bp::search_path("tee"),
+                        (workingDir / "stderr").string(),
+                        bp::std_in < std_err_pipe,
+                        bp::std_out > stderr,
+                        opts.ctx);
+    // clang-format on
+  } else {
+    // clang-format off
+    std_out = bp::child(bp::search_path("tee"),
+                        (workingDir / "stdout").string(),
+                        bp::std_in < std_out_pipe,
+                        bp::std_out > bp::null,
+                        opts.ctx);
+    std_err = bp::child(bp::search_path("tee"),
+                        (workingDir / "stderr").string(),
+                        bp::std_in < std_err_pipe,
+                        bp::std_out > bp::null,
+                        opts.ctx);
+    // clang-format on
+  }
+
+  /* Spawn `oid` with tracing on and IOs redirected */
+  // clang-format off
+  bp::child targetProc(
+      targetExe,
+      bp::std_in  < bp::null,
+      bp::std_out > std_out_pipe,
+      bp::std_err > std_err_pipe,
+      opts.ctx);
+  // clang-format on
+
+  return Proc{
+      opts.ctx,
+      std::move(targetProc),
+      std::move(std_out),
+      std::move(std_err),
+  };
+}
diff --git a/test/integration/runner_common.h b/test/integration/runner_common.h
index d2b5e57..8c6c8eb 100644
--- a/test/integration/runner_common.h
+++ b/test/integration/runner_common.h
@@ -22,6 +22,11 @@ struct OilOpts {
   std::string targetArgs;
 };
 
+struct OilgenOpts {
+  boost::asio::io_context& ctx;
+  std::string_view targetSrc;
+};
+
 struct Proc {
   boost::asio::io_context& ctx;
   boost::process::child proc;
@@ -81,3 +86,12 @@ class OilIntegration : public IntegrationBase {
                     std::string configPrefix,
                     std::string configSuffix);
 };
+
+class OilgenIntegration : public IntegrationBase {
+ protected:
+  std::string TmpDirStr() override;
+
+  Proc runOilgenTarget(OilgenOpts opts,
+                       std::string configPrefix,
+                       std::string configSuffix);
+};

From c715e09505c5c88e18f9a153dbda7de59c03e582 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 29 Dec 2023 17:47:01 +0000
Subject: [PATCH 092/188] tbv2: calculate total memory footprint

TODO: Review and clean up any todos.
TODO: Add size to all expect_json_v2.

Add the option to calculate total size (inclusive size) by wrapping the
existing iterator. This change provides a new iterator, `SizedIterator`, which
wraps an existing iterator and adds a new field `size` to the output element.

This is achieved with a two pass algorithm on the existing iterator:
1. Gather metadata for each element. This includes the total size up until that
   element and the range of elements that should be included in the size.
2. Return the result from the underlying iterator with the additional
   field.

Test plan:
- Added to the integration tests for full coverage.
---
 include/oi/exporters/Json.h         | 146 ++++++++++++++++++++++++++-
 include/oi/result/SizedResult-inl.h | 148 ++++++++++++++++++++++++++++
 include/oi/result/SizedResult.h     |  84 ++++++++++++++++
 oi/CMakeLists.txt                   |   4 -
 oi/exporters/Json.cpp               | 146 ---------------------------
 test/integration/CMakeLists.txt     |   2 +-
 test/integration/gen_tests.py       |   5 +-
 7 files changed, 379 insertions(+), 156 deletions(-)
 create mode 100644 include/oi/result/SizedResult-inl.h
 create mode 100644 include/oi/result/SizedResult.h
 delete mode 100644 oi/exporters/Json.cpp

diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h
index 31aa808..e5075e3 100644
--- a/include/oi/exporters/Json.h
+++ b/include/oi/exporters/Json.h
@@ -17,8 +17,10 @@
 #define INCLUDED_OI_EXPORTERS_JSON_H 1
 
 #include <oi/IntrospectionResult.h>
+#include <oi/result/SizedResult.h>
 
 #include <ostream>
+#include <string_view>
 
 namespace oi::exporters {
 
@@ -26,19 +28,157 @@ class Json {
  public:
   Json(std::ostream& out);
 
-  void print(const IntrospectionResult&);
-  void print(IntrospectionResult::const_iterator& it,
-             IntrospectionResult::const_iterator& end);
+  template <typename Res>
+  void print(const Res& r) {
+    auto begin = r.begin();
+    auto end = r.end();
+    return print(begin, end);
+  }
+  template <typename It>
+  void print(It& it, const It& end);
 
   void setPretty(bool pretty) {
     pretty_ = pretty;
   }
 
  private:
+  std::string_view tab() const;
+  std::string_view space() const;
+  std::string_view endl() const;
+  static std::string makeIndent(size_t depth);
+
+  void printStringField(std::string_view name, std::string_view value);
+  void printBoolField(std::string_view name, bool value);
+  void printUnsignedField(std::string_view name, uint64_t value);
+  void printPointerField(std::string_view name, uintptr_t value);
+  template <typename Rng>
+  void printListField(std::string_view name, const Rng& range);
+
+  void printFields(const result::Element&);
+  template <typename El>
+  void printFields(const result::SizedElement<El>&);
+
   bool pretty_ = false;
   std::ostream& out_;
+
+  std::string indent_;
 };
 
+inline Json::Json(std::ostream& out) : out_(out) {
+}
+
+inline std::string_view Json::tab() const {
+  return pretty_ ? "  " : "";
+}
+inline std::string_view Json::space() const {
+  return pretty_ ? " " : "";
+}
+inline std::string_view Json::endl() const {
+  return pretty_ ? "\n" : "";
+}
+inline std::string Json::makeIndent(size_t depth) {
+  depth = std::max(depth, 1UL);
+  return std::string((depth - 1) * 4, ' ');
+}
+
+inline void Json::printStringField(std::string_view name, std::string_view value) {
+    out_ << tab() << '"' << name << '"' << ':' << space() << "\"" << value << "\"," << endl() << indent_;
+}
+inline void Json::printBoolField(std::string_view name, bool value) {
+    out_ << tab() << '"' << name << '"' << ':' << space() << value << ',' << endl() << indent_;
+}
+inline void Json::printUnsignedField(std::string_view name, uint64_t value) {
+    out_ << tab() << '"' << name << '"' << ':' << space() << value << ',' << endl() << indent_;
+}
+inline void Json::printPointerField(std::string_view name, uintptr_t value) {
+      out_ << tab() << '"' << name << '"' << space() << "\"0x" << std::hex <<value
+           << std::dec << "\"," << endl()
+           << indent_;
+}
+template <typename Rng>
+void Json::printListField(std::string_view name, const Rng& range) {
+  out_ << tab() << '"' << name << '"' << ':' << space() << '[';
+  bool first = true;
+  for (const auto& el : range) {
+    if (!std::exchange(first, false))
+      out_ << ',' << space();
+    out_ << '"' << el << '"';
+  }
+  out_ << "]," << endl() << indent_;
+}
+
+template <typename El>
+void Json::printFields(const result::SizedElement<El>& el) {
+  printUnsignedField("size", el.size);
+
+  printFields(el.inner());
+}
+
+inline void Json::printFields(const result::Element& el) {
+  printStringField("name", el.name);
+  printListField("typePath", el.type_path);
+  printListField("typeNames", el.type_names);
+  printUnsignedField("staticSize", el.static_size);
+  printUnsignedField("exclusiveSize", el.exclusive_size);
+  if (el.pointer.has_value())
+    printUnsignedField("pointer", *el.pointer);
+
+  if (const auto* s = std::get_if<result::Element::Scalar>(&el.data)) {
+    printUnsignedField("data", s->n);
+  } else if (const auto* p = std::get_if<result::Element::Pointer>(&el.data)) {
+    printPointerField("data", p->p);
+  } else if (const auto* str = std::get_if<std::string>(&el.data)) {
+    printStringField("data", *str);
+  }
+
+  if (el.container_stats.has_value()) {
+    printUnsignedField("length", el.container_stats->length);
+    printUnsignedField("capacity", el.container_stats->capacity);
+  }
+  if (el.is_set_stats.has_value())
+    printUnsignedField("is_set", el.is_set_stats->is_set);
+}
+
+template <typename It>
+void Json::print(It& it, const It& end) {
+  const auto depth = it->type_path.size();
+
+  indent_ = pretty_ ? makeIndent(depth) : "";
+  const auto lastIndent =
+      pretty_ ? makeIndent(depth - 1) : "";
+
+  out_ << '[' << endl() << indent_;
+
+  bool first = true;
+  while (it != end) {
+    if (it->type_path.size() < depth) {
+      // no longer a sibling, must be a sibling of the parent
+      break;
+    }
+
+    if (!std::exchange(first, false))
+      out_ << ',' << endl() << indent_;
+
+    out_ << '{' << endl() << indent_;
+
+    printFields(*it);
+
+    out_ << tab() << "\"members\":" << space();
+    if (++it != end && it->type_path.size() > depth) {
+      print(it, end);
+    } else {
+      out_ << "[]" << endl();
+    }
+
+    out_ << indent_ << "}";
+  }
+  if (depth == 1) {
+    out_ << endl() << ']' << endl();
+  } else {
+    out_ << endl() << lastIndent << tab() << ']' << endl();
+  }
+}
+
 }  // namespace oi::exporters
 
 #endif
diff --git a/include/oi/result/SizedResult-inl.h b/include/oi/result/SizedResult-inl.h
new file mode 100644
index 0000000..f35fd21
--- /dev/null
+++ b/include/oi/result/SizedResult-inl.h
@@ -0,0 +1,148 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#if defined(INCLUDED_OI_RESULT_SIZED_ELEMENT_INL_H) || \
+    !defined(INCLUDED_OI_RESULT_SIZED_ELEMENT_H)
+static_assert(false,
+              "SizedResult-inl.h provides inline declarations for "
+              "SizedResult.h and should only "
+              "be included by SizedResult.h");
+#endif
+#define INCLUDED_OI_RESULT_SIZED_ELEMENT_INL_H 1
+
+#include <iostream>   // TODO: remove
+#include <stdexcept>  // TODO: remove
+#include <utility>
+#include <vector>
+
+#include "SizedResult.h"
+
+namespace oi::result {
+
+template <typename Res>
+SizedResult<Res>::SizedResult(Res res) : res_{std::move(res)} {
+}
+
+template <typename Res>
+typename SizedResult<Res>::const_iterator SizedResult<Res>::begin() const {
+  return ++const_iterator{res_.begin(), res_.end()};
+}
+template <typename Res>
+typename SizedResult<Res>::const_iterator SizedResult<Res>::end() const {
+  return res_.end();
+}
+
+template <typename Res>
+SizedResult<Res>::const_iterator::const_iterator(It it, const It& end) : data_{it} {
+  struct StackEntry {
+    size_t index;
+    size_t depth;
+  };
+  std::vector<StackEntry> stack;
+
+  size_t size = 0;
+  size_t count = -1;
+  for (; it != end; ++it) {
+    ++count;
+
+    auto depth = it->type_path.size();
+    while (!stack.empty() && stack.back().depth >= depth) {
+      auto i = stack.back().index;
+      stack.pop_back();
+      helpers_[i].last_child = count - 1;
+    }
+
+    size += it->exclusive_size;
+    helpers_.emplace_back(SizeHelper{ .size = size });
+    stack.emplace_back(StackEntry{ .index = count, .depth = depth });
+  }
+  while (!stack.empty()) {
+    auto i = stack.back().index;
+    stack.pop_back();
+    helpers_[i].last_child = count;
+  }
+}
+
+template <typename Res>
+SizedResult<Res>::const_iterator::const_iterator(It end) : data_{end} {
+}
+
+template <typename Res>
+typename SizedResult<Res>::const_iterator
+SizedResult<Res>::const_iterator::operator++(int) {
+  auto old = *this;
+  operator++();
+  return old;
+}
+
+template <typename Res>
+typename SizedResult<Res>::const_iterator&
+SizedResult<Res>::const_iterator::operator++() {
+  // The below iterator is already pointing at the first element while this iterator is not. Skip incrementing it the first time around.
+  if (count_ != 0)
+    ++data_;
+
+  if (count_ == helpers_.size()) {
+    next_.reset();
+    return *this;
+  }
+
+  size_t size = helpers_[helpers_[count_].last_child].size;
+  if (count_ != 0)
+    size -= helpers_[count_ - 1].size;
+
+
+  next_.emplace(*data_, size);
+  ++count_;
+  return *this;
+}
+
+template <typename Res>
+const typename SizedResult<Res>::Element&
+SizedResult<Res>::const_iterator::operator*() const {
+  // auto* leak = new SizedResult<Res>::Element{*data_, (size_t)-1};
+  // return *leak;
+  return *next_;
+}
+template <typename Res>
+const typename SizedResult<Res>::Element*
+SizedResult<Res>::const_iterator::operator->() const {
+  // auto* leak = new SizedResult<Res>::Element{*data_, (size_t)-1};
+  // return leak;
+  return &*next_;
+}
+
+template <typename Res>
+bool SizedResult<Res>::const_iterator::operator==(
+    const SizedResult<Res>::const_iterator& that) const {
+  return this->data_ == that.data_;
+}
+
+template <typename Res>
+bool SizedResult<Res>::const_iterator::operator!=(
+    const SizedResult<Res>::const_iterator& that) const {
+  return !(*this == that);
+}
+
+template <typename El>
+SizedElement<El>::SizedElement(const El& el, size_t size_) : El{el}, size{size_} {
+}
+
+template <typename El>
+const El& SizedElement<El>::inner() const {
+  return static_cast<const El&>(*this);
+}
+
+}  // namespace oi::result
diff --git a/include/oi/result/SizedResult.h b/include/oi/result/SizedResult.h
new file mode 100644
index 0000000..983b804
--- /dev/null
+++ b/include/oi/result/SizedResult.h
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef INCLUDED_OI_RESULT_SIZED_ELEMENT_H
+#define INCLUDED_OI_RESULT_SIZED_ELEMENT_H 1
+
+#include <cstddef>
+#include <optional>
+#include <type_traits>
+#include <vector>
+
+namespace oi::result {
+
+template <typename El>
+struct SizedElement : public El {
+  SizedElement(const El& el, size_t size);
+  const El& inner() const;
+
+  size_t size;
+};
+
+template <typename Res>
+class SizedResult {
+ private:
+  using It = std::decay_t<decltype(std::declval<Res&>().begin())>;
+  using ParentEl = std::decay_t<decltype(std::declval<It&>().operator*())>;
+
+ public:
+  using Element = SizedElement<ParentEl>;
+
+ private:
+  struct SizeHelper {
+    size_t size = -1;
+    size_t last_child = -1;
+  };
+
+ public:
+  class const_iterator {
+    friend SizedResult;
+
+   public:
+    bool operator==(const const_iterator& that) const;
+    bool operator!=(const const_iterator& that) const;
+    const Element& operator*() const;
+    const Element* operator->() const;
+    const_iterator& operator++();
+    const_iterator operator++(int);
+
+   private:
+    const_iterator(It start, const It& end);
+    const_iterator(It end);
+
+    It data_;
+
+    std::vector<SizeHelper> helpers_;
+    size_t count_ = 0;
+    std::optional<Element> next_;
+  };
+
+  SizedResult(Res res);
+
+  const_iterator begin() const;
+  const_iterator end() const;
+
+ private:
+  Res res_;
+};
+
+}  // namespace oi::result
+
+#include "SizedResult-inl.h"
+#endif
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index 3754f41..64dbcae 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -55,10 +55,6 @@ target_link_libraries(codegen
   glog::glog
 )
 
-add_library(exporters_json exporters/Json.cpp)
-target_include_directories(exporters_json PUBLIC ${CMAKE_SOURCE_DIR}/include)
-target_link_libraries(exporters_json oil)
-
 add_library(exporters_csv exporters/CSV.cpp)
 target_include_directories(exporters_csv PUBLIC ${CMAKE_SOURCE_DIR}/include)
 target_link_libraries(exporters_csv oil)
diff --git a/oi/exporters/Json.cpp b/oi/exporters/Json.cpp
deleted file mode 100644
index 8a4d33c..0000000
--- a/oi/exporters/Json.cpp
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include <oi/exporters/Json.h>
-
-#include <algorithm>
-#include <stdexcept>
-#include <variant>
-
-template <class>
-inline constexpr bool always_false_v = false;
-
-namespace oi::exporters {
-namespace {
-
-template <typename It>
-void printStringList(std::ostream& out, It it, It end, bool pretty) {
-  out << '[';
-  for (; it != end; ++it) {
-    out << '"' << (*it) << '"';
-    if (it != end - 1)
-      out << (pretty ? ", " : ",");
-  }
-  out << ']';
-}
-
-std::string makeIndent(size_t depth) {
-  depth = std::max(depth, 1UL);
-  return std::string((depth - 1) * 4, ' ');
-}
-
-}  // namespace
-
-Json::Json(std::ostream& out) : out_(out) {
-}
-
-void Json::print(const IntrospectionResult& r) {
-  auto begin = r.cbegin();
-  auto end = r.cend();
-  return print(begin, end);
-}
-
-void Json::print(IntrospectionResult::const_iterator& it,
-                 IntrospectionResult::const_iterator& end) {
-  const auto firstTypePathSize = it->type_path.size();
-
-  const auto indent = pretty_ ? makeIndent(firstTypePathSize) : "";
-  const auto lastIndent =
-      pretty_ ? makeIndent(std::max(firstTypePathSize, 1UL) - 1) : "";
-  const auto* tab = pretty_ ? "  " : "";
-  const auto* space = pretty_ ? " " : "";
-  const auto* endl = pretty_ ? "\n" : "";
-
-  out_ << '[' << endl << indent;
-
-  bool first = true;
-  while (it != end) {
-    if (it->type_path.size() < firstTypePathSize) {
-      // no longer a sibling, must be a sibling of the type we're printing
-      break;
-    }
-
-    if (!first)
-      out_ << ',' << endl << indent;
-    first = false;
-
-    out_ << '{' << endl << indent;
-
-    out_ << tab << "\"name\"" << space << ':' << space << "\"" << it->name
-         << "\"," << endl
-         << indent;
-
-    out_ << tab << "\"typePath\"" << space << ':' << space << "";
-    printStringList(out_, it->type_path.begin(), it->type_path.end(), pretty_);
-    out_ << (pretty_ ? ",\n" : ",") << indent;
-
-    out_ << tab << "\"typeNames\"" << space << ':' << space;
-    printStringList(
-        out_, it->type_names.begin(), it->type_names.end(), pretty_);
-    out_ << ',' << endl << indent;
-
-    out_ << tab << "\"staticSize\":" << space << it->static_size << ',' << endl
-         << indent;
-    out_ << tab << "\"exclusiveSize\":" << space << it->exclusive_size << ','
-         << endl
-         << indent;
-
-    if (it->pointer.has_value()) {
-      out_ << tab << "\"pointer\":" << space << *(it->pointer) << ',' << endl
-           << indent;
-    }
-
-    if (auto* s = std::get_if<result::Element::Scalar>(&it->data)) {
-      out_ << tab << "\"data\":" << space << s->n << ',' << endl << indent;
-    } else if (auto* p = std::get_if<result::Element::Pointer>(&it->data)) {
-      out_ << tab << "\"data\":" << space << "\"0x" << std::hex << p->p
-           << std::dec << "\"," << endl
-           << indent;
-    } else if (auto* str = std::get_if<std::string>(&it->data)) {
-      out_ << tab << "\"data\":" << space << "\"" << *str << "\"," << endl
-           << indent;
-    }
-
-    if (it->container_stats.has_value()) {
-      out_ << tab << "\"length\":" << space << it->container_stats->length
-           << ',' << endl
-           << indent;
-      out_ << tab << "\"capacity\":" << space << it->container_stats->capacity
-           << ',' << endl
-           << indent;
-    }
-    if (it->is_set_stats.has_value()) {
-      out_ << tab << "\"is_set\":" << space << it->is_set_stats->is_set << ','
-           << endl
-           << indent;
-    }
-
-    out_ << tab << "\"members\":" << space;
-    if (++it != end && it->type_path.size() > firstTypePathSize) {
-      print(it, end);
-    } else {
-      out_ << "[]" << endl;
-    }
-
-    out_ << indent << "}";
-  }
-  if (firstTypePathSize == 1) {
-    out_ << endl << ']' << endl;
-  } else {
-    out_ << endl << lastIndent << tab << ']' << endl;
-  }
-}
-
-}  // namespace oi::exporters
diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 12cb239..4d235b5 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -50,7 +50,7 @@ add_executable(integration_test_target
   ${INTEGRATION_TEST_TARGET_SRC}
   folly_shims.cpp)
 target_compile_options(integration_test_target PRIVATE -O1)
-target_link_libraries(integration_test_target PRIVATE oil_jit exporters_json Boost::headers ${Boost_LIBRARIES})
+target_link_libraries(integration_test_target PRIVATE oil_jit Boost::headers ${Boost_LIBRARIES})
 
 add_executable(integration_test_runner ${INTEGRATION_TEST_RUNNER_SRC} runner_common.cpp)
 target_include_directories(integration_test_runner PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py
index 2dbc3ff..b72e371 100644
--- a/test/integration/gen_tests.py
+++ b/test/integration/gen_tests.py
@@ -40,6 +40,7 @@ def add_headers(f, custom_headers, thrift_headers):
 
 #include <oi/oi.h>
 #include <oi/exporters/Json.h>
+#include <oi/result/SizedResult.h>
 
 """
     )
@@ -145,8 +146,8 @@ def add_test_setup(f, config):
         oil_func_body += "    auto pr = oi::exporters::Json(std::cout);\n"
         oil_func_body += "    pr.setPretty(true);\n"
         for i in range(len(case["param_types"])):
-            oil_func_body += f"    auto ret{i} = oi::setupAndIntrospect(a{i}, opts);\n"
-            oil_func_body += f"    pr.print(*ret{i});\n"
+            oil_func_body += f"    auto ret{i} = oi::result::SizedResult(*oi::setupAndIntrospect(a{i}, opts));\n"
+            oil_func_body += f"    pr.print(ret{i});\n"
 
         f.write(
             define_traceable_func(

From 5d35c6840ee43fe1ece4bf985b2cba2e522883df Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 2 Jan 2024 14:28:17 +0000
Subject: [PATCH 093/188] circleci: run oid in legacy tests

Currently the CodeGen v1 CI tests are filtered to only run OIL tests. This is
incorrect, as OIL always runs with CodeGen v2. Change it so only OID tests are
run as legacy.

Also restructured the CI file a bit to group the build steps together and the
test steps together. I can revert this if it's inconvenient but I think it
makes it clearer.

Test plan:
- Checked the CI and it's running OID tests now
---
 .circleci/config.yml | 54 ++++++++++++++++++++++++--------------------
 1 file changed, 29 insertions(+), 25 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 0a75d99..0efae92 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -5,52 +5,56 @@ workflows:
     jobs:
       - lint
 
+      # Build
       - build:
           name: build-gcc
           cc: /usr/bin/gcc
           cxx: /usr/bin/g++
           warnings_as_errors: "OFF"
-      - test:
-          name: test-codegenv1-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-Ftype-graph"
-          tests_regex: "OilIntegration\\..*"
-      - test:
-          name: test-gcc
-          requires:
-            - build-gcc
-          tests_regex: "OidIntegration\\..*"
-          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
-      - coverage:
-          name: coverage
-          requires:
-            - test-gcc
-      - coverage:
-          name: coverage-codegenv1
-          requires:
-            - test-codegenv1-gcc
-
       - build:
           name: build-clang
           cc: /usr/bin/clang-12
           cxx: /usr/bin/clang++-12
           warnings_as_errors: "ON"
+
+      # Legacy tests
+      - test:
+          name: test-codegenv1-gcc
+          requires:
+            - build-gcc
+          oid_test_args: "-Ftype-graph"
+          tests_regex: "OidIntegration\\..*"
       - test:
           name: test-codegenv1-clang
           requires:
             - build-clang
           oid_test_args: "-Ftype-graph"
-          tests_regex: "OilIntegration\\..*"
+          tests_regex: "OidIntegration\\..*"
           # Tests disabled due to bad DWARF generated by the old clang compiler in CI
-          exclude_regex: "OilIntegration.fbstring_.*|OilIntegration.capture_keys_string|OilIntegration.capture_keys_multi_level"
+          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_a"
+
+      # Current tests
+      - test:
+          name: test-gcc
+          requires:
+            - build-gcc
+          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
       - test:
           name: test-clang
           requires:
             - build-clang
-          tests_regex: "OidIntegration\\..*"
           # Tests disabled due to bad DWARF generated by the old clang compiler in CI
-          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_a"
+          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_a|OilIntegration.capture_keys_string|OilIntegration.capture_keys_multi_level"
+
+      # Coverage
+      - coverage:
+          name: coverage-codegenv1
+          requires:
+            - test-codegenv1-gcc
+      - coverage:
+          name: coverage
+          requires:
+            - test-gcc
 
 executors:
   nix-docker:

From 13052b4f87b544ec76224caa869b7eec48c5de83 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 2 Jan 2024 15:48:32 +0000
Subject: [PATCH 094/188] circleci: run oid in legacy tests

Remove the CodeGen v1 sections of the CI config because both OID and OIL use
CodeGen v2.

Test plan:
- CI
---
 .circleci/config.yml | 22 +---------------------
 1 file changed, 1 insertion(+), 21 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 0a75d99..b91cc2c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -10,47 +10,27 @@ workflows:
           cc: /usr/bin/gcc
           cxx: /usr/bin/g++
           warnings_as_errors: "OFF"
-      - test:
-          name: test-codegenv1-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-Ftype-graph"
-          tests_regex: "OilIntegration\\..*"
       - test:
           name: test-gcc
           requires:
             - build-gcc
-          tests_regex: "OidIntegration\\..*"
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
       - coverage:
           name: coverage
           requires:
             - test-gcc
-      - coverage:
-          name: coverage-codegenv1
-          requires:
-            - test-codegenv1-gcc
 
       - build:
           name: build-clang
           cc: /usr/bin/clang-12
           cxx: /usr/bin/clang++-12
           warnings_as_errors: "ON"
-      - test:
-          name: test-codegenv1-clang
-          requires:
-            - build-clang
-          oid_test_args: "-Ftype-graph"
-          tests_regex: "OilIntegration\\..*"
-          # Tests disabled due to bad DWARF generated by the old clang compiler in CI
-          exclude_regex: "OilIntegration.fbstring_.*|OilIntegration.capture_keys_string|OilIntegration.capture_keys_multi_level"
       - test:
           name: test-clang
           requires:
             - build-clang
-          tests_regex: "OidIntegration\\..*"
           # Tests disabled due to bad DWARF generated by the old clang compiler in CI
-          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_a"
+          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_a|OilIntegration.fbstring_.*|OilIntegration.capture_keys_string|OilIntegration.capture_keys_multi_level"
 
 executors:
   nix-docker:

From 803b799e2fb558c90b070581f3edca71c1661ad5 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 2 Jan 2024 15:51:55 +0000
Subject: [PATCH 095/188] circleci: clean up codegen v1 runs

Remove the CodeGen v1 sections of the CI config because both OID and OIL use
CodeGen v2.

Test plan:
- CI
---
 .circleci/config.yml | 22 +---------------------
 1 file changed, 1 insertion(+), 21 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 0a75d99..b91cc2c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -10,47 +10,27 @@ workflows:
           cc: /usr/bin/gcc
           cxx: /usr/bin/g++
           warnings_as_errors: "OFF"
-      - test:
-          name: test-codegenv1-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-Ftype-graph"
-          tests_regex: "OilIntegration\\..*"
       - test:
           name: test-gcc
           requires:
             - build-gcc
-          tests_regex: "OidIntegration\\..*"
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
       - coverage:
           name: coverage
           requires:
             - test-gcc
-      - coverage:
-          name: coverage-codegenv1
-          requires:
-            - test-codegenv1-gcc
 
       - build:
           name: build-clang
           cc: /usr/bin/clang-12
           cxx: /usr/bin/clang++-12
           warnings_as_errors: "ON"
-      - test:
-          name: test-codegenv1-clang
-          requires:
-            - build-clang
-          oid_test_args: "-Ftype-graph"
-          tests_regex: "OilIntegration\\..*"
-          # Tests disabled due to bad DWARF generated by the old clang compiler in CI
-          exclude_regex: "OilIntegration.fbstring_.*|OilIntegration.capture_keys_string|OilIntegration.capture_keys_multi_level"
       - test:
           name: test-clang
           requires:
             - build-clang
-          tests_regex: "OidIntegration\\..*"
           # Tests disabled due to bad DWARF generated by the old clang compiler in CI
-          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_a"
+          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_a|OilIntegration.fbstring_.*|OilIntegration.capture_keys_string|OilIntegration.capture_keys_multi_level"
 
 executors:
   nix-docker:

From 5fa72c41da60df6eda2f412722e32570c7c0729c Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 2 Jan 2024 15:52:38 +0000
Subject: [PATCH 096/188] oilgen: add to integration test framework

TODO: Replace the references to local paths.

oilgen (the basis of Ahead Of Time compilation for OIL) has never been passed
through our large test suite and has instead had more focused testing and large
examples. When consuming DWARF information in a similar fashion to JIT OIL this
was okay, but with the new Clang AST based mechanism it means we have very
little coverage.

This change adds an oilgen test for every test case that has an oil test.
Relying on the build system to create the test target as before would make it
difficult to have failing tests, so we move the build into the integration test
runner. This involves:
1. Writing the input source to a file.
2. Consuming it with oilgen to get the implementation object file.
3. Compiling the input source and linking it with this file.
4. Running the newly created target.

This approach can give the full error message at any stage that fails and will
fail the test appropriately. The downside is the build system integration is
more difficult, as we need the correct compiler flags for the target and to use
the correct compiler. It would be very tricky to replicate this in a build
system that's not CMake, so we will likely only run these tests in open source.

Test plan:
- CI
---
 test/integration/CMakeLists.txt    |   5 +-
 test/integration/gen_tests.py      | 149 ++++++++++++++++++++------
 test/integration/runner_common.cpp | 161 +++++++++++++++++++++++++++++
 test/integration/runner_common.h   |  14 +++
 4 files changed, 296 insertions(+), 33 deletions(-)

diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 12cb239..42346e0 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -63,7 +63,10 @@ target_link_libraries(integration_test_runner PRIVATE
 target_compile_definitions(integration_test_runner PRIVATE
   TARGET_EXE_PATH="${CMAKE_CURRENT_BINARY_DIR}/integration_test_target"
   OID_EXE_PATH="$<TARGET_FILE:oid>"
-  CONFIG_FILE_PATH="${CMAKE_BINARY_DIR}/testing.oid.toml")
+  OILGEN_EXE_PATH="$<TARGET_FILE:oilgen>"
+  CONFIG_FILE_PATH="${CMAKE_BINARY_DIR}/testing.oid.toml"
+  CXX="${CMAKE_CXX_COMPILER}"
+)
 
 if (${THRIFT_FOUND})
   foreach(THRIFT_TEST IN LISTS THRIFT_TESTS)
diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py
index 2dbc3ff..c1b4bc7 100644
--- a/test/integration/gen_tests.py
+++ b/test/integration/gen_tests.py
@@ -50,6 +50,41 @@ def add_headers(f, custom_headers, thrift_headers):
         f.write(f'#include "{header}"\n')
 
 
+def add_test_getters(f, case_name, case):
+    param_types = ", ".join(
+        f"std::remove_cvref_t<{param}>" for param in case["param_types"]
+    )
+    if "arg_types" in case:
+        arg_types = ", ".join(case["arg_types"])
+    else:
+        arg_types = param_types
+
+    f.write(
+        f"\n"
+        f"  std::tuple<{arg_types}> get_{case_name}() {{\n"
+        f'{case["setup"]}\n'
+        f"  }}\n"
+    )
+
+
+def get_param_str(param, i):
+    if "]" in param:
+        # Array param
+
+        if ")" in param:
+            # "int(&)[5]" ->  "int (&a0)[5]"
+            start, end = param.split(")")
+            return f"{start}a{i}){end}"
+
+        # "int[5]" -> "int a0[5]"
+        # "int[5][10]" -> "int a0[5][10]"
+        type_name, array_size = param.split("[", 1)
+        return f"{type_name} a{i}[{array_size}"
+
+    # Non-array param, e.g. "int&" -> "int& a0"
+    return f"{param} a{i}"
+
+
 def add_test_setup(f, config):
     ns = get_namespace(config["suite"])
     # fmt: off
@@ -64,23 +99,6 @@ def add_test_setup(f, config):
     )
     # fmt: on
 
-    def get_param_str(param, i):
-        if "]" in param:
-            # Array param
-
-            if ")" in param:
-                # "int(&)[5]" ->  "int (&a0)[5]"
-                start, end = param.split(")")
-                return f"{start}a{i}){end}"
-
-            # "int[5]" -> "int a0[5]"
-            # "int[5][10]" -> "int a0[5][10]"
-            type_name, array_size = param.split("[", 1)
-            return f"{type_name} a{i}[{array_size}"
-
-        # Non-array param, e.g. "int&" -> "int& a0"
-        return f"{param} a{i}"
-
     def define_traceable_func(name, params, body):
         return (
             f"\n"
@@ -98,21 +116,7 @@ def add_test_setup(f, config):
             # target func for it
             continue
 
-        # generate getter for an object of this type
-        param_types = ", ".join(
-            f"std::remove_cvref_t<{param}>" for param in case["param_types"]
-        )
-        if "arg_types" in case:
-            arg_types = ", ".join(case["arg_types"])
-        else:
-            arg_types = param_types
-
-        f.write(
-            f"\n"
-            f"  std::tuple<{arg_types}> get_{case_name}() {{\n"
-            f'{case["setup"]}\n'
-            f"  }}\n"
-        )
+        add_test_getters(f, case_name, case)
 
         # generate oid and oil targets
         params_str = ", ".join(
@@ -265,6 +269,7 @@ def add_tests(f, config):
     for case_name, case in config["cases"].items():
         add_oid_integration_test(f, config, case_name, case)
         add_oil_integration_test(f, config, case_name, case)
+        add_oilgen_integration_test(f, config, case_name, case)
 
 
 def add_oid_integration_test(f, config, case_name, case):
@@ -399,6 +404,86 @@ def add_oil_integration_test(f, config, case_name, case):
     f.write(f"}}\n")
 
 
+def add_oilgen_integration_test(f, config, case_name, case):
+    case_str = get_case_name(config["suite"], case_name)
+    exit_code = case.get("expect_oil_exit_code", 0)
+
+    if "oil_disable" in case or "target_function" in case:
+        return
+
+    config_prefix = case.get("config_prefix", "")
+    config_suffix = case.get("config_suffix", "")
+
+    f.write(
+        f"\n"
+        f"TEST_F(OilgenIntegration, {case_str}) {{\n"
+        f"{generate_skip(case, 'oil')}"
+    )
+
+    f.write('  constexpr std::string_view targetSrc = R"--(')
+    headers = set(config.get("includes", []))
+    add_headers(f, sorted(headers), [])
+
+    f.write(
+        f"\n"
+        f'{config.get("raw_definitions", "")}\n'
+        f"#pragma clang diagnostic push\n"
+        f'#pragma clang diagnostic ignored "-Wunused-private-field"\n'
+        f'{config.get("definitions", "")}\n'
+        f"#pragma clang diagnostic pop\n"
+    )
+    add_test_getters(f, case_name, case)
+
+    main = "int main() {\n"
+    main += "  auto pr = oi::exporters::Json(std::cout);\n"
+    main += "  pr.setPretty(true);\n"
+    main += f"  auto val = get_{case_name}();\n"
+    for i in range(len(case["param_types"])):
+        main += f"  auto ret{i} = oi::introspect(std::get<{i}>(val));\n"
+        main += f"  pr.print(ret{i});\n"
+    main += "}\n"
+
+    f.write(main)
+    f.write(')--";\n')
+
+    f.write(
+        f'  std::string configPrefix = R"--({config_prefix})--";\n'
+        f'  std::string configSuffix = R"--({config_suffix})--";\n'
+        f"  ba::io_context ctx;\n"
+        f"  auto target = runOilgenTarget({{\n"
+        f"    .ctx = ctx,\n"
+        f"    .targetSrc = targetSrc,\n"
+        f"  }}, std::move(configPrefix), std::move(configSuffix));\n\n"
+        f"  ASSERT_EQ(exit_code(target), {exit_code});\n"
+    )
+
+    key = "expect_json"
+    if "expect_json_v2" in case:
+        key = "expect_json_v2"
+    if key in case:
+        try:
+            json.loads(case[key])
+        except json.decoder.JSONDecodeError as error:
+            print(
+                f"\x1b[31m`expect_json` value for test case {config['suite']}.{case_name} was invalid JSON: {error}\x1b[0m",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+        f.write(
+            f"\n"
+            f"  std::stringstream expected_json_ss;\n"
+            f'  expected_json_ss << R"--({case[key]})--";\n'
+            f"  auto result_json_ss = std::stringstream(stdout_);\n"
+            f"  bpt::ptree expected_json, actual_json;\n"
+            f"  bpt::read_json(expected_json_ss, expected_json);\n"
+            f"  bpt::read_json(result_json_ss, actual_json);\n"
+            f"  compare_json(expected_json, actual_json);\n"
+        )
+
+    f.write(f"}}\n")
+
+
 def generate_skip(case, specific):
     possibly_skip = ""
     skip_reason = case.get("skip", False)
diff --git a/test/integration/runner_common.cpp b/test/integration/runner_common.cpp
index c86bd59..a0d8d4d 100644
--- a/test/integration/runner_common.cpp
+++ b/test/integration/runner_common.cpp
@@ -445,3 +445,164 @@ Proc OilIntegration::runOilTarget(OilOpts opts,
               std::move(std_out),
               std::move(std_err)};
 }
+
+std::string OilgenIntegration::TmpDirStr() {
+  return std::string("/tmp/oilgen-integration-XXXXXX");
+}
+
+Proc OilgenIntegration::runOilgenTarget(OilgenOpts opts,
+                                        std::string configPrefix,
+                                        std::string configSuffix) {
+  // Run an oilgen test in three stages.
+  // 1. Generate the OIL implementation .o from the input source.
+  // 2. Compile the input source and link to the implementation.
+  // 3. Run the now complete target.
+
+  auto path = workingDir / "input_src.cpp";
+  {
+    std::ofstream file{path, std::ios_base::app};
+    file << opts.targetSrc;
+  }
+
+  std::string generatorExe =
+      std::string(OILGEN_EXE_PATH) + " " + path.native() + " ";
+  if (auto prefix = writeCustomConfig("prefix", configPrefix)) {
+    generatorExe += "--config-file ";
+    generatorExe += *prefix;
+    generatorExe += " ";
+  }
+  generatorExe += "--config-file ";
+  generatorExe += configFile;
+  if (auto suffix = writeCustomConfig("suffix", configSuffix)) {
+    generatorExe += " ";
+    generatorExe += "--config-file ";
+    generatorExe += *suffix;
+  }
+
+  // TODO: get this from the CMake arguments for integration_test_target.cpp
+  // somehow
+  // clang-format off
+  std::vector<std::string> clangArgs{
+      "-DOIL_AOT_COMPILATION=1",
+      "-isystem",
+      "/data/users/jakehillion/object-introspection-sl/include",
+      "--std=c++20",
+      "-resource-dir",
+      "/usr/lib64/clang/15.0.7",
+      "-fpic",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../include/c++/12",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../include/c++/12/x86_64-redhat-linux",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../include/c++/12/backward",
+      "-isystem",
+      "/usr/lib64/clang/15.0.7/include",
+      "-isystem",
+      "/usr/local/include",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../x86_64-redhat-linux/include",
+      "-isystem",
+      "/include",
+      "-isystem",
+      "/usr/include",
+      "-fdebug-compilation-dir=/data/users/jakehillion/Downloads/libclang-oilgen-testing",
+      "-fgnuc-version=4.2.1",
+      "-fcxx-exceptions",
+      "-fexceptions",
+      "-faddrsig",
+      "-D__GCC_HAVE_DWARF2_CFI_ASM=1",
+      "-x",
+      "c++",
+  };
+  // clang-format on
+  for (const auto& arg : clangArgs) {
+    generatorExe += " --extra-arg=";
+    generatorExe += arg;
+  }
+
+  if (verbose) {
+    std::cerr << "Running: " << generatorExe << std::endl;
+  }
+
+  bp::child generatorProc{generatorExe, opts.ctx};
+  generatorProc.wait();
+  if (generatorProc.exit_code() != 0)
+    throw std::runtime_error("generation failed!");
+
+  std::string compilerExe =
+      std::string(CXX) +
+      " a.o input_src.cpp "
+      "/data/users/jakehillion/object-introspection-sl/oi/exporters/Json.cpp "
+      "/data/users/jakehillion/object-introspection-sl/oi/"
+      "IntrospectionResult.cpp "
+      "/data/users/jakehillion/object-introspection-sl/oi/exporters/"
+      "ParsedData.cpp";
+  for (const auto& arg : clangArgs) {
+    compilerExe += ' ';
+    compilerExe += arg;
+  }
+
+  if (verbose) {
+    std::cerr << "Running: " << compilerExe << std::endl;
+  }
+
+  bp::child compilerProc{compilerExe, opts.ctx};
+  compilerProc.wait();
+  if (compilerProc.exit_code() != 0)
+    throw std::runtime_error("compilation failed");
+
+  std::string targetExe = "./a.out";
+  if (verbose) {
+    std::cerr << "Running: " << targetExe << std::endl;
+  }
+
+  // Use tee to write the output to files. If verbose is on, also redirect the
+  // output to stderr.
+  bp::async_pipe std_out_pipe(opts.ctx), std_err_pipe(opts.ctx);
+  bp::child std_out, std_err;
+  if (verbose) {
+    // clang-format off
+    std_out = bp::child(bp::search_path("tee"),
+                        (workingDir / "stdout").string(),
+                        bp::std_in < std_out_pipe,
+                        bp::std_out > stderr,
+                        opts.ctx);
+    std_err = bp::child(bp::search_path("tee"),
+                        (workingDir / "stderr").string(),
+                        bp::std_in < std_err_pipe,
+                        bp::std_out > stderr,
+                        opts.ctx);
+    // clang-format on
+  } else {
+    // clang-format off
+    std_out = bp::child(bp::search_path("tee"),
+                        (workingDir / "stdout").string(),
+                        bp::std_in < std_out_pipe,
+                        bp::std_out > bp::null,
+                        opts.ctx);
+    std_err = bp::child(bp::search_path("tee"),
+                        (workingDir / "stderr").string(),
+                        bp::std_in < std_err_pipe,
+                        bp::std_out > bp::null,
+                        opts.ctx);
+    // clang-format on
+  }
+
+  /* Spawn `oid` with tracing on and IOs redirected */
+  // clang-format off
+  bp::child targetProc(
+      targetExe,
+      bp::std_in  < bp::null,
+      bp::std_out > std_out_pipe,
+      bp::std_err > std_err_pipe,
+      opts.ctx);
+  // clang-format on
+
+  return Proc{
+      opts.ctx,
+      std::move(targetProc),
+      std::move(std_out),
+      std::move(std_err),
+  };
+}
diff --git a/test/integration/runner_common.h b/test/integration/runner_common.h
index d2b5e57..8c6c8eb 100644
--- a/test/integration/runner_common.h
+++ b/test/integration/runner_common.h
@@ -22,6 +22,11 @@ struct OilOpts {
   std::string targetArgs;
 };
 
+struct OilgenOpts {
+  boost::asio::io_context& ctx;
+  std::string_view targetSrc;
+};
+
 struct Proc {
   boost::asio::io_context& ctx;
   boost::process::child proc;
@@ -81,3 +86,12 @@ class OilIntegration : public IntegrationBase {
                     std::string configPrefix,
                     std::string configSuffix);
 };
+
+class OilgenIntegration : public IntegrationBase {
+ protected:
+  std::string TmpDirStr() override;
+
+  Proc runOilgenTarget(OilgenOpts opts,
+                       std::string configPrefix,
+                       std::string configSuffix);
+};

From e38d8625a43d84f7ef7a4c97d06e1ad654f61ebd Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 2 Jan 2024 18:35:03 +0000
Subject: [PATCH 097/188] tbv2: calculate total memory footprint

TODO: Review and clean up any todos.
TODO: Add size to all expect_json_v2.

Add the option to calculate total size (inclusive size) by wrapping the
existing iterator. This change provides a new iterator, `SizedIterator`, which
wraps an existing iterator and adds a new field `size` to the output element.

This is achieved with a two pass algorithm on the existing iterator:
1. Gather metadata for each element. This includes the total size up until that
   element and the range of elements that should be included in the size.
2. Return the result from the underlying iterator with the additional
   field.

Test plan:
- Added to the integration tests for full coverage.
---
 include/oi/exporters/Json.h                   | 141 +++++++++++++++-
 include/oi/result/SizedResult-inl.h           | 150 ++++++++++++++++++
 include/oi/result/SizedResult.h               |  84 ++++++++++
 oi/CMakeLists.txt                             |   4 -
 oi/exporters/Json.cpp                         | 146 -----------------
 test/integration/CMakeLists.txt               |   2 +-
 test/integration/anonymous.toml               |  15 +-
 test/integration/arrays.toml                  |  14 +-
 test/integration/enums.toml                   |  16 +-
 test/integration/enums_params.toml            |  10 +-
 test/integration/fbstring.toml                |   7 +
 test/integration/folly_f14_fast_map.toml      |  26 +--
 test/integration/folly_f14_fast_set.toml      |   9 +-
 test/integration/folly_f14_node_map.toml      |  26 +--
 test/integration/folly_f14_node_set.toml      |   9 +-
 test/integration/folly_f14_value_map.toml     |  26 +--
 test/integration/folly_f14_value_set.toml     |   9 +-
 test/integration/folly_f14_vector_map.toml    |  26 +--
 test/integration/folly_f14_vector_set.toml    |   9 +-
 test/integration/folly_small_vector.toml      |  28 ++--
 test/integration/folly_sorted_vector_map.toml |  30 ++--
 test/integration/gen_tests.py                 |   5 +-
 test/integration/inheritance_access.toml      |  22 ++-
 test/integration/inheritance_multiple.toml    |  13 +-
 test/integration/packed.toml                  |   8 +-
 test/integration/padding.toml                 |  26 +--
 test/integration/pointers.toml                |  32 ++--
 test/integration/primitives.toml              |  38 ++---
 test/integration/runner_common.cpp            |   2 +
 test/integration/simple.toml                  |  20 ++-
 test/integration/sorted_vector_set.toml       |   2 +
 test/integration/std_list_del_allocator.toml  |   5 +-
 test/integration/std_vector.toml              |  18 +--
 33 files changed, 631 insertions(+), 347 deletions(-)
 create mode 100644 include/oi/result/SizedResult-inl.h
 create mode 100644 include/oi/result/SizedResult.h
 delete mode 100644 oi/exporters/Json.cpp

diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h
index 31aa808..7cf4910 100644
--- a/include/oi/exporters/Json.h
+++ b/include/oi/exporters/Json.h
@@ -17,8 +17,10 @@
 #define INCLUDED_OI_EXPORTERS_JSON_H 1
 
 #include <oi/IntrospectionResult.h>
+#include <oi/result/SizedResult.h>
 
 #include <ostream>
+#include <string_view>
 
 namespace oi::exporters {
 
@@ -26,19 +28,152 @@ class Json {
  public:
   Json(std::ostream& out);
 
-  void print(const IntrospectionResult&);
-  void print(IntrospectionResult::const_iterator& it,
-             IntrospectionResult::const_iterator& end);
+  template <typename Res>
+  void print(const Res& r) {
+    auto begin = r.begin();
+    auto end = r.end();
+    return print(begin, end);
+  }
+  template <typename It>
+  void print(It& it, const It& end);
 
   void setPretty(bool pretty) {
     pretty_ = pretty;
   }
 
  private:
+  std::string_view tab() const;
+  std::string_view space() const;
+  std::string_view endl() const;
+  static std::string makeIndent(size_t depth);
+
+  void printStringField(std::string_view name, std::string_view value, std::string_view indent);
+  void printBoolField(std::string_view name, bool value, std::string_view indent);
+  void printUnsignedField(std::string_view name, uint64_t value, std::string_view indent);
+  void printPointerField(std::string_view name, uintptr_t value, std::string_view indent);
+  template <typename Rng>
+  void printListField(std::string_view name, const Rng& range, std::string_view indent);
+
+  void printFields(const result::Element&, std::string_view indent);
+  template <typename El>
+  void printFields(const result::SizedElement<El>&, std::string_view indent);
+
   bool pretty_ = false;
   std::ostream& out_;
 };
 
+inline Json::Json(std::ostream& out) : out_(out) {
+}
+
+inline std::string_view Json::tab() const {
+  return pretty_ ? "  " : "";
+}
+inline std::string_view Json::space() const {
+  return pretty_ ? " " : "";
+}
+inline std::string_view Json::endl() const {
+  return pretty_ ? "\n" : "";
+}
+inline std::string Json::makeIndent(size_t depth) {
+  depth = std::max(depth, 1UL);
+  return std::string((depth - 1) * 4, ' ');
+}
+
+inline void Json::printStringField(std::string_view name,
+                                   std::string_view value, std::string_view indent) {
+  out_ << tab() << '"' << name << '"' << ':' << space() << "\"" << value
+       << "\"," << endl() << indent;
+}
+inline void Json::printBoolField(std::string_view name, bool value, std::string_view indent) {
+  out_ << tab() << '"' << name << "\":" << space() << value << ',' << endl()
+       << indent;
+}
+inline void Json::printUnsignedField(std::string_view name, uint64_t value, std::string_view indent) {
+  out_ << tab() << '"' << name << "\":" << space() << value << ',' << endl()
+       << indent;
+}
+inline void Json::printPointerField(std::string_view name, uintptr_t value, std::string_view indent) {
+  out_ << tab() << '"' << name << "\":" << space() << "\"0x" << std::hex
+       << value << std::dec << "\"," << endl() << indent;
+}
+template <typename Rng>
+void Json::printListField(std::string_view name, const Rng& range, std::string_view indent) {
+  out_ << tab() << '"' << name << '"' << ':' << space() << '[';
+  bool first = true;
+  for (const auto& el : range) {
+    if (!std::exchange(first, false))
+      out_ << ',' << space();
+    out_ << '"' << el << '"';
+  }
+  out_ << "]," << endl() << indent;
+}
+
+template <typename El>
+void Json::printFields(const result::SizedElement<El>& el, std::string_view indent) {
+  printUnsignedField("size", el.size, indent);
+
+  printFields(el.inner(), indent);
+}
+
+inline void Json::printFields(const result::Element& el, std::string_view indent) {
+  printStringField("name", el.name, indent);
+  printListField("typePath", el.type_path, indent);
+  printListField("typeNames", el.type_names, indent);
+  printUnsignedField("staticSize", el.static_size, indent);
+  printUnsignedField("exclusiveSize", el.exclusive_size, indent);
+  if (el.pointer.has_value())
+    printUnsignedField("pointer", *el.pointer, indent);
+
+  if (const auto* s = std::get_if<result::Element::Scalar>(&el.data)) {
+    printUnsignedField("data", s->n, indent);
+  } else if (const auto* p = std::get_if<result::Element::Pointer>(&el.data)) {
+    printPointerField("data", p->p, indent);
+  } else if (const auto* str = std::get_if<std::string>(&el.data)) {
+    printStringField("data", *str, indent);
+  }
+
+  if (el.container_stats.has_value()) {
+    printUnsignedField("length", el.container_stats->length, indent);
+    printUnsignedField("capacity", el.container_stats->capacity, indent);
+  }
+  if (el.is_set_stats.has_value())
+    printUnsignedField("is_set", el.is_set_stats->is_set, indent);
+}
+
+template <typename It>
+void Json::print(It& it, const It& end) {
+  const auto depth = it->type_path.size();
+
+  const auto thisIndent = pretty_ ? makeIndent(depth) : "";
+  const auto lastIndent = pretty_ ? makeIndent(depth - 1) : "";
+
+  out_ << '[' << endl() << thisIndent;
+
+  bool first = true;
+  while (it != end && it->type_path.size() >= depth) {
+    if (!std::exchange(first, false))
+      out_ << ',' << endl() << thisIndent;
+
+    out_ << '{' << endl() << thisIndent;
+
+    printFields(*it, thisIndent);
+
+    out_ << tab() << "\"members\":" << space();
+    if (++it != end && it->type_path.size() > depth) {
+      print(it, end);
+    } else {
+      out_ << "[]" << endl();
+    }
+
+    out_ << thisIndent << "}";
+  }
+  if (depth == 1) {
+    out_ << endl() << ']' << endl();
+  } else {
+    out_ << endl() << lastIndent << tab() << ']' << endl();
+  }
+}
+
 }  // namespace oi::exporters
 
 #endif
diff --git a/include/oi/result/SizedResult-inl.h b/include/oi/result/SizedResult-inl.h
new file mode 100644
index 0000000..ded348b
--- /dev/null
+++ b/include/oi/result/SizedResult-inl.h
@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#if defined(INCLUDED_OI_RESULT_SIZED_ELEMENT_INL_H) || \
+    !defined(INCLUDED_OI_RESULT_SIZED_ELEMENT_H)
+static_assert(false,
+              "SizedResult-inl.h provides inline declarations for "
+              "SizedResult.h and should only "
+              "be included by SizedResult.h");
+#endif
+#define INCLUDED_OI_RESULT_SIZED_ELEMENT_INL_H 1
+
+#include <iostream>   // TODO: remove
+#include <stdexcept>  // TODO: remove
+#include <utility>
+#include <vector>
+
+#include "SizedResult.h"
+
+namespace oi::result {
+
+template <typename Res>
+SizedResult<Res>::SizedResult(Res res) : res_{std::move(res)} {
+}
+
+template <typename Res>
+typename SizedResult<Res>::const_iterator SizedResult<Res>::begin() const {
+  return ++const_iterator{res_.begin(), res_.end()};
+}
+template <typename Res>
+typename SizedResult<Res>::const_iterator SizedResult<Res>::end() const {
+  return res_.end();
+}
+
+template <typename Res>
+SizedResult<Res>::const_iterator::const_iterator(It it, const It& end)
+    : data_{it} {
+  struct StackEntry {
+    size_t index;
+    size_t depth;
+  };
+  std::vector<StackEntry> stack;
+
+  size_t size = 0;
+  size_t count = -1;
+  for (; it != end; ++it) {
+    ++count;
+
+    auto depth = it->type_path.size();
+    while (!stack.empty() && stack.back().depth >= depth) {
+      auto i = stack.back().index;
+      stack.pop_back();
+      helpers_[i].last_child = count - 1;
+    }
+
+    size += it->exclusive_size;
+    helpers_.emplace_back(SizeHelper{.size = size});
+    stack.emplace_back(StackEntry{.index = count, .depth = depth});
+  }
+  while (!stack.empty()) {
+    auto i = stack.back().index;
+    stack.pop_back();
+    helpers_[i].last_child = count;
+  }
+}
+
+template <typename Res>
+SizedResult<Res>::const_iterator::const_iterator(It end) : data_{end} {
+}
+
+template <typename Res>
+typename SizedResult<Res>::const_iterator
+SizedResult<Res>::const_iterator::operator++(int) {
+  auto old = *this;
+  operator++();
+  return old;
+}
+
+template <typename Res>
+typename SizedResult<Res>::const_iterator&
+SizedResult<Res>::const_iterator::operator++() {
+  // The below iterator is already pointing at the first element while this
+  // iterator is not. Skip incrementing it the first time around.
+  if (count_ != 0)
+    ++data_;
+
+  if (count_ == helpers_.size()) {
+    next_.reset();
+    return *this;
+  }
+
+  size_t size = helpers_[helpers_[count_].last_child].size;
+  if (count_ != 0)
+    size -= helpers_[count_ - 1].size;
+
+  next_.emplace(*data_, size);
+  ++count_;
+  return *this;
+}
+
+template <typename Res>
+const typename SizedResult<Res>::Element&
+SizedResult<Res>::const_iterator::operator*() const {
+  // auto* leak = new SizedResult<Res>::Element{*data_, (size_t)-1};
+  // return *leak;
+  return *next_;
+}
+template <typename Res>
+const typename SizedResult<Res>::Element*
+SizedResult<Res>::const_iterator::operator->() const {
+  // auto* leak = new SizedResult<Res>::Element{*data_, (size_t)-1};
+  // return leak;
+  return &*next_;
+}
+
+template <typename Res>
+bool SizedResult<Res>::const_iterator::operator==(
+    const SizedResult<Res>::const_iterator& that) const {
+  return this->data_ == that.data_;
+}
+
+template <typename Res>
+bool SizedResult<Res>::const_iterator::operator!=(
+    const SizedResult<Res>::const_iterator& that) const {
+  return !(*this == that);
+}
+
+template <typename El>
+SizedElement<El>::SizedElement(const El& el, size_t size_)
+    : El{el}, size{size_} {
+}
+
+template <typename El>
+const El& SizedElement<El>::inner() const {
+  return static_cast<const El&>(*this);
+}
+
+}  // namespace oi::result
diff --git a/include/oi/result/SizedResult.h b/include/oi/result/SizedResult.h
new file mode 100644
index 0000000..983b804
--- /dev/null
+++ b/include/oi/result/SizedResult.h
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef INCLUDED_OI_RESULT_SIZED_ELEMENT_H
+#define INCLUDED_OI_RESULT_SIZED_ELEMENT_H 1
+
+#include <cstddef>
+#include <optional>
+#include <type_traits>
+#include <vector>
+
+namespace oi::result {
+
+template <typename El>
+struct SizedElement : public El {
+  SizedElement(const El& el, size_t size);
+  const El& inner() const;
+
+  size_t size;
+};
+
+template <typename Res>
+class SizedResult {
+ private:
+  using It = std::decay_t<decltype(std::declval<Res&>().begin())>;
+  using ParentEl = std::decay_t<decltype(std::declval<It&>().operator*())>;
+
+ public:
+  using Element = SizedElement<ParentEl>;
+
+ private:
+  struct SizeHelper {
+    size_t size = -1;
+    size_t last_child = -1;
+  };
+
+ public:
+  class const_iterator {
+    friend SizedResult;
+
+   public:
+    bool operator==(const const_iterator& that) const;
+    bool operator!=(const const_iterator& that) const;
+    const Element& operator*() const;
+    const Element* operator->() const;
+    const_iterator& operator++();
+    const_iterator operator++(int);
+
+   private:
+    const_iterator(It start, const It& end);
+    const_iterator(It end);
+
+    It data_;
+
+    std::vector<SizeHelper> helpers_;
+    size_t count_ = 0;
+    std::optional<Element> next_;
+  };
+
+  SizedResult(Res res);
+
+  const_iterator begin() const;
+  const_iterator end() const;
+
+ private:
+  Res res_;
+};
+
+}  // namespace oi::result
+
+#include "SizedResult-inl.h"
+#endif
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index 3754f41..64dbcae 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -55,10 +55,6 @@ target_link_libraries(codegen
   glog::glog
 )
 
-add_library(exporters_json exporters/Json.cpp)
-target_include_directories(exporters_json PUBLIC ${CMAKE_SOURCE_DIR}/include)
-target_link_libraries(exporters_json oil)
-
 add_library(exporters_csv exporters/CSV.cpp)
 target_include_directories(exporters_csv PUBLIC ${CMAKE_SOURCE_DIR}/include)
 target_link_libraries(exporters_csv oil)
diff --git a/oi/exporters/Json.cpp b/oi/exporters/Json.cpp
deleted file mode 100644
index 8a4d33c..0000000
--- a/oi/exporters/Json.cpp
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include <oi/exporters/Json.h>
-
-#include <algorithm>
-#include <stdexcept>
-#include <variant>
-
-template <class>
-inline constexpr bool always_false_v = false;
-
-namespace oi::exporters {
-namespace {
-
-template <typename It>
-void printStringList(std::ostream& out, It it, It end, bool pretty) {
-  out << '[';
-  for (; it != end; ++it) {
-    out << '"' << (*it) << '"';
-    if (it != end - 1)
-      out << (pretty ? ", " : ",");
-  }
-  out << ']';
-}
-
-std::string makeIndent(size_t depth) {
-  depth = std::max(depth, 1UL);
-  return std::string((depth - 1) * 4, ' ');
-}
-
-}  // namespace
-
-Json::Json(std::ostream& out) : out_(out) {
-}
-
-void Json::print(const IntrospectionResult& r) {
-  auto begin = r.cbegin();
-  auto end = r.cend();
-  return print(begin, end);
-}
-
-void Json::print(IntrospectionResult::const_iterator& it,
-                 IntrospectionResult::const_iterator& end) {
-  const auto firstTypePathSize = it->type_path.size();
-
-  const auto indent = pretty_ ? makeIndent(firstTypePathSize) : "";
-  const auto lastIndent =
-      pretty_ ? makeIndent(std::max(firstTypePathSize, 1UL) - 1) : "";
-  const auto* tab = pretty_ ? "  " : "";
-  const auto* space = pretty_ ? " " : "";
-  const auto* endl = pretty_ ? "\n" : "";
-
-  out_ << '[' << endl << indent;
-
-  bool first = true;
-  while (it != end) {
-    if (it->type_path.size() < firstTypePathSize) {
-      // no longer a sibling, must be a sibling of the type we're printing
-      break;
-    }
-
-    if (!first)
-      out_ << ',' << endl << indent;
-    first = false;
-
-    out_ << '{' << endl << indent;
-
-    out_ << tab << "\"name\"" << space << ':' << space << "\"" << it->name
-         << "\"," << endl
-         << indent;
-
-    out_ << tab << "\"typePath\"" << space << ':' << space << "";
-    printStringList(out_, it->type_path.begin(), it->type_path.end(), pretty_);
-    out_ << (pretty_ ? ",\n" : ",") << indent;
-
-    out_ << tab << "\"typeNames\"" << space << ':' << space;
-    printStringList(
-        out_, it->type_names.begin(), it->type_names.end(), pretty_);
-    out_ << ',' << endl << indent;
-
-    out_ << tab << "\"staticSize\":" << space << it->static_size << ',' << endl
-         << indent;
-    out_ << tab << "\"exclusiveSize\":" << space << it->exclusive_size << ','
-         << endl
-         << indent;
-
-    if (it->pointer.has_value()) {
-      out_ << tab << "\"pointer\":" << space << *(it->pointer) << ',' << endl
-           << indent;
-    }
-
-    if (auto* s = std::get_if<result::Element::Scalar>(&it->data)) {
-      out_ << tab << "\"data\":" << space << s->n << ',' << endl << indent;
-    } else if (auto* p = std::get_if<result::Element::Pointer>(&it->data)) {
-      out_ << tab << "\"data\":" << space << "\"0x" << std::hex << p->p
-           << std::dec << "\"," << endl
-           << indent;
-    } else if (auto* str = std::get_if<std::string>(&it->data)) {
-      out_ << tab << "\"data\":" << space << "\"" << *str << "\"," << endl
-           << indent;
-    }
-
-    if (it->container_stats.has_value()) {
-      out_ << tab << "\"length\":" << space << it->container_stats->length
-           << ',' << endl
-           << indent;
-      out_ << tab << "\"capacity\":" << space << it->container_stats->capacity
-           << ',' << endl
-           << indent;
-    }
-    if (it->is_set_stats.has_value()) {
-      out_ << tab << "\"is_set\":" << space << it->is_set_stats->is_set << ','
-           << endl
-           << indent;
-    }
-
-    out_ << tab << "\"members\":" << space;
-    if (++it != end && it->type_path.size() > firstTypePathSize) {
-      print(it, end);
-    } else {
-      out_ << "[]" << endl;
-    }
-
-    out_ << indent << "}";
-  }
-  if (firstTypePathSize == 1) {
-    out_ << endl << ']' << endl;
-  } else {
-    out_ << endl << lastIndent << tab << ']' << endl;
-  }
-}
-
-}  // namespace oi::exporters
diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 12cb239..4d235b5 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -50,7 +50,7 @@ add_executable(integration_test_target
   ${INTEGRATION_TEST_TARGET_SRC}
   folly_shims.cpp)
 target_compile_options(integration_test_target PRIVATE -O1)
-target_link_libraries(integration_test_target PRIVATE oil_jit exporters_json Boost::headers ${Boost_LIBRARIES})
+target_link_libraries(integration_test_target PRIVATE oil_jit Boost::headers ${Boost_LIBRARIES})
 
 add_executable(integration_test_runner ${INTEGRATION_TEST_RUNNER_SRC} runner_common.cpp)
 target_include_directories(integration_test_runner PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
diff --git a/test/integration/anonymous.toml b/test/integration/anonymous.toml
index 6a91293..15b9f10 100644
--- a/test/integration/anonymous.toml
+++ b/test/integration/anonymous.toml
@@ -88,10 +88,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":12,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size": 12,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":4, "dynamicSize":0},
-        {"name":"c", "staticSize":4, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"c", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}
       ]}]'''
 
   [cases.anon_struct]
@@ -207,10 +209,11 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize": 24,
       "exclusiveSize": 10,
+      "size": 24,
       "members": [
-        {"name":"__oi_anon_0", "staticSize":2, "exclusiveSize":2},
-        {"name":"__oi_anon_2", "staticSize":8, "exclusiveSize":8},
-        {"name":"e", "staticSize":4, "exclusiveSize":4, "typeNames":["int32_t"]}
+        {"name":"__oi_anon_0", "staticSize":2, "exclusiveSize":2, "size":2},
+        {"name":"__oi_anon_2", "staticSize":8, "exclusiveSize":8, "size":8},
+        {"name":"e", "staticSize":4, "exclusiveSize":4, "size":4, "typeNames":["int32_t"]}
       ]
     }]'''
 
diff --git a/test/integration/arrays.toml b/test/integration/arrays.toml
index 4a06b13..5074b62 100644
--- a/test/integration/arrays.toml
+++ b/test/integration/arrays.toml
@@ -33,9 +33,11 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":40,
       "exclusiveSize":0,
+      "size":40,
       "members":[{
         "staticSize":40,
         "exclusiveSize":0,
+        "size":40,
         "length":10,
         "capacity":10
       }]}]'''
@@ -56,9 +58,11 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":1,
       "exclusiveSize":1,
+      "size":1,
       "members":[{
         "staticSize":0,
-        "exclusiveSize":0
+        "exclusiveSize":0,
+        "size":1
       }]}]'''
   [cases.multidim_legacy] # Test for legacy behaviour. Remove with OICodeGen
     oil_disable = 'oil only runs on codegen v2'
@@ -86,10 +90,10 @@ definitions = '''
           {"staticSize":12, "dynamicSize":0, "exclusiveSize":12, "length":3, "capacity":3, "elementStaticSize":4}]
       }]}]'''
     expect_json_v2 = '''[
-      {"staticSize":24, "exclusiveSize":0, "members":[
-        {"staticSize":24, "exclusiveSize":0, "length":2, "capacity":2, "members":[
-          {"staticSize":12, "exclusiveSize":0, "length":3, "capacity":3},
-          {"staticSize":12, "exclusiveSize":0, "length":3, "capacity":3}]
+      {"staticSize":24, "exclusiveSize":0, "size":24, "members":[
+        {"staticSize":24, "exclusiveSize":0, "size":24, "length":2, "capacity":2, "members":[
+          {"staticSize":12, "exclusiveSize":0, "size":12, "length":3, "capacity":3},
+          {"staticSize":12, "exclusiveSize":0, "size":12, "length":3, "capacity":3}]
       }]}]'''
   [cases.direct_int10]
     skip = "Direct array arguments don't work"
diff --git a/test/integration/enums.toml b/test/integration/enums.toml
index f112b66..fa91cbc 100644
--- a/test/integration/enums.toml
+++ b/test/integration/enums.toml
@@ -36,31 +36,31 @@ definitions = '''
     param_types = ["ScopedEnum"]
     setup = "return {};"
     expect_json = '[{"staticSize":4, "dynamicSize":0}]'
-    expect_json_v2 = '[{"typeNames": ["ScopedEnum"], "staticSize":4, "exclusiveSize":4}]'
+    expect_json_v2 = '[{"typeNames": ["ScopedEnum"], "staticSize":4, "exclusiveSize":4, "size":4}]'
   [cases.scoped_uint8]
     param_types = ["ScopedEnumUint8"]
     setup = "return {};"
     expect_json = '[{"staticSize":1, "dynamicSize":0}]'
-    expect_json_v2 = '[{"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1}]'
+    expect_json_v2 = '[{"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1}]'
   [cases.unscoped]
     param_types = ["UNSCOPED_ENUM"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.anonymous]
     skip = "TreeBuilder crashes" # https://github.com/facebookexperimental/object-introspection/issues/232
     param_types = ["Holder&"]
     setup = "return {};"
     expect_json = '''[
-      {"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "members":[
-        {"name":"e", "staticSize":4, "dynamicSize":0, "exclusiveSize":4}
+      {"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4, "members":[
+        {"name":"e", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}
     ]}]'''
   [cases.paired_with_element]
     param_types = ["Pair<ScopedEnumUint8, uint8_t>"]
     setup = "return {};"
     expect_json = '[{"staticSize":2, "dynamicSize":0}]'
     expect_json_v2 = '''[
-      {"staticSize": 2, "exclusiveSize": 0, "members": [
-        {"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1},
-        {"typeNames": ["uint8_t"], "staticSize":1, "exclusiveSize":1}
+      {"staticSize": 2, "exclusiveSize": 0, "size":2, "members": [
+        {"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1},
+        {"typeNames": ["uint8_t"], "staticSize":1, "exclusiveSize":1, "size":1}
       ]}
     ]'''
diff --git a/test/integration/enums_params.toml b/test/integration/enums_params.toml
index cf61d6f..dd66f31 100644
--- a/test/integration/enums_params.toml
+++ b/test/integration/enums_params.toml
@@ -39,20 +39,20 @@ definitions = '''
     param_types = ["const std::array<int, static_cast<size_t>(MyNS::ScopedEnum::Two)>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":8, "length":2, "capacity":2, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":8, "dynamicSize":0, "length":2, "capacity":2}]'
+    expect_json_v2 = '[{"staticSize":8, "exclusiveSize":0, "size":8, "length":2, "capacity":2}]'
 
   [cases.scoped_enum_val]
     param_types = ["const MyClass<MyNS::ScopedEnum::One>&"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4}]'
   [cases.scoped_enum_val_gaps]
     param_types = ["const ClassGaps<MyNS::EnumWithGaps::Twenty>&"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4}]'
   [cases.scoped_enum_val_negative]
     param_types = ["const ClassGaps<MyNS::EnumWithGaps::MinusTwo>&"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4}]'
 
   [cases.unscoped_enum_type]
     param_types = ["const std::vector<MyNS::UNSCOPED_ENUM>&"]
@@ -61,4 +61,4 @@ definitions = '''
     param_types = ["const std::array<int, MyNS::ONE>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":4, "length":1, "capacity":1, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":4, "length":1, "capacity":1}]'
+    expect_json_v2 = '[{"staticSize":4, "exclusiveSize":0, "size":4, "length":1, "capacity":1}]'
diff --git a/test/integration/fbstring.toml b/test/integration/fbstring.toml
index 6e68d99..cec73ca 100644
--- a/test/integration/fbstring.toml
+++ b/test/integration/fbstring.toml
@@ -28,6 +28,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 24,
+      "size": 24,
       "length": 0,
       "capacity": 23
     }]'''
@@ -60,6 +61,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 24,
+      "size": 24,
       "length": 6,
       "capacity": 23
     }]'''
@@ -92,6 +94,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 50,
+      "size": 50,
       "length": 26,
       "capacity": 26
     }]'''
@@ -124,6 +127,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 1056,
+      "size": 1056,
       "length": 1024,
       "capacity": 1024
     }]'''
@@ -137,17 +141,20 @@ includes = ["folly/FBString.h", "utility"]
     expect_json_v2 = '''[{
       "staticSize": 48,
       "exclusiveSize": 0,
+      "size": 1080,
       "members": [
         {
           "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
           "staticSize": 24,
           "exclusiveSize": 1056,
+          "size": 1056,
           "length": 1024,
           "capacity": 1024
         }, {
           "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
           "staticSize": 24,
           "exclusiveSize": 24,
+          "size": 24,
           "length": 1024,
           "capacity": 1024
         }
diff --git a/test/integration/folly_f14_fast_map.toml b/test/integration/folly_f14_fast_map.toml
index 5f9e509..273c222 100644
--- a/test/integration/folly_f14_fast_map.toml
+++ b/test/integration/folly_f14_fast_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 464,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "size": 72, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "size": 104, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "size": 104,"length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 40,
+          "size": 184,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_fast_set.toml b/test/integration/folly_f14_fast_set.toml
index 486b9ba..24d8abe 100644
--- a/test/integration/folly_f14_fast_set.toml
+++ b/test/integration/folly_f14_fast_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size":304,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "size":56, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "size":88, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "size":72, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "size":88, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_f14_node_map.toml b/test/integration/folly_f14_node_map.toml
index 78d414d..07b5b89 100644
--- a/test/integration/folly_f14_node_map.toml
+++ b/test/integration/folly_f14_node_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size":668,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "size": 96, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "size": 148, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "size": 160, "length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 120,
+          "size": 264,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_node_set.toml b/test/integration/folly_f14_node_set.toml
index 497976c..a68cc7b 100644
--- a/test/integration/folly_f14_node_set.toml
+++ b/test/integration/folly_f14_node_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 500,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 120, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "size": 84, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "size": 128, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "size": 132, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 120, "size": 156, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_f14_value_map.toml b/test/integration/folly_f14_value_map.toml
index fac9829..a1a09e8 100644
--- a/test/integration/folly_f14_value_map.toml
+++ b/test/integration/folly_f14_value_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 464,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "size": 72, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "size": 104, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "size": 104, "length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 40,
+          "size": 184,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_value_set.toml b/test/integration/folly_f14_value_set.toml
index 3f19b31..2cf03df 100644
--- a/test/integration/folly_f14_value_set.toml
+++ b/test/integration/folly_f14_value_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 304,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "size": 56, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "size": 88, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "size": 72, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "size": 88, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_f14_vector_map.toml b/test/integration/folly_f14_vector_map.toml
index 9912141..4126243 100644
--- a/test/integration/folly_f14_vector_map.toml
+++ b/test/integration/folly_f14_vector_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 576,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 64, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 60, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 80, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 64, "size": 88, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 60, "size": 120, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 80, "size": 136, "length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 88,
+          "size": 232,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_vector_set.toml b/test/integration/folly_f14_vector_set.toml
index 3ffad84..c67f649 100644
--- a/test/integration/folly_f14_vector_set.toml
+++ b/test/integration/folly_f14_vector_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 400,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 60, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 64, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 76, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 84, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 60, "size": 72, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 64, "size": 104, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 76, "size": 104, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 84, "size": 120, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_small_vector.toml b/test/integration/folly_small_vector.toml
index 5ae53f3..b97b5ff 100644
--- a/test/integration/folly_small_vector.toml
+++ b/test/integration/folly_small_vector.toml
@@ -4,23 +4,23 @@ includes = ["folly/small_vector.h", "vector"]
     param_types = ["const folly::small_vector<int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "length":0, "capacity":2, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "length":0, "capacity":2}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "size":16, "length":0, "capacity":2}]'
   [cases.int_default_inlined]
     param_types = ["const folly::small_vector<int>&"]
     setup = "return {{1,2}};"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "length":2, "capacity":2, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":8, "length":2, "capacity":2}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":8, "size":16, "length":2, "capacity":2}]'
   [cases.int_default_overflow]
     param_types = ["const folly::small_vector<int>&"]
     setup = "return {{1,2,3,4}};"
     expect_json = '[{"staticSize":16, "dynamicSize":24, "exclusiveSize":40, "length":4, "capacity":6, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":24, "length":4, "capacity":6}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":24, "size":40, "length":4, "capacity":6}]'
 
   [cases.vector_3_empty]
     param_types = ["const folly::small_vector<std::vector<int>, 3>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":80, "dynamicSize":0, "exclusiveSize":80, "length":0, "capacity":3, "elementStaticSize":24}]'
-    expect_json_v2 = '[{"staticSize":80, "exclusiveSize":80, "length":0, "capacity":3}]'
+    expect_json_v2 = '[{"staticSize":80, "exclusiveSize":80, "size":80, "length":0, "capacity":3}]'
   [cases.vector_3_inlined]
     param_types = ["const folly::small_vector<std::vector<int>, 3>&"]
     setup = "return {{ {1,2,3}, {4}, {5,6} }};"
@@ -31,10 +31,10 @@ includes = ["folly/small_vector.h", "vector"]
         {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
     ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":80, "length":3, "exclusiveSize":8, "capacity":3, "members":[
-        {"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3},
-        {"staticSize":24, "exclusiveSize":24, "length":1, "capacity":1},
-        {"staticSize":24, "exclusiveSize":24, "length":2, "capacity":2}
+      {"staticSize":80, "length":3, "exclusiveSize":8, "size":104, "capacity":3, "members":[
+        {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity":3},
+        {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity":1},
+        {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity":2}
     ]}]'''
   [cases.vector_3_overflow]
     param_types = ["const folly::small_vector<std::vector<int>, 3>&"]
@@ -47,15 +47,15 @@ includes = ["folly/small_vector.h", "vector"]
         {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4}
     ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":80, "exclusiveSize":104, "length":4, "capacity":5, "members":[
-        {"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3},
-        {"staticSize":24, "exclusiveSize":24, "length":1, "capacity":1},
-        {"staticSize":24, "exclusiveSize":24, "length":2, "capacity":2},
-        {"staticSize":24, "exclusiveSize":24, "length":1, "capacity":1}
+      {"staticSize":80, "exclusiveSize":104, "size":228, "length":4, "capacity":5, "members":[
+        {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity":3},
+        {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity":1},
+        {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity":2},
+        {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity":1}
     ]}]'''
 
   [cases.int_always_heap]
     param_types = ["const folly::small_vector<int, 0>&"]
     setup = "return {{1}};"
     expect_json = '[{"staticSize":16, "dynamicSize":4, "exclusiveSize":20, "length":1, "capacity":1, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "length":1, "capacity":1}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "size":20, "length":1, "capacity":1}]'
diff --git a/test/integration/folly_sorted_vector_map.toml b/test/integration/folly_sorted_vector_map.toml
index 6d3a2a5..44f91e0 100644
--- a/test/integration/folly_sorted_vector_map.toml
+++ b/test/integration/folly_sorted_vector_map.toml
@@ -4,19 +4,19 @@ includes = ["folly/sorted_vector_types.h", "vector"]
     param_types = ["const folly::sorted_vector_map<int, int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "exclusiveSize":24, "length":0, "capacity":0, "elementStaticSize":8}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size":24, "length":0, "capacity":0, "members":[]}]'
   [cases.int_int_some]
     param_types = ["const folly::sorted_vector_map<int, int>&"]
     setup = "return {{ {1,2}, {3,4} }};"
     expect_json = '[{"staticSize":24, "dynamicSize":16, "exclusiveSize":40, "length":2, "capacity":2, "elementStaticSize":8}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":2, "capacity":2, "members":[
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":40, "length":2, "capacity":2, "members":[
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]},
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}
     ]}]'''
   [cases.int_int_reserve]
@@ -27,14 +27,14 @@ includes = ["folly/sorted_vector_types.h", "vector"]
       return m;
     '''
     expect_json = '[{"staticSize":24, "dynamicSize":80, "exclusiveSize":104, "length":2, "capacity":10, "elementStaticSize":8}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":88, "length":2, "capacity":10, "members":[
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":88, "size":104, "length":2, "capacity":10, "members":[
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]},
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}
     ]}]'''
 
diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py
index 2dbc3ff..b72e371 100644
--- a/test/integration/gen_tests.py
+++ b/test/integration/gen_tests.py
@@ -40,6 +40,7 @@ def add_headers(f, custom_headers, thrift_headers):
 
 #include <oi/oi.h>
 #include <oi/exporters/Json.h>
+#include <oi/result/SizedResult.h>
 
 """
     )
@@ -145,8 +146,8 @@ def add_test_setup(f, config):
         oil_func_body += "    auto pr = oi::exporters::Json(std::cout);\n"
         oil_func_body += "    pr.setPretty(true);\n"
         for i in range(len(case["param_types"])):
-            oil_func_body += f"    auto ret{i} = oi::setupAndIntrospect(a{i}, opts);\n"
-            oil_func_body += f"    pr.print(*ret{i});\n"
+            oil_func_body += f"    auto ret{i} = oi::result::SizedResult(*oi::setupAndIntrospect(a{i}, opts));\n"
+            oil_func_body += f"    pr.print(ret{i});\n"
 
         f.write(
             define_traceable_func(
diff --git a/test/integration/inheritance_access.toml b/test/integration/inheritance_access.toml
index faeb35c..9751052 100644
--- a/test/integration/inheritance_access.toml
+++ b/test/integration/inheritance_access.toml
@@ -21,27 +21,33 @@ definitions = '''
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"},
-        {"name":"public_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"},
+        {"name":"public_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
   [cases.protected]
     param_types = ["const Protected&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"},
-        {"name":"protected_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"},
+        {"name":"protected_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
   [cases.private]
     param_types = ["const Private&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"},
-        {"name":"private_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"},
+        {"name":"private_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
   [cases.public_as_base]
     param_types = ["const Base&"]
@@ -49,6 +55,8 @@ definitions = '''
     setup = "return {};"
     expect_json = '''[{
       "staticSize":4,
+      "exclusiveSize":0,
+      "size":4,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
diff --git a/test/integration/inheritance_multiple.toml b/test/integration/inheritance_multiple.toml
index 61d725f..53f956f 100644
--- a/test/integration/inheritance_multiple.toml
+++ b/test/integration/inheritance_multiple.toml
@@ -37,11 +37,12 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":24,
       "exclusiveSize":0,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "exclusiveSize":4},
-        {"name":"b", "staticSize":4, "exclusiveSize":4},
-        {"name":"c", "staticSize":4, "exclusiveSize":4},
-        {"name":"d", "staticSize":4, "exclusiveSize":4},
-        {"name":"e", "staticSize":4, "exclusiveSize":4},
-        {"name":"f", "staticSize":4, "exclusiveSize":4}
+        {"name":"a", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"c", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"d", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"e", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"f", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}]'''
diff --git a/test/integration/packed.toml b/test/integration/packed.toml
index 2bbef02..0e45479 100644
--- a/test/integration/packed.toml
+++ b/test/integration/packed.toml
@@ -12,8 +12,10 @@ definitions = '''
     expect_json = '''[{
       "staticSize":17,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size":17,
       "members":[
-        {"name":"p", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":1, "dynamicSize":0},
-        {"name":"x", "staticSize":8, "dynamicSize":0}
+        {"name":"p", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1},
+        {"name":"x", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
diff --git a/test/integration/padding.toml b/test/integration/padding.toml
index 6cc17d1..5001be6 100644
--- a/test/integration/padding.toml
+++ b/test/integration/padding.toml
@@ -65,10 +65,11 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize": 7,
       "members":[
-        { "name":"a", "staticSize":8, "dynamicSize":0 },
-        { "name":"b", "staticSize":1, "dynamicSize":0 },
-        { "name":"c", "staticSize":8, "dynamicSize":0 }
+        { "name":"a", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
+        { "name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1 },
+        { "name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 }
       ]}]'''
 
   [cases.nested_padding]
@@ -77,18 +78,22 @@ definitions = '''
     expect_json = '''[{
       "staticSize":48,
       "dynamicSize":0,
+      "exclusiveSize":7,
+      "size":48,
       "members":[
-        { "name":"a", "staticSize":8, "dynamicSize":0 },
-        { "name":"b", "staticSize":1, "dynamicSize":0 },
-        { "name":"c", "staticSize":8, "dynamicSize":0 },
+        { "name":"a", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
+        { "name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1 },
+        { "name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
         {
           "name":"d",
           "staticSize":24,
           "dynamicSize":0,
+          "exclusiveSize":7,
+          "size":24,
           "members": [
-            { "name":"a", "staticSize":8, "dynamicSize":0 },
-            { "name":"b", "staticSize":1, "dynamicSize":0 },
-            { "name":"c", "staticSize":8, "dynamicSize":0 }
+            { "name":"a", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
+            { "name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1 },
+            { "name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 }
         ]}
       ]}]'''
 
@@ -103,5 +108,6 @@ definitions = '''
     }]'''
     expect_json_v2 = '''[{
       "staticSize": 104,
-      "exclusiveSize": 32
+      "exclusiveSize": 32,
+      "size": 104
     }]'''
diff --git a/test/integration/pointers.toml b/test/integration/pointers.toml
index 389ba29..71bf2ad 100644
--- a/test/integration/pointers.toml
+++ b/test/integration/pointers.toml
@@ -153,10 +153,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize":4,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_primitive_ptrs_null]
     param_types = ["const PrimitivePtrs&"]
@@ -165,10 +167,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize":4,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
 
 
@@ -189,8 +193,10 @@ definitions = '''
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"vec", "staticSize":8, "dynamicSize":0}
+        {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_vector_ptr_null]
     param_types = ["const VectorPtr&"]
@@ -199,8 +205,10 @@ definitions = '''
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"vec", "staticSize":8, "dynamicSize":0}
+        {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
 
 
@@ -242,10 +250,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize":4,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.feature_config]
     oil_disable = "oil can't chase raw pointers safely"
diff --git a/test/integration/primitives.toml b/test/integration/primitives.toml
index 429712f..94784fb 100644
--- a/test/integration/primitives.toml
+++ b/test/integration/primitives.toml
@@ -2,76 +2,76 @@
   [cases.short]
     param_types = ["short"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":2, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":2, "dynamicSize":0, "exclusiveSize":2, "size":2}]'
   [cases.ushort]
     param_types = ["unsigned short"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":2, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":2, "dynamicSize":0, "exclusiveSize":2, "size":2}]'
   [cases.int]
     param_types = ["int"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.uint]
     param_types = ["unsigned int"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.long]
     param_types = ["long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.ulong]
     param_types = ["unsigned long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.longlong]
     param_types = ["long long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.ulonglong]
     param_types = ["unsigned long long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.bool]
     param_types = ["bool"]
     setup = "return true;"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.char]
     param_types = ["char"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.uchar]
     param_types = ["unsigned char"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.schar]
     param_types = ["signed char"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.wchar_t]
     param_types = ["wchar_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.char8_t]
     param_types = ["char8_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.char16_t]
     param_types = ["char16_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":2, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":2, "dynamicSize":0, "exclusiveSize":2, "size":2}]'
   [cases.char32_t]
     param_types = ["char32_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.float]
     param_types = ["float"]
     setup = "return 3.14;"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.double]
     param_types = ["double"]
     setup = "return 3.14;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.long_double]
     param_types = ["long double"]
     setup = "return 3.14;"
-    expect_json = '[{"staticSize":16, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "size":16}]'
diff --git a/test/integration/runner_common.cpp b/test/integration/runner_common.cpp
index c86bd59..3c8525a 100644
--- a/test/integration/runner_common.cpp
+++ b/test/integration/runner_common.cpp
@@ -365,6 +365,8 @@ void IntegrationBase::compare_json(const bpt::ptree& expected_json,
         }
       } else if (key == "dynamicSize" && val.get_value<size_t>() == 0) {
         continue;
+      } else if (key == "size") {
+        continue;
       }
 
       ADD_FAILURE() << "Expected key not found in output: " << curr_key;
diff --git a/test/integration/simple.toml b/test/integration/simple.toml
index 71a215a..0945f70 100644
--- a/test/integration/simple.toml
+++ b/test/integration/simple.toml
@@ -23,10 +23,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
+      "exclusiveSize": 3,
+      "size": 16,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":1, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
+        {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
   [cases.class]
     param_types = ["const SimpleClass&"]
@@ -34,15 +36,19 @@ definitions = '''
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
+      "exclusiveSize": 3,
+      "size": 16,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":1, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
+        {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
   [cases.union]
     param_types = ["const SimpleUnion&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
-      "dynamicSize":0
+      "dynamicSize":0,
+      "exclusiveSize":8,
+      "size":8
     }]'''
diff --git a/test/integration/sorted_vector_set.toml b/test/integration/sorted_vector_set.toml
index 8e37abd..0a4e416 100644
--- a/test/integration/sorted_vector_set.toml
+++ b/test/integration/sorted_vector_set.toml
@@ -23,6 +23,7 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize": 24,
       "exclusiveSize": 24,
+      "size": 24,
       "length": 0,
       "capacity": 0
     }]'''
@@ -56,6 +57,7 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize": 24,
       "exclusiveSize": 28,
+      "size": 40,
       "length": 3,
       "capacity": 4
     }]'''
diff --git a/test/integration/std_list_del_allocator.toml b/test/integration/std_list_del_allocator.toml
index b986996..c85d3ab 100644
--- a/test/integration/std_list_del_allocator.toml
+++ b/test/integration/std_list_del_allocator.toml
@@ -60,7 +60,8 @@ includes = ["list"]
     expect_json_v2 = '''[{
       "staticSize": 48,
       "exclusiveSize": 0,
+      "size": 60,
       "members": [
-        {"name": "v1", "staticSize": 24, "exclusiveSize": 24, "length": 1, "capacity": 1},
-        {"name": "v2", "staticSize": 24, "exclusiveSize": 24, "length": 2, "capacity": 2}
+        {"name": "v1", "staticSize": 24, "exclusiveSize": 24, "size": 28, "length": 1, "capacity": 1},
+        {"name": "v2", "staticSize": 24, "exclusiveSize": 24, "size": 32, "length": 2, "capacity": 2}
       ]}]'''
diff --git a/test/integration/std_vector.toml b/test/integration/std_vector.toml
index d466891..0cdaa85 100644
--- a/test/integration/std_vector.toml
+++ b/test/integration/std_vector.toml
@@ -13,12 +13,12 @@ definitions = '''
     param_types = ["const std::vector<int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size": 24, "length":0, "capacity":0, "members":[]}]'
   [cases.int_some]
     param_types = ["const std::vector<int>&"]
     setup = "return {{1,2,3}};"
     expect_json = '[{"staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":36,"length":3, "capacity":3, "members":[
       {"staticSize":4, "exclusiveSize":4},
       {"staticSize":4, "exclusiveSize":4},
       {"staticSize":4, "exclusiveSize":4}
@@ -27,7 +27,7 @@ definitions = '''
     param_types = ["const std::vector<SimpleStruct>&"]
     setup = "return {{{}, {}, {}}};"
     expect_json = '[{"staticSize":24, "dynamicSize":48, "length":3, "capacity":3, "elementStaticSize":16}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":72, "length":3, "capacity":3, "members":[
       {"staticSize":16, "exclusiveSize":3},
       {"staticSize":16, "exclusiveSize":3},
       {"staticSize":16, "exclusiveSize":3}
@@ -46,7 +46,7 @@ definitions = '''
     param_types = ["const std::vector<std::vector<int>>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":24}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size":24, "length":0, "capacity":0, "members":[]}]'
   [cases.vector_int_some]
     param_types = ["const std::vector<std::vector<int>>&"]
     setup = "return {{{1,2,3},{4},{5,6}}};"
@@ -62,10 +62,10 @@ definitions = '''
         {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4},
         {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
       ]}]'''
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[
-      {"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[]},
-      {"staticSize":24, "exclusiveSize":24, "length":1, "capacity": 1, "members":[]},
-      {"staticSize":24, "exclusiveSize":24, "length":2, "capacity": 2, "members":[]}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":120, "length":3, "capacity": 3, "members":[
+      {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity": 3, "members":[]},
+      {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity": 1, "members":[]},
+      {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity": 2, "members":[]}
     ]}]'''
   [cases.reserve]
     param_types = ["const std::vector<int>&"]
@@ -75,7 +75,7 @@ definitions = '''
       return ret;
     '''
     expect_json = '[{"staticSize":24, "dynamicSize":40, "exclusiveSize":64, "length":3, "capacity":10, "elementStaticSize":4}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":52, "length":3, "capacity":10, "members":[
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":52, "size":64, "length":3, "capacity":10, "members":[
       {"staticSize":4, "exclusiveSize":4},
       {"staticSize":4, "exclusiveSize":4},
       {"staticSize":4, "exclusiveSize":4}

From 8d1b505425b685908200bc228332eef6a562c56a Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 2 Jan 2024 21:33:51 +0000
Subject: [PATCH 098/188] tbv2: calculate total memory footprint

TODO: Review and clean up any todos.

Add the option to calculate total size (inclusive size) by wrapping the
existing iterator. This change provides a new iterator, `SizedIterator`, which
wraps an existing iterator and adds a new field `size` to the output element.

This is achieved with a two pass algorithm on the existing iterator:
1. Gather metadata for each element. This includes the total size up until that
   element and the range of elements that should be included in the size.
2. Return the result from the underlying iterator with the additional
   field.

This algorithm is `O(N)` time on the number of elements in the iterator and
`O(N)` time, storing 16 bytes per element. This isn't super expensive but is a
lot more than the current algorithm which requires close to constant space.
Because of this I've implemented it as a wrapper on the iterator rather than on
by default, though it is now on in every one of our integration test cases.

Test plan:
- Added to the integration tests for full coverage.
---
 include/oi/exporters/Json.h                   | 162 +++++++++++++++++-
 include/oi/result/SizedResult-inl.h           | 150 ++++++++++++++++
 include/oi/result/SizedResult.h               |  84 +++++++++
 oi/CMakeLists.txt                             |   4 -
 oi/exporters/Json.cpp                         | 146 ----------------
 test/integration/CMakeLists.txt               |   2 +-
 test/integration/anonymous.toml               |  15 +-
 test/integration/arrays.toml                  |  14 +-
 test/integration/enums.toml                   |  16 +-
 test/integration/enums_params.toml            |  10 +-
 test/integration/fbstring.toml                |   7 +
 test/integration/folly_f14_fast_map.toml      |  26 +--
 test/integration/folly_f14_fast_set.toml      |   9 +-
 test/integration/folly_f14_node_map.toml      |  26 +--
 test/integration/folly_f14_node_set.toml      |   9 +-
 test/integration/folly_f14_value_map.toml     |  26 +--
 test/integration/folly_f14_value_set.toml     |   9 +-
 test/integration/folly_f14_vector_map.toml    |  26 +--
 test/integration/folly_f14_vector_set.toml    |   9 +-
 test/integration/folly_small_vector.toml      |  28 +--
 test/integration/folly_sorted_vector_map.toml |  30 ++--
 test/integration/gen_tests.py                 |   5 +-
 test/integration/inheritance_access.toml      |  22 ++-
 test/integration/inheritance_multiple.toml    |  13 +-
 test/integration/packed.toml                  |   8 +-
 test/integration/padding.toml                 |  26 +--
 test/integration/pointers.toml                |  32 ++--
 test/integration/primitives.toml              |  38 ++--
 test/integration/runner_common.cpp            |   2 +
 test/integration/simple.toml                  |  20 ++-
 test/integration/sorted_vector_set.toml       |   2 +
 test/integration/std_list.toml                |  28 +--
 test/integration/std_list_del_allocator.toml  |   5 +-
 .../std_map_custom_comparator.toml            |   8 +-
 .../std_multimap_custom_comparator.toml       |   6 +-
 .../std_multiset_custom_comparator.toml       |   9 +-
 test/integration/std_optional.toml            |   5 +
 test/integration/std_pair.toml                |  24 +--
 .../std_set_custom_comparator.toml            |   9 +-
 test/integration/std_smart_ptr.toml           |  33 ++--
 .../std_unordered_map_custom_operator.toml    |  10 +-
 ...td_unordered_multimap_custom_operator.toml |  10 +-
 ...td_unordered_multiset_custom_operator.toml |  10 +-
 .../std_unordered_set_custom_operator.toml    |  10 +-
 test/integration/std_vector.toml              |  36 ++--
 .../integration/std_vector_del_allocator.toml |  17 +-
 test/integration/templates.toml               |   4 +
 test/integration/thrift_unions.toml           |  15 +-
 test/integration/typedefed_parent.toml        |  10 +-
 test/integration/unions.toml                  |  30 ++--
 50 files changed, 801 insertions(+), 454 deletions(-)
 create mode 100644 include/oi/result/SizedResult-inl.h
 create mode 100644 include/oi/result/SizedResult.h
 delete mode 100644 oi/exporters/Json.cpp

diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h
index 31aa808..dd88ade 100644
--- a/include/oi/exporters/Json.h
+++ b/include/oi/exporters/Json.h
@@ -17,8 +17,10 @@
 #define INCLUDED_OI_EXPORTERS_JSON_H 1
 
 #include <oi/IntrospectionResult.h>
+#include <oi/result/SizedResult.h>
 
 #include <ostream>
+#include <string_view>
 
 namespace oi::exporters {
 
@@ -26,19 +28,173 @@ class Json {
  public:
   Json(std::ostream& out);
 
-  void print(const IntrospectionResult&);
-  void print(IntrospectionResult::const_iterator& it,
-             IntrospectionResult::const_iterator& end);
+  template <typename Res>
+  void print(const Res& r) {
+    auto begin = r.begin();
+    auto end = r.end();
+    return print(begin, end);
+  }
+  template <typename It>
+  void print(It& it, const It& end);
 
   void setPretty(bool pretty) {
     pretty_ = pretty;
   }
 
  private:
+  std::string_view tab() const;
+  std::string_view space() const;
+  std::string_view endl() const;
+  static std::string makeIndent(size_t depth);
+
+  void printStringField(std::string_view name,
+                        std::string_view value,
+                        std::string_view indent);
+  void printBoolField(std::string_view name,
+                      bool value,
+                      std::string_view indent);
+  void printUnsignedField(std::string_view name,
+                          uint64_t value,
+                          std::string_view indent);
+  void printPointerField(std::string_view name,
+                         uintptr_t value,
+                         std::string_view indent);
+  template <typename Rng>
+  void printListField(std::string_view name,
+                      const Rng& range,
+                      std::string_view indent);
+
+  void printFields(const result::Element&, std::string_view indent);
+  template <typename El>
+  void printFields(const result::SizedElement<El>&, std::string_view indent);
+
   bool pretty_ = false;
   std::ostream& out_;
 };
 
+inline Json::Json(std::ostream& out) : out_(out) {
+}
+
+inline std::string_view Json::tab() const {
+  return pretty_ ? "  " : "";
+}
+inline std::string_view Json::space() const {
+  return pretty_ ? " " : "";
+}
+inline std::string_view Json::endl() const {
+  return pretty_ ? "\n" : "";
+}
+inline std::string Json::makeIndent(size_t depth) {
+  depth = std::max(depth, 1UL);
+  return std::string((depth - 1) * 4, ' ');
+}
+
+inline void Json::printStringField(std::string_view name,
+                                   std::string_view value,
+                                   std::string_view indent) {
+  out_ << tab() << '"' << name << '"' << ':' << space() << "\"" << value
+       << "\"," << endl() << indent;
+}
+inline void Json::printBoolField(std::string_view name,
+                                 bool value,
+                                 std::string_view indent) {
+  out_ << tab() << '"' << name << "\":" << space() << value << ',' << endl()
+       << indent;
+}
+inline void Json::printUnsignedField(std::string_view name,
+                                     uint64_t value,
+                                     std::string_view indent) {
+  out_ << tab() << '"' << name << "\":" << space() << value << ',' << endl()
+       << indent;
+}
+inline void Json::printPointerField(std::string_view name,
+                                    uintptr_t value,
+                                    std::string_view indent) {
+  out_ << tab() << '"' << name << "\":" << space() << "\"0x" << std::hex
+       << value << std::dec << "\"," << endl() << indent;
+}
+template <typename Rng>
+void Json::printListField(std::string_view name,
+                          const Rng& range,
+                          std::string_view indent) {
+  out_ << tab() << '"' << name << '"' << ':' << space() << '[';
+  bool first = true;
+  for (const auto& el : range) {
+    if (!std::exchange(first, false))
+      out_ << ',' << space();
+    out_ << '"' << el << '"';
+  }
+  out_ << "]," << endl() << indent;
+}
+
+template <typename El>
+void Json::printFields(const result::SizedElement<El>& el,
+                       std::string_view indent) {
+  printUnsignedField("size", el.size, indent);
+
+  printFields(el.inner(), indent);
+}
+
+inline void Json::printFields(const result::Element& el,
+                              std::string_view indent) {
+  printStringField("name", el.name, indent);
+  printListField("typePath", el.type_path, indent);
+  printListField("typeNames", el.type_names, indent);
+  printUnsignedField("staticSize", el.static_size, indent);
+  printUnsignedField("exclusiveSize", el.exclusive_size, indent);
+  if (el.pointer.has_value())
+    printUnsignedField("pointer", *el.pointer, indent);
+
+  if (const auto* s = std::get_if<result::Element::Scalar>(&el.data)) {
+    printUnsignedField("data", s->n, indent);
+  } else if (const auto* p = std::get_if<result::Element::Pointer>(&el.data)) {
+    printPointerField("data", p->p, indent);
+  } else if (const auto* str = std::get_if<std::string>(&el.data)) {
+    printStringField("data", *str, indent);
+  }
+
+  if (el.container_stats.has_value()) {
+    printUnsignedField("length", el.container_stats->length, indent);
+    printUnsignedField("capacity", el.container_stats->capacity, indent);
+  }
+  if (el.is_set_stats.has_value())
+    printUnsignedField("is_set", el.is_set_stats->is_set, indent);
+}
+
+template <typename It>
+void Json::print(It& it, const It& end) {
+  const auto depth = it->type_path.size();
+
+  const auto thisIndent = pretty_ ? makeIndent(depth) : "";
+  const auto lastIndent = pretty_ ? makeIndent(depth - 1) : "";
+
+  out_ << '[' << endl() << thisIndent;
+
+  bool first = true;
+  while (it != end && it->type_path.size() >= depth) {
+    if (!std::exchange(first, false))
+      out_ << ',' << endl() << thisIndent;
+
+    out_ << '{' << endl() << thisIndent;
+
+    printFields(*it, thisIndent);
+
+    out_ << tab() << "\"members\":" << space();
+    if (++it != end && it->type_path.size() > depth) {
+      print(it, end);
+    } else {
+      out_ << "[]" << endl();
+    }
+
+    out_ << thisIndent << "}";
+  }
+  if (depth == 1) {
+    out_ << endl() << ']' << endl();
+  } else {
+    out_ << endl() << lastIndent << tab() << ']' << endl();
+  }
+}
+
 }  // namespace oi::exporters
 
 #endif
diff --git a/include/oi/result/SizedResult-inl.h b/include/oi/result/SizedResult-inl.h
new file mode 100644
index 0000000..ded348b
--- /dev/null
+++ b/include/oi/result/SizedResult-inl.h
@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#if defined(INCLUDED_OI_RESULT_SIZED_ELEMENT_INL_H) || \
+    !defined(INCLUDED_OI_RESULT_SIZED_ELEMENT_H)
+static_assert(false,
+              "SizedResult-inl.h provides inline declarations for "
+              "SizedResult.h and should only "
+              "be included by SizedResult.h");
+#endif
+#define INCLUDED_OI_RESULT_SIZED_ELEMENT_INL_H 1
+
+#include <iostream>   // TODO: remove
+#include <stdexcept>  // TODO: remove
+#include <utility>
+#include <vector>
+
+#include "SizedResult.h"
+
+namespace oi::result {
+
+template <typename Res>
+SizedResult<Res>::SizedResult(Res res) : res_{std::move(res)} {
+}
+
+template <typename Res>
+typename SizedResult<Res>::const_iterator SizedResult<Res>::begin() const {
+  return ++const_iterator{res_.begin(), res_.end()};
+}
+template <typename Res>
+typename SizedResult<Res>::const_iterator SizedResult<Res>::end() const {
+  return res_.end();
+}
+
+template <typename Res>
+SizedResult<Res>::const_iterator::const_iterator(It it, const It& end)
+    : data_{it} {
+  struct StackEntry {
+    size_t index;
+    size_t depth;
+  };
+  std::vector<StackEntry> stack;
+
+  size_t size = 0;
+  size_t count = -1;
+  for (; it != end; ++it) {
+    ++count;
+
+    auto depth = it->type_path.size();
+    while (!stack.empty() && stack.back().depth >= depth) {
+      auto i = stack.back().index;
+      stack.pop_back();
+      helpers_[i].last_child = count - 1;
+    }
+
+    size += it->exclusive_size;
+    helpers_.emplace_back(SizeHelper{.size = size});
+    stack.emplace_back(StackEntry{.index = count, .depth = depth});
+  }
+  while (!stack.empty()) {
+    auto i = stack.back().index;
+    stack.pop_back();
+    helpers_[i].last_child = count;
+  }
+}
+
+template <typename Res>
+SizedResult<Res>::const_iterator::const_iterator(It end) : data_{end} {
+}
+
+template <typename Res>
+typename SizedResult<Res>::const_iterator
+SizedResult<Res>::const_iterator::operator++(int) {
+  auto old = *this;
+  operator++();
+  return old;
+}
+
+template <typename Res>
+typename SizedResult<Res>::const_iterator&
+SizedResult<Res>::const_iterator::operator++() {
+  // The below iterator is already pointing at the first element while this
+  // iterator is not. Skip incrementing it the first time around.
+  if (count_ != 0)
+    ++data_;
+
+  if (count_ == helpers_.size()) {
+    next_.reset();
+    return *this;
+  }
+
+  size_t size = helpers_[helpers_[count_].last_child].size;
+  if (count_ != 0)
+    size -= helpers_[count_ - 1].size;
+
+  next_.emplace(*data_, size);
+  ++count_;
+  return *this;
+}
+
+template <typename Res>
+const typename SizedResult<Res>::Element&
+SizedResult<Res>::const_iterator::operator*() const {
+  // auto* leak = new SizedResult<Res>::Element{*data_, (size_t)-1};
+  // return *leak;
+  return *next_;
+}
+template <typename Res>
+const typename SizedResult<Res>::Element*
+SizedResult<Res>::const_iterator::operator->() const {
+  // auto* leak = new SizedResult<Res>::Element{*data_, (size_t)-1};
+  // return leak;
+  return &*next_;
+}
+
+template <typename Res>
+bool SizedResult<Res>::const_iterator::operator==(
+    const SizedResult<Res>::const_iterator& that) const {
+  return this->data_ == that.data_;
+}
+
+template <typename Res>
+bool SizedResult<Res>::const_iterator::operator!=(
+    const SizedResult<Res>::const_iterator& that) const {
+  return !(*this == that);
+}
+
+template <typename El>
+SizedElement<El>::SizedElement(const El& el, size_t size_)
+    : El{el}, size{size_} {
+}
+
+template <typename El>
+const El& SizedElement<El>::inner() const {
+  return static_cast<const El&>(*this);
+}
+
+}  // namespace oi::result
diff --git a/include/oi/result/SizedResult.h b/include/oi/result/SizedResult.h
new file mode 100644
index 0000000..983b804
--- /dev/null
+++ b/include/oi/result/SizedResult.h
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef INCLUDED_OI_RESULT_SIZED_ELEMENT_H
+#define INCLUDED_OI_RESULT_SIZED_ELEMENT_H 1
+
+#include <cstddef>
+#include <optional>
+#include <type_traits>
+#include <vector>
+
+namespace oi::result {
+
+template <typename El>
+struct SizedElement : public El {
+  SizedElement(const El& el, size_t size);
+  const El& inner() const;
+
+  size_t size;
+};
+
+template <typename Res>
+class SizedResult {
+ private:
+  using It = std::decay_t<decltype(std::declval<Res&>().begin())>;
+  using ParentEl = std::decay_t<decltype(std::declval<It&>().operator*())>;
+
+ public:
+  using Element = SizedElement<ParentEl>;
+
+ private:
+  struct SizeHelper {
+    size_t size = -1;
+    size_t last_child = -1;
+  };
+
+ public:
+  class const_iterator {
+    friend SizedResult;
+
+   public:
+    bool operator==(const const_iterator& that) const;
+    bool operator!=(const const_iterator& that) const;
+    const Element& operator*() const;
+    const Element* operator->() const;
+    const_iterator& operator++();
+    const_iterator operator++(int);
+
+   private:
+    const_iterator(It start, const It& end);
+    const_iterator(It end);
+
+    It data_;
+
+    std::vector<SizeHelper> helpers_;
+    size_t count_ = 0;
+    std::optional<Element> next_;
+  };
+
+  SizedResult(Res res);
+
+  const_iterator begin() const;
+  const_iterator end() const;
+
+ private:
+  Res res_;
+};
+
+}  // namespace oi::result
+
+#include "SizedResult-inl.h"
+#endif
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index 3754f41..64dbcae 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -55,10 +55,6 @@ target_link_libraries(codegen
   glog::glog
 )
 
-add_library(exporters_json exporters/Json.cpp)
-target_include_directories(exporters_json PUBLIC ${CMAKE_SOURCE_DIR}/include)
-target_link_libraries(exporters_json oil)
-
 add_library(exporters_csv exporters/CSV.cpp)
 target_include_directories(exporters_csv PUBLIC ${CMAKE_SOURCE_DIR}/include)
 target_link_libraries(exporters_csv oil)
diff --git a/oi/exporters/Json.cpp b/oi/exporters/Json.cpp
deleted file mode 100644
index 8a4d33c..0000000
--- a/oi/exporters/Json.cpp
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include <oi/exporters/Json.h>
-
-#include <algorithm>
-#include <stdexcept>
-#include <variant>
-
-template <class>
-inline constexpr bool always_false_v = false;
-
-namespace oi::exporters {
-namespace {
-
-template <typename It>
-void printStringList(std::ostream& out, It it, It end, bool pretty) {
-  out << '[';
-  for (; it != end; ++it) {
-    out << '"' << (*it) << '"';
-    if (it != end - 1)
-      out << (pretty ? ", " : ",");
-  }
-  out << ']';
-}
-
-std::string makeIndent(size_t depth) {
-  depth = std::max(depth, 1UL);
-  return std::string((depth - 1) * 4, ' ');
-}
-
-}  // namespace
-
-Json::Json(std::ostream& out) : out_(out) {
-}
-
-void Json::print(const IntrospectionResult& r) {
-  auto begin = r.cbegin();
-  auto end = r.cend();
-  return print(begin, end);
-}
-
-void Json::print(IntrospectionResult::const_iterator& it,
-                 IntrospectionResult::const_iterator& end) {
-  const auto firstTypePathSize = it->type_path.size();
-
-  const auto indent = pretty_ ? makeIndent(firstTypePathSize) : "";
-  const auto lastIndent =
-      pretty_ ? makeIndent(std::max(firstTypePathSize, 1UL) - 1) : "";
-  const auto* tab = pretty_ ? "  " : "";
-  const auto* space = pretty_ ? " " : "";
-  const auto* endl = pretty_ ? "\n" : "";
-
-  out_ << '[' << endl << indent;
-
-  bool first = true;
-  while (it != end) {
-    if (it->type_path.size() < firstTypePathSize) {
-      // no longer a sibling, must be a sibling of the type we're printing
-      break;
-    }
-
-    if (!first)
-      out_ << ',' << endl << indent;
-    first = false;
-
-    out_ << '{' << endl << indent;
-
-    out_ << tab << "\"name\"" << space << ':' << space << "\"" << it->name
-         << "\"," << endl
-         << indent;
-
-    out_ << tab << "\"typePath\"" << space << ':' << space << "";
-    printStringList(out_, it->type_path.begin(), it->type_path.end(), pretty_);
-    out_ << (pretty_ ? ",\n" : ",") << indent;
-
-    out_ << tab << "\"typeNames\"" << space << ':' << space;
-    printStringList(
-        out_, it->type_names.begin(), it->type_names.end(), pretty_);
-    out_ << ',' << endl << indent;
-
-    out_ << tab << "\"staticSize\":" << space << it->static_size << ',' << endl
-         << indent;
-    out_ << tab << "\"exclusiveSize\":" << space << it->exclusive_size << ','
-         << endl
-         << indent;
-
-    if (it->pointer.has_value()) {
-      out_ << tab << "\"pointer\":" << space << *(it->pointer) << ',' << endl
-           << indent;
-    }
-
-    if (auto* s = std::get_if<result::Element::Scalar>(&it->data)) {
-      out_ << tab << "\"data\":" << space << s->n << ',' << endl << indent;
-    } else if (auto* p = std::get_if<result::Element::Pointer>(&it->data)) {
-      out_ << tab << "\"data\":" << space << "\"0x" << std::hex << p->p
-           << std::dec << "\"," << endl
-           << indent;
-    } else if (auto* str = std::get_if<std::string>(&it->data)) {
-      out_ << tab << "\"data\":" << space << "\"" << *str << "\"," << endl
-           << indent;
-    }
-
-    if (it->container_stats.has_value()) {
-      out_ << tab << "\"length\":" << space << it->container_stats->length
-           << ',' << endl
-           << indent;
-      out_ << tab << "\"capacity\":" << space << it->container_stats->capacity
-           << ',' << endl
-           << indent;
-    }
-    if (it->is_set_stats.has_value()) {
-      out_ << tab << "\"is_set\":" << space << it->is_set_stats->is_set << ','
-           << endl
-           << indent;
-    }
-
-    out_ << tab << "\"members\":" << space;
-    if (++it != end && it->type_path.size() > firstTypePathSize) {
-      print(it, end);
-    } else {
-      out_ << "[]" << endl;
-    }
-
-    out_ << indent << "}";
-  }
-  if (firstTypePathSize == 1) {
-    out_ << endl << ']' << endl;
-  } else {
-    out_ << endl << lastIndent << tab << ']' << endl;
-  }
-}
-
-}  // namespace oi::exporters
diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 12cb239..4d235b5 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -50,7 +50,7 @@ add_executable(integration_test_target
   ${INTEGRATION_TEST_TARGET_SRC}
   folly_shims.cpp)
 target_compile_options(integration_test_target PRIVATE -O1)
-target_link_libraries(integration_test_target PRIVATE oil_jit exporters_json Boost::headers ${Boost_LIBRARIES})
+target_link_libraries(integration_test_target PRIVATE oil_jit Boost::headers ${Boost_LIBRARIES})
 
 add_executable(integration_test_runner ${INTEGRATION_TEST_RUNNER_SRC} runner_common.cpp)
 target_include_directories(integration_test_runner PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
diff --git a/test/integration/anonymous.toml b/test/integration/anonymous.toml
index 6a91293..15b9f10 100644
--- a/test/integration/anonymous.toml
+++ b/test/integration/anonymous.toml
@@ -88,10 +88,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":12,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size": 12,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":4, "dynamicSize":0},
-        {"name":"c", "staticSize":4, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"c", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}
       ]}]'''
 
   [cases.anon_struct]
@@ -207,10 +209,11 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize": 24,
       "exclusiveSize": 10,
+      "size": 24,
       "members": [
-        {"name":"__oi_anon_0", "staticSize":2, "exclusiveSize":2},
-        {"name":"__oi_anon_2", "staticSize":8, "exclusiveSize":8},
-        {"name":"e", "staticSize":4, "exclusiveSize":4, "typeNames":["int32_t"]}
+        {"name":"__oi_anon_0", "staticSize":2, "exclusiveSize":2, "size":2},
+        {"name":"__oi_anon_2", "staticSize":8, "exclusiveSize":8, "size":8},
+        {"name":"e", "staticSize":4, "exclusiveSize":4, "size":4, "typeNames":["int32_t"]}
       ]
     }]'''
 
diff --git a/test/integration/arrays.toml b/test/integration/arrays.toml
index 4a06b13..5074b62 100644
--- a/test/integration/arrays.toml
+++ b/test/integration/arrays.toml
@@ -33,9 +33,11 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":40,
       "exclusiveSize":0,
+      "size":40,
       "members":[{
         "staticSize":40,
         "exclusiveSize":0,
+        "size":40,
         "length":10,
         "capacity":10
       }]}]'''
@@ -56,9 +58,11 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":1,
       "exclusiveSize":1,
+      "size":1,
       "members":[{
         "staticSize":0,
-        "exclusiveSize":0
+        "exclusiveSize":0,
+        "size":1
       }]}]'''
   [cases.multidim_legacy] # Test for legacy behaviour. Remove with OICodeGen
     oil_disable = 'oil only runs on codegen v2'
@@ -86,10 +90,10 @@ definitions = '''
           {"staticSize":12, "dynamicSize":0, "exclusiveSize":12, "length":3, "capacity":3, "elementStaticSize":4}]
       }]}]'''
     expect_json_v2 = '''[
-      {"staticSize":24, "exclusiveSize":0, "members":[
-        {"staticSize":24, "exclusiveSize":0, "length":2, "capacity":2, "members":[
-          {"staticSize":12, "exclusiveSize":0, "length":3, "capacity":3},
-          {"staticSize":12, "exclusiveSize":0, "length":3, "capacity":3}]
+      {"staticSize":24, "exclusiveSize":0, "size":24, "members":[
+        {"staticSize":24, "exclusiveSize":0, "size":24, "length":2, "capacity":2, "members":[
+          {"staticSize":12, "exclusiveSize":0, "size":12, "length":3, "capacity":3},
+          {"staticSize":12, "exclusiveSize":0, "size":12, "length":3, "capacity":3}]
       }]}]'''
   [cases.direct_int10]
     skip = "Direct array arguments don't work"
diff --git a/test/integration/enums.toml b/test/integration/enums.toml
index f112b66..fa91cbc 100644
--- a/test/integration/enums.toml
+++ b/test/integration/enums.toml
@@ -36,31 +36,31 @@ definitions = '''
     param_types = ["ScopedEnum"]
     setup = "return {};"
     expect_json = '[{"staticSize":4, "dynamicSize":0}]'
-    expect_json_v2 = '[{"typeNames": ["ScopedEnum"], "staticSize":4, "exclusiveSize":4}]'
+    expect_json_v2 = '[{"typeNames": ["ScopedEnum"], "staticSize":4, "exclusiveSize":4, "size":4}]'
   [cases.scoped_uint8]
     param_types = ["ScopedEnumUint8"]
     setup = "return {};"
     expect_json = '[{"staticSize":1, "dynamicSize":0}]'
-    expect_json_v2 = '[{"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1}]'
+    expect_json_v2 = '[{"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1}]'
   [cases.unscoped]
     param_types = ["UNSCOPED_ENUM"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.anonymous]
     skip = "TreeBuilder crashes" # https://github.com/facebookexperimental/object-introspection/issues/232
     param_types = ["Holder&"]
     setup = "return {};"
     expect_json = '''[
-      {"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "members":[
-        {"name":"e", "staticSize":4, "dynamicSize":0, "exclusiveSize":4}
+      {"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4, "members":[
+        {"name":"e", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}
     ]}]'''
   [cases.paired_with_element]
     param_types = ["Pair<ScopedEnumUint8, uint8_t>"]
     setup = "return {};"
     expect_json = '[{"staticSize":2, "dynamicSize":0}]'
     expect_json_v2 = '''[
-      {"staticSize": 2, "exclusiveSize": 0, "members": [
-        {"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1},
-        {"typeNames": ["uint8_t"], "staticSize":1, "exclusiveSize":1}
+      {"staticSize": 2, "exclusiveSize": 0, "size":2, "members": [
+        {"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1},
+        {"typeNames": ["uint8_t"], "staticSize":1, "exclusiveSize":1, "size":1}
       ]}
     ]'''
diff --git a/test/integration/enums_params.toml b/test/integration/enums_params.toml
index cf61d6f..dd66f31 100644
--- a/test/integration/enums_params.toml
+++ b/test/integration/enums_params.toml
@@ -39,20 +39,20 @@ definitions = '''
     param_types = ["const std::array<int, static_cast<size_t>(MyNS::ScopedEnum::Two)>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":8, "length":2, "capacity":2, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":8, "dynamicSize":0, "length":2, "capacity":2}]'
+    expect_json_v2 = '[{"staticSize":8, "exclusiveSize":0, "size":8, "length":2, "capacity":2}]'
 
   [cases.scoped_enum_val]
     param_types = ["const MyClass<MyNS::ScopedEnum::One>&"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4}]'
   [cases.scoped_enum_val_gaps]
     param_types = ["const ClassGaps<MyNS::EnumWithGaps::Twenty>&"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4}]'
   [cases.scoped_enum_val_negative]
     param_types = ["const ClassGaps<MyNS::EnumWithGaps::MinusTwo>&"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4}]'
 
   [cases.unscoped_enum_type]
     param_types = ["const std::vector<MyNS::UNSCOPED_ENUM>&"]
@@ -61,4 +61,4 @@ definitions = '''
     param_types = ["const std::array<int, MyNS::ONE>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":4, "length":1, "capacity":1, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":4, "length":1, "capacity":1}]'
+    expect_json_v2 = '[{"staticSize":4, "exclusiveSize":0, "size":4, "length":1, "capacity":1}]'
diff --git a/test/integration/fbstring.toml b/test/integration/fbstring.toml
index 6e68d99..cec73ca 100644
--- a/test/integration/fbstring.toml
+++ b/test/integration/fbstring.toml
@@ -28,6 +28,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 24,
+      "size": 24,
       "length": 0,
       "capacity": 23
     }]'''
@@ -60,6 +61,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 24,
+      "size": 24,
       "length": 6,
       "capacity": 23
     }]'''
@@ -92,6 +94,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 50,
+      "size": 50,
       "length": 26,
       "capacity": 26
     }]'''
@@ -124,6 +127,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 1056,
+      "size": 1056,
       "length": 1024,
       "capacity": 1024
     }]'''
@@ -137,17 +141,20 @@ includes = ["folly/FBString.h", "utility"]
     expect_json_v2 = '''[{
       "staticSize": 48,
       "exclusiveSize": 0,
+      "size": 1080,
       "members": [
         {
           "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
           "staticSize": 24,
           "exclusiveSize": 1056,
+          "size": 1056,
           "length": 1024,
           "capacity": 1024
         }, {
           "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
           "staticSize": 24,
           "exclusiveSize": 24,
+          "size": 24,
           "length": 1024,
           "capacity": 1024
         }
diff --git a/test/integration/folly_f14_fast_map.toml b/test/integration/folly_f14_fast_map.toml
index 5f9e509..273c222 100644
--- a/test/integration/folly_f14_fast_map.toml
+++ b/test/integration/folly_f14_fast_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 464,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "size": 72, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "size": 104, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "size": 104,"length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 40,
+          "size": 184,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_fast_set.toml b/test/integration/folly_f14_fast_set.toml
index 486b9ba..24d8abe 100644
--- a/test/integration/folly_f14_fast_set.toml
+++ b/test/integration/folly_f14_fast_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size":304,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "size":56, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "size":88, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "size":72, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "size":88, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_f14_node_map.toml b/test/integration/folly_f14_node_map.toml
index 78d414d..07b5b89 100644
--- a/test/integration/folly_f14_node_map.toml
+++ b/test/integration/folly_f14_node_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size":668,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "size": 96, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "size": 148, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "size": 160, "length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 120,
+          "size": 264,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_node_set.toml b/test/integration/folly_f14_node_set.toml
index 497976c..a68cc7b 100644
--- a/test/integration/folly_f14_node_set.toml
+++ b/test/integration/folly_f14_node_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 500,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 120, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "size": 84, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "size": 128, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "size": 132, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 120, "size": 156, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_f14_value_map.toml b/test/integration/folly_f14_value_map.toml
index fac9829..a1a09e8 100644
--- a/test/integration/folly_f14_value_map.toml
+++ b/test/integration/folly_f14_value_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 464,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "size": 72, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "size": 104, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "size": 104, "length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 40,
+          "size": 184,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_value_set.toml b/test/integration/folly_f14_value_set.toml
index 3f19b31..2cf03df 100644
--- a/test/integration/folly_f14_value_set.toml
+++ b/test/integration/folly_f14_value_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 304,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "size": 56, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "size": 88, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "size": 72, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "size": 88, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_f14_vector_map.toml b/test/integration/folly_f14_vector_map.toml
index 9912141..4126243 100644
--- a/test/integration/folly_f14_vector_map.toml
+++ b/test/integration/folly_f14_vector_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 576,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 64, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 60, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 80, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 64, "size": 88, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 60, "size": 120, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 80, "size": 136, "length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 88,
+          "size": 232,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_vector_set.toml b/test/integration/folly_f14_vector_set.toml
index 3ffad84..c67f649 100644
--- a/test/integration/folly_f14_vector_set.toml
+++ b/test/integration/folly_f14_vector_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 400,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 60, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 64, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 76, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 84, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 60, "size": 72, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 64, "size": 104, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 76, "size": 104, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 84, "size": 120, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_small_vector.toml b/test/integration/folly_small_vector.toml
index 5ae53f3..b97b5ff 100644
--- a/test/integration/folly_small_vector.toml
+++ b/test/integration/folly_small_vector.toml
@@ -4,23 +4,23 @@ includes = ["folly/small_vector.h", "vector"]
     param_types = ["const folly::small_vector<int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "length":0, "capacity":2, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "length":0, "capacity":2}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "size":16, "length":0, "capacity":2}]'
   [cases.int_default_inlined]
     param_types = ["const folly::small_vector<int>&"]
     setup = "return {{1,2}};"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "length":2, "capacity":2, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":8, "length":2, "capacity":2}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":8, "size":16, "length":2, "capacity":2}]'
   [cases.int_default_overflow]
     param_types = ["const folly::small_vector<int>&"]
     setup = "return {{1,2,3,4}};"
     expect_json = '[{"staticSize":16, "dynamicSize":24, "exclusiveSize":40, "length":4, "capacity":6, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":24, "length":4, "capacity":6}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":24, "size":40, "length":4, "capacity":6}]'
 
   [cases.vector_3_empty]
     param_types = ["const folly::small_vector<std::vector<int>, 3>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":80, "dynamicSize":0, "exclusiveSize":80, "length":0, "capacity":3, "elementStaticSize":24}]'
-    expect_json_v2 = '[{"staticSize":80, "exclusiveSize":80, "length":0, "capacity":3}]'
+    expect_json_v2 = '[{"staticSize":80, "exclusiveSize":80, "size":80, "length":0, "capacity":3}]'
   [cases.vector_3_inlined]
     param_types = ["const folly::small_vector<std::vector<int>, 3>&"]
     setup = "return {{ {1,2,3}, {4}, {5,6} }};"
@@ -31,10 +31,10 @@ includes = ["folly/small_vector.h", "vector"]
         {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
     ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":80, "length":3, "exclusiveSize":8, "capacity":3, "members":[
-        {"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3},
-        {"staticSize":24, "exclusiveSize":24, "length":1, "capacity":1},
-        {"staticSize":24, "exclusiveSize":24, "length":2, "capacity":2}
+      {"staticSize":80, "length":3, "exclusiveSize":8, "size":104, "capacity":3, "members":[
+        {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity":3},
+        {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity":1},
+        {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity":2}
     ]}]'''
   [cases.vector_3_overflow]
     param_types = ["const folly::small_vector<std::vector<int>, 3>&"]
@@ -47,15 +47,15 @@ includes = ["folly/small_vector.h", "vector"]
         {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4}
     ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":80, "exclusiveSize":104, "length":4, "capacity":5, "members":[
-        {"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3},
-        {"staticSize":24, "exclusiveSize":24, "length":1, "capacity":1},
-        {"staticSize":24, "exclusiveSize":24, "length":2, "capacity":2},
-        {"staticSize":24, "exclusiveSize":24, "length":1, "capacity":1}
+      {"staticSize":80, "exclusiveSize":104, "size":228, "length":4, "capacity":5, "members":[
+        {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity":3},
+        {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity":1},
+        {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity":2},
+        {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity":1}
     ]}]'''
 
   [cases.int_always_heap]
     param_types = ["const folly::small_vector<int, 0>&"]
     setup = "return {{1}};"
     expect_json = '[{"staticSize":16, "dynamicSize":4, "exclusiveSize":20, "length":1, "capacity":1, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "length":1, "capacity":1}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "size":20, "length":1, "capacity":1}]'
diff --git a/test/integration/folly_sorted_vector_map.toml b/test/integration/folly_sorted_vector_map.toml
index 6d3a2a5..44f91e0 100644
--- a/test/integration/folly_sorted_vector_map.toml
+++ b/test/integration/folly_sorted_vector_map.toml
@@ -4,19 +4,19 @@ includes = ["folly/sorted_vector_types.h", "vector"]
     param_types = ["const folly::sorted_vector_map<int, int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "exclusiveSize":24, "length":0, "capacity":0, "elementStaticSize":8}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size":24, "length":0, "capacity":0, "members":[]}]'
   [cases.int_int_some]
     param_types = ["const folly::sorted_vector_map<int, int>&"]
     setup = "return {{ {1,2}, {3,4} }};"
     expect_json = '[{"staticSize":24, "dynamicSize":16, "exclusiveSize":40, "length":2, "capacity":2, "elementStaticSize":8}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":2, "capacity":2, "members":[
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":40, "length":2, "capacity":2, "members":[
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]},
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}
     ]}]'''
   [cases.int_int_reserve]
@@ -27,14 +27,14 @@ includes = ["folly/sorted_vector_types.h", "vector"]
       return m;
     '''
     expect_json = '[{"staticSize":24, "dynamicSize":80, "exclusiveSize":104, "length":2, "capacity":10, "elementStaticSize":8}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":88, "length":2, "capacity":10, "members":[
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":88, "size":104, "length":2, "capacity":10, "members":[
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]},
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}
     ]}]'''
 
diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py
index 2dbc3ff..b72e371 100644
--- a/test/integration/gen_tests.py
+++ b/test/integration/gen_tests.py
@@ -40,6 +40,7 @@ def add_headers(f, custom_headers, thrift_headers):
 
 #include <oi/oi.h>
 #include <oi/exporters/Json.h>
+#include <oi/result/SizedResult.h>
 
 """
     )
@@ -145,8 +146,8 @@ def add_test_setup(f, config):
         oil_func_body += "    auto pr = oi::exporters::Json(std::cout);\n"
         oil_func_body += "    pr.setPretty(true);\n"
         for i in range(len(case["param_types"])):
-            oil_func_body += f"    auto ret{i} = oi::setupAndIntrospect(a{i}, opts);\n"
-            oil_func_body += f"    pr.print(*ret{i});\n"
+            oil_func_body += f"    auto ret{i} = oi::result::SizedResult(*oi::setupAndIntrospect(a{i}, opts));\n"
+            oil_func_body += f"    pr.print(ret{i});\n"
 
         f.write(
             define_traceable_func(
diff --git a/test/integration/inheritance_access.toml b/test/integration/inheritance_access.toml
index faeb35c..9751052 100644
--- a/test/integration/inheritance_access.toml
+++ b/test/integration/inheritance_access.toml
@@ -21,27 +21,33 @@ definitions = '''
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"},
-        {"name":"public_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"},
+        {"name":"public_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
   [cases.protected]
     param_types = ["const Protected&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"},
-        {"name":"protected_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"},
+        {"name":"protected_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
   [cases.private]
     param_types = ["const Private&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"},
-        {"name":"private_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"},
+        {"name":"private_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
   [cases.public_as_base]
     param_types = ["const Base&"]
@@ -49,6 +55,8 @@ definitions = '''
     setup = "return {};"
     expect_json = '''[{
       "staticSize":4,
+      "exclusiveSize":0,
+      "size":4,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
diff --git a/test/integration/inheritance_multiple.toml b/test/integration/inheritance_multiple.toml
index 61d725f..53f956f 100644
--- a/test/integration/inheritance_multiple.toml
+++ b/test/integration/inheritance_multiple.toml
@@ -37,11 +37,12 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":24,
       "exclusiveSize":0,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "exclusiveSize":4},
-        {"name":"b", "staticSize":4, "exclusiveSize":4},
-        {"name":"c", "staticSize":4, "exclusiveSize":4},
-        {"name":"d", "staticSize":4, "exclusiveSize":4},
-        {"name":"e", "staticSize":4, "exclusiveSize":4},
-        {"name":"f", "staticSize":4, "exclusiveSize":4}
+        {"name":"a", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"c", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"d", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"e", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"f", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}]'''
diff --git a/test/integration/packed.toml b/test/integration/packed.toml
index 2bbef02..0e45479 100644
--- a/test/integration/packed.toml
+++ b/test/integration/packed.toml
@@ -12,8 +12,10 @@ definitions = '''
     expect_json = '''[{
       "staticSize":17,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size":17,
       "members":[
-        {"name":"p", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":1, "dynamicSize":0},
-        {"name":"x", "staticSize":8, "dynamicSize":0}
+        {"name":"p", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1},
+        {"name":"x", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
diff --git a/test/integration/padding.toml b/test/integration/padding.toml
index 6cc17d1..5001be6 100644
--- a/test/integration/padding.toml
+++ b/test/integration/padding.toml
@@ -65,10 +65,11 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize": 7,
       "members":[
-        { "name":"a", "staticSize":8, "dynamicSize":0 },
-        { "name":"b", "staticSize":1, "dynamicSize":0 },
-        { "name":"c", "staticSize":8, "dynamicSize":0 }
+        { "name":"a", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
+        { "name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1 },
+        { "name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 }
       ]}]'''
 
   [cases.nested_padding]
@@ -77,18 +78,22 @@ definitions = '''
     expect_json = '''[{
       "staticSize":48,
       "dynamicSize":0,
+      "exclusiveSize":7,
+      "size":48,
       "members":[
-        { "name":"a", "staticSize":8, "dynamicSize":0 },
-        { "name":"b", "staticSize":1, "dynamicSize":0 },
-        { "name":"c", "staticSize":8, "dynamicSize":0 },
+        { "name":"a", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
+        { "name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1 },
+        { "name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
         {
           "name":"d",
           "staticSize":24,
           "dynamicSize":0,
+          "exclusiveSize":7,
+          "size":24,
           "members": [
-            { "name":"a", "staticSize":8, "dynamicSize":0 },
-            { "name":"b", "staticSize":1, "dynamicSize":0 },
-            { "name":"c", "staticSize":8, "dynamicSize":0 }
+            { "name":"a", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
+            { "name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1 },
+            { "name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 }
         ]}
       ]}]'''
 
@@ -103,5 +108,6 @@ definitions = '''
     }]'''
     expect_json_v2 = '''[{
       "staticSize": 104,
-      "exclusiveSize": 32
+      "exclusiveSize": 32,
+      "size": 104
     }]'''
diff --git a/test/integration/pointers.toml b/test/integration/pointers.toml
index 389ba29..71bf2ad 100644
--- a/test/integration/pointers.toml
+++ b/test/integration/pointers.toml
@@ -153,10 +153,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize":4,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_primitive_ptrs_null]
     param_types = ["const PrimitivePtrs&"]
@@ -165,10 +167,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize":4,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
 
 
@@ -189,8 +193,10 @@ definitions = '''
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"vec", "staticSize":8, "dynamicSize":0}
+        {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_vector_ptr_null]
     param_types = ["const VectorPtr&"]
@@ -199,8 +205,10 @@ definitions = '''
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"vec", "staticSize":8, "dynamicSize":0}
+        {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
 
 
@@ -242,10 +250,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize":4,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.feature_config]
     oil_disable = "oil can't chase raw pointers safely"
diff --git a/test/integration/primitives.toml b/test/integration/primitives.toml
index 429712f..94784fb 100644
--- a/test/integration/primitives.toml
+++ b/test/integration/primitives.toml
@@ -2,76 +2,76 @@
   [cases.short]
     param_types = ["short"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":2, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":2, "dynamicSize":0, "exclusiveSize":2, "size":2}]'
   [cases.ushort]
     param_types = ["unsigned short"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":2, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":2, "dynamicSize":0, "exclusiveSize":2, "size":2}]'
   [cases.int]
     param_types = ["int"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.uint]
     param_types = ["unsigned int"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.long]
     param_types = ["long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.ulong]
     param_types = ["unsigned long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.longlong]
     param_types = ["long long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.ulonglong]
     param_types = ["unsigned long long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.bool]
     param_types = ["bool"]
     setup = "return true;"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.char]
     param_types = ["char"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.uchar]
     param_types = ["unsigned char"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.schar]
     param_types = ["signed char"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.wchar_t]
     param_types = ["wchar_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.char8_t]
     param_types = ["char8_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.char16_t]
     param_types = ["char16_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":2, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":2, "dynamicSize":0, "exclusiveSize":2, "size":2}]'
   [cases.char32_t]
     param_types = ["char32_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.float]
     param_types = ["float"]
     setup = "return 3.14;"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.double]
     param_types = ["double"]
     setup = "return 3.14;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.long_double]
     param_types = ["long double"]
     setup = "return 3.14;"
-    expect_json = '[{"staticSize":16, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "size":16}]'
diff --git a/test/integration/runner_common.cpp b/test/integration/runner_common.cpp
index c86bd59..3c8525a 100644
--- a/test/integration/runner_common.cpp
+++ b/test/integration/runner_common.cpp
@@ -365,6 +365,8 @@ void IntegrationBase::compare_json(const bpt::ptree& expected_json,
         }
       } else if (key == "dynamicSize" && val.get_value<size_t>() == 0) {
         continue;
+      } else if (key == "size") {
+        continue;
       }
 
       ADD_FAILURE() << "Expected key not found in output: " << curr_key;
diff --git a/test/integration/simple.toml b/test/integration/simple.toml
index 71a215a..0945f70 100644
--- a/test/integration/simple.toml
+++ b/test/integration/simple.toml
@@ -23,10 +23,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
+      "exclusiveSize": 3,
+      "size": 16,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":1, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
+        {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
   [cases.class]
     param_types = ["const SimpleClass&"]
@@ -34,15 +36,19 @@ definitions = '''
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
+      "exclusiveSize": 3,
+      "size": 16,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":1, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
+        {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
   [cases.union]
     param_types = ["const SimpleUnion&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
-      "dynamicSize":0
+      "dynamicSize":0,
+      "exclusiveSize":8,
+      "size":8
     }]'''
diff --git a/test/integration/sorted_vector_set.toml b/test/integration/sorted_vector_set.toml
index 8e37abd..0a4e416 100644
--- a/test/integration/sorted_vector_set.toml
+++ b/test/integration/sorted_vector_set.toml
@@ -23,6 +23,7 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize": 24,
       "exclusiveSize": 24,
+      "size": 24,
       "length": 0,
       "capacity": 0
     }]'''
@@ -56,6 +57,7 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize": 24,
       "exclusiveSize": 28,
+      "size": 40,
       "length": 3,
       "capacity": 4
     }]'''
diff --git a/test/integration/std_list.toml b/test/integration/std_list.toml
index dfe2d8f..92c039a 100644
--- a/test/integration/std_list.toml
+++ b/test/integration/std_list.toml
@@ -13,30 +13,30 @@ definitions = '''
     param_types = ["const std::list<int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size":24, "length":0, "capacity":0, "members":[]}]'
   [cases.int_some]
     param_types = ["const std::list<int>&"]
     setup = "return {{1,2,3}};"
     expect_json = '[{"staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity":3, "members":[
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4}
     ]}]'''
   [cases.struct_some]
     param_types = ["const std::list<SimpleStruct>&"]
     setup = "return {{{}, {}, {}}};"
     expect_json = '[{"staticSize":24, "dynamicSize":48, "length":3, "capacity":3, "elementStaticSize":16}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
-      {"staticSize":16, "exclusiveSize":3},
-      {"staticSize":16, "exclusiveSize":3},
-      {"staticSize":16, "exclusiveSize":3}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":72, "length":3, "capacity":3, "members":[
+      {"staticSize":16, "exclusiveSize":3, "size":16},
+      {"staticSize":16, "exclusiveSize":3, "size":16},
+      {"staticSize":16, "exclusiveSize":3, "size":16}
     ]}]'''
   [cases.list_int_empty]
     param_types = ["const std::list<std::list<int>>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":24}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size":24, "length":0, "capacity":0, "members":[]}]'
   [cases.list_int_some]
     param_types = ["const std::list<std::list<int>>&"]
     setup = "return {{{1,2,3},{4},{5,6}}};"
@@ -52,8 +52,8 @@ definitions = '''
         {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4},
         {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
       ]}]'''
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[
-      {"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[]},
-      {"staticSize":24, "exclusiveSize":24, "length":1, "capacity": 1, "members":[]},
-      {"staticSize":24, "exclusiveSize":24, "length":2, "capacity": 2, "members":[]}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":120, "length":3, "capacity": 3, "members":[
+      {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity": 3},
+      {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity": 1},
+      {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity": 2}
     ]}]'''
diff --git a/test/integration/std_list_del_allocator.toml b/test/integration/std_list_del_allocator.toml
index b986996..c85d3ab 100644
--- a/test/integration/std_list_del_allocator.toml
+++ b/test/integration/std_list_del_allocator.toml
@@ -60,7 +60,8 @@ includes = ["list"]
     expect_json_v2 = '''[{
       "staticSize": 48,
       "exclusiveSize": 0,
+      "size": 60,
       "members": [
-        {"name": "v1", "staticSize": 24, "exclusiveSize": 24, "length": 1, "capacity": 1},
-        {"name": "v2", "staticSize": 24, "exclusiveSize": 24, "length": 2, "capacity": 2}
+        {"name": "v1", "staticSize": 24, "exclusiveSize": 24, "size": 28, "length": 1, "capacity": 1},
+        {"name": "v2", "staticSize": 24, "exclusiveSize": 24, "size": 32, "length": 2, "capacity": 2}
       ]}]'''
diff --git a/test/integration/std_map_custom_comparator.toml b/test/integration/std_map_custom_comparator.toml
index f8edd24..5f30b36 100644
--- a/test/integration/std_map_custom_comparator.toml
+++ b/test/integration/std_map_custom_comparator.toml
@@ -70,10 +70,12 @@ includes = ["map", "functional"]
     expect_json_v2 = '''[{
       "staticSize":8184,
       "exclusiveSize":0,
+      "size":9168,
       "members":[
         {"name":"m1",
          "staticSize":48,
          "exclusiveSize":48,
+         "size":192,
          "length":3,
          "capacity":3,
          "members": [
@@ -81,7 +83,7 @@ includes = ["map", "functional"]
           {},
           {}
          ]},
-        {"name":"m2", "staticSize":48, "exclusiveSize":48, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":48, "exclusiveSize":48, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":8040, "exclusiveSize":8040, "length":9, "capacity":9}
+        {"name":"m2", "staticSize":48, "exclusiveSize":48, "size":248, "length":5, "capacity":5},
+        {"name":"m3", "staticSize":48, "exclusiveSize":48, "size":328, "length":7, "capacity":7},
+        {"name":"m4", "staticSize":8040, "exclusiveSize":8040, "size":8400, "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_multimap_custom_comparator.toml b/test/integration/std_multimap_custom_comparator.toml
index 7f8171f..883c4cd 100644
--- a/test/integration/std_multimap_custom_comparator.toml
+++ b/test/integration/std_multimap_custom_comparator.toml
@@ -38,16 +38,18 @@ includes = ["map"]
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize":0,
+      "size":440,
       "members":[
         {"name":"m1",
          "staticSize":48,
          "exclusiveSize":48,
+         "size":192,
          "length":3,
          "capacity":3,
          "members": [
-          {"name":"[]", "staticSize":48, "exclusiveSize":36},
+          {"name":"[]", "staticSize":48, "exclusiveSize":36, "size":48},
           {},
           {}
          ]},
-        {"name":"m2", "staticSize":48, "exclusiveSize":48, "length":5, "capacity":5}
+        {"name":"m2", "staticSize":48, "exclusiveSize":48, "size":248, "length":5, "capacity":5}
       ]}]'''
diff --git a/test/integration/std_multiset_custom_comparator.toml b/test/integration/std_multiset_custom_comparator.toml
index 04d5da8..e8ef1aa 100644
--- a/test/integration/std_multiset_custom_comparator.toml
+++ b/test/integration/std_multiset_custom_comparator.toml
@@ -65,9 +65,10 @@ includes = ["set", "functional"]
     expect_json_v2 = '''[{
       "staticSize":8184,
       "exclusiveSize": 0,
+      "size":9144,
       "members":[
-        {"name":"m1", "staticSize":48, "exclusiveSize": 156, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":48, "exclusiveSize": 228, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":48, "exclusiveSize": 300, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":8040, "exclusiveSize": 8364, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":48, "exclusiveSize": 156, "size": 168, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":48, "exclusiveSize": 228, "size": 248, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":48, "exclusiveSize": 300, "size": 328, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":8040, "exclusiveSize": 8364, "size": 8400, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/std_optional.toml b/test/integration/std_optional.toml
index 7475e28..9da0271 100644
--- a/test/integration/std_optional.toml
+++ b/test/integration/std_optional.toml
@@ -20,6 +20,7 @@ includes = ["optional", "cstdint", "vector"]
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 16,
         "length": 0,
         "capacity": 1,
         "members": []
@@ -46,6 +47,7 @@ includes = ["optional", "cstdint", "vector"]
       {
         "staticSize": 16,
         "exclusiveSize": 8,
+        "size": 16,
         "length": 1,
         "capacity": 1,
         "members": [
@@ -77,6 +79,7 @@ includes = ["optional", "cstdint", "vector"]
       {
         "staticSize": 32,
         "exclusiveSize": 32,
+        "size": 32,
         "length": 0,
         "capacity": 1,
         "members": []
@@ -111,12 +114,14 @@ includes = ["optional", "cstdint", "vector"]
       {
         "staticSize": 32,
         "exclusiveSize": 8,
+        "size": 72,
         "length": 1,
         "capacity": 1,
         "members": [
           {
             "staticSize": 24,
             "exclusiveSize": 24,
+            "size": 64,
             "length": 5,
             "capacity": 5
           }
diff --git a/test/integration/std_pair.toml b/test/integration/std_pair.toml
index c60e784..1780951 100644
--- a/test/integration/std_pair.toml
+++ b/test/integration/std_pair.toml
@@ -14,9 +14,9 @@ includes = ["vector", "utility", "cstdint"]
     ]
     '''
     expect_json_v2 = '''[
-      {"staticSize": 16, "exclusiveSize": 0, "members": [
-        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8},
-        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8}
+      {"staticSize": 16, "exclusiveSize": 0, "size": 16, "members": [
+        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8, "size": 8},
+        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8, "size": 8}
       ]}
     ]'''
   [cases.uint64_uint32]
@@ -34,9 +34,9 @@ includes = ["vector", "utility", "cstdint"]
     ]
     '''
     expect_json_v2 = '''[
-      {"staticSize": 16, "exclusiveSize": 4, "members": [
-        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8},
-        {"typeNames": ["uint32_t"], "staticSize": 4, "exclusiveSize": 4}
+      {"staticSize": 16, "exclusiveSize": 4, "size": 16, "members": [
+        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8, "size": 8},
+        {"typeNames": ["uint32_t"], "staticSize": 4, "exclusiveSize": 4, "size": 4}
       ]}
     ]'''
 
@@ -48,9 +48,9 @@ includes = ["vector", "utility", "cstdint"]
     param_types = ["std::pair<uint64_t, uint64_t*>&"]
     setup = "return {{0, nullptr}};"
     expect_json_v2 = '''[
-      {"staticSize": 16, "exclusiveSize": 0, "members": [
-        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8},
-        {"typeNames": ["uintptr_t (stubbed)"], "staticSize": 8, "exclusiveSize": 8}
+      {"staticSize": 16, "exclusiveSize": 0, "size": 16, "members": [
+        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8, "size": 8},
+        {"typeNames": ["uintptr_t (stubbed)"], "staticSize": 8, "exclusiveSize": 8, "size": 8}
       ]}
     ]'''
 
@@ -82,8 +82,8 @@ includes = ["vector", "utility", "cstdint"]
     ]
     '''
     expect_json_v2 = '''[
-      {"staticSize": 48, "exclusiveSize": 0, "members": [
-        {"typeNames": ["std::vector<uint64_t, std::allocator<uint64_t>>"], "staticSize": 24, "exclusiveSize": 24},
-        {"typeNames": ["std::vector<uint64_t, std::allocator<uint64_t>>"], "staticSize": 24, "exclusiveSize": 24}
+      {"staticSize": 48, "exclusiveSize": 0, "size": 104, "members": [
+        {"typeNames": ["std::vector<uint64_t, std::allocator<uint64_t>>"], "staticSize": 24, "exclusiveSize": 24, "size": 48},
+        {"typeNames": ["std::vector<uint64_t, std::allocator<uint64_t>>"], "staticSize": 24, "exclusiveSize": 24, "size": 56}
       ]}
     ]'''
diff --git a/test/integration/std_set_custom_comparator.toml b/test/integration/std_set_custom_comparator.toml
index 66031f7..49d07f8 100644
--- a/test/integration/std_set_custom_comparator.toml
+++ b/test/integration/std_set_custom_comparator.toml
@@ -65,9 +65,10 @@ includes = ["set", "functional"]
     expect_json_v2 = '''[{
       "staticSize":8184,
       "exclusiveSize": 0,
+      "size": 9144,
       "members":[
-        {"name":"m1", "staticSize":48, "exclusiveSize": 156, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":48, "exclusiveSize": 228, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":48, "exclusiveSize": 300, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":8040, "exclusiveSize": 8364, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":48, "exclusiveSize": 156, "size": 168, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":48, "exclusiveSize": 228, "size": 248, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":48, "exclusiveSize": 300, "size": 328, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":8040, "exclusiveSize": 8364, "size": 8400, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/std_smart_ptr.toml b/test/integration/std_smart_ptr.toml
index 8fc66bf..dfb86ae 100644
--- a/test/integration/std_smart_ptr.toml
+++ b/test/integration/std_smart_ptr.toml
@@ -26,6 +26,7 @@ definitions = '''
       {
         "staticSize": 8,
         "exclusiveSize": 8,
+        "size": 8,
         "length": 0,
         "capacity": 1
       }
@@ -50,6 +51,7 @@ definitions = '''
       {
         "staticSize": 8,
         "exclusiveSize": 8,
+        "size": 16,
         "length": 1,
         "capacity": 1
       }
@@ -74,6 +76,7 @@ definitions = '''
       {
         "staticSize": 8,
         "exclusiveSize": 8,
+        "size": 8,
         "length": 0,
         "capacity": 1
       }
@@ -106,6 +109,7 @@ definitions = '''
       {
         "staticSize": 8,
         "exclusiveSize": 8,
+        "size": 72,
         "length": 1,
         "capacity": 1,
         "members": [
@@ -134,7 +138,8 @@ definitions = '''
     [
       {
         "staticSize": 16,
-        "exclusiveSize": 16
+        "exclusiveSize": 16,
+        "size": 16
       }
     ]
     '''
@@ -153,7 +158,8 @@ definitions = '''
     [
       {
         "staticSize": 16,
-        "exclusiveSize": 16
+        "exclusiveSize": 16,
+        "size": 16
       }
     ]
     '''
@@ -176,6 +182,7 @@ definitions = '''
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 16,
         "length": 0,
         "capacity": 1
       }
@@ -200,6 +207,7 @@ definitions = '''
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 24,
         "length": 1,
         "capacity": 1,
         "members": [
@@ -227,6 +235,7 @@ definitions = '''
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 16,
         "length": 0,
         "capacity": 1
       }
@@ -257,12 +266,14 @@ definitions = '''
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 80,
         "length": 1,
         "capacity": 1,
         "members": [
           {
             "staticSize": 24,
             "exclusiveSize": 24,
+            "size": 64,
             "length": 5,
             "capacity": 5
           }
@@ -278,7 +289,8 @@ definitions = '''
       {
         "staticSize": 16,
         "dynamicSize": 0,
-        "exclusiveSize": 16
+        "exclusiveSize": 16,
+        "size": 16
       }
     ]
     '''
@@ -290,7 +302,8 @@ definitions = '''
       {
         "staticSize": 16,
         "dynamicSize": 0,
-        "exclusiveSize": 16
+        "exclusiveSize": 16,
+        "size": 16
       }
     ]
     '''
@@ -307,7 +320,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_void_empty]
     param_types = ["std::weak_ptr<void>&"]
     setup = "return std::weak_ptr<void>();"
@@ -321,7 +334,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_present]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
     setup = '''
@@ -339,7 +352,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_expired]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
     setup = '''
@@ -357,7 +370,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_present_chase]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
     cli_options = ["-fchase-raw-pointers"]
@@ -376,7 +389,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_expired_chase]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
     cli_options = ["-fchase-raw-pointers"]
@@ -393,4 +406,4 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
diff --git a/test/integration/std_unordered_map_custom_operator.toml b/test/integration/std_unordered_map_custom_operator.toml
index 3dcf55d..ea78bff 100644
--- a/test/integration/std_unordered_map_custom_operator.toml
+++ b/test/integration/std_unordered_map_custom_operator.toml
@@ -66,9 +66,11 @@ includes = ["unordered_map"]
       ]}]'''
     expect_json_v2 = '''[{
       "staticSize":480,
+      "exclusiveSize": 0,
+      "size": 1952,
       "members":[
-        {"name":"m1", "staticSize":56, "exclusiveSize":220, "length":3, "capacity":3},
-        {"name":"m2", "staticSize":120, "exclusiveSize":324, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":120, "exclusiveSize":364, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":184, "exclusiveSize":468, "length":9, "capacity":9}
+        {"name":"m1", "staticSize":56, "exclusiveSize":220, "size":292, "length":3, "capacity":3},
+        {"name":"m2", "staticSize":120, "exclusiveSize":324, "size":444,  "length":5, "capacity":5},
+        {"name":"m3", "staticSize":120, "exclusiveSize":364, "size":532,  "length":7, "capacity":7},
+        {"name":"m4", "staticSize":184, "exclusiveSize":468, "size":684,  "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_unordered_multimap_custom_operator.toml b/test/integration/std_unordered_multimap_custom_operator.toml
index 37fe213..f36a296 100644
--- a/test/integration/std_unordered_multimap_custom_operator.toml
+++ b/test/integration/std_unordered_multimap_custom_operator.toml
@@ -66,9 +66,11 @@ includes = ["unordered_map"]
       ]}]'''
     expect_json_v2 = '''[{
       "staticSize":480,
+      "exclusiveSize":0,
+      "size":1952,
       "members":[
-        {"name":"m1", "staticSize":56, "exclusiveSize":220, "length":3, "capacity":3},
-        {"name":"m2", "staticSize":120, "exclusiveSize":324, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":120, "exclusiveSize":364, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":184, "exclusiveSize":468, "length":9, "capacity":9}
+        {"name":"m1", "staticSize":56, "exclusiveSize":220, "size":292, "length":3, "capacity":3},
+        {"name":"m2", "staticSize":120, "exclusiveSize":324, "size":444,  "length":5, "capacity":5},
+        {"name":"m3", "staticSize":120, "exclusiveSize":364, "size":532,  "length":7, "capacity":7},
+        {"name":"m4", "staticSize":184, "exclusiveSize":468, "size":684,  "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_unordered_multiset_custom_operator.toml b/test/integration/std_unordered_multiset_custom_operator.toml
index ef08dcb..d9ce1cc 100644
--- a/test/integration/std_unordered_multiset_custom_operator.toml
+++ b/test/integration/std_unordered_multiset_custom_operator.toml
@@ -66,9 +66,11 @@ includes = ["unordered_set"]
       ]}]'''
     expect_json_v2 = '''[{
       "staticSize":480,
+      "exclusiveSize":0,
+      "size":1472,
       "members":[
-        {"name":"m1", "staticSize":56, "exclusiveSize":220, "length":3, "capacity":3},
-        {"name":"m2", "staticSize":120, "exclusiveSize":324, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":120, "exclusiveSize":364, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":184, "exclusiveSize":468, "length":9, "capacity":9}
+        {"name":"m1", "staticSize":56, "exclusiveSize":220, "size":232, "length":3, "capacity":3},
+        {"name":"m2", "staticSize":120, "exclusiveSize":324, "size":344, "length":5, "capacity":5},
+        {"name":"m3", "staticSize":120, "exclusiveSize":364, "size":392, "length":7, "capacity":7},
+        {"name":"m4", "staticSize":184, "exclusiveSize":468, "size":504, "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_unordered_set_custom_operator.toml b/test/integration/std_unordered_set_custom_operator.toml
index 3c01f6c..797c5ea 100644
--- a/test/integration/std_unordered_set_custom_operator.toml
+++ b/test/integration/std_unordered_set_custom_operator.toml
@@ -66,9 +66,11 @@ includes = ["unordered_set"]
       ]}]'''
     expect_json_v2 = '''[{
       "staticSize":480,
+      "exclusiveSize":0,
+      "size":1472,
       "members":[
-        {"name":"m1", "staticSize":56, "exclusiveSize":220, "length":3, "capacity":3},
-        {"name":"m2", "staticSize":120, "exclusiveSize":324, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":120, "exclusiveSize":364, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":184, "exclusiveSize":468, "length":9, "capacity":9}
+        {"name":"m1", "staticSize":56, "exclusiveSize":220, "size":232,"length":3, "capacity":3},
+        {"name":"m2", "staticSize":120, "exclusiveSize":324, "size":344, "length":5, "capacity":5},
+        {"name":"m3", "staticSize":120, "exclusiveSize":364, "size":392, "length":7, "capacity":7},
+        {"name":"m4", "staticSize":184, "exclusiveSize":468, "size":504, "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_vector.toml b/test/integration/std_vector.toml
index d466891..20e00d8 100644
--- a/test/integration/std_vector.toml
+++ b/test/integration/std_vector.toml
@@ -13,24 +13,24 @@ definitions = '''
     param_types = ["const std::vector<int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size": 24, "length":0, "capacity":0, "members":[]}]'
   [cases.int_some]
     param_types = ["const std::vector<int>&"]
     setup = "return {{1,2,3}};"
     expect_json = '[{"staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":36,"length":3, "capacity":3, "members":[
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4}
     ]}]'''
   [cases.struct_some]
     param_types = ["const std::vector<SimpleStruct>&"]
     setup = "return {{{}, {}, {}}};"
     expect_json = '[{"staticSize":24, "dynamicSize":48, "length":3, "capacity":3, "elementStaticSize":16}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
-      {"staticSize":16, "exclusiveSize":3},
-      {"staticSize":16, "exclusiveSize":3},
-      {"staticSize":16, "exclusiveSize":3}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":72, "length":3, "capacity":3, "members":[
+      {"staticSize":16, "exclusiveSize":3, "size":16},
+      {"staticSize":16, "exclusiveSize":3, "size":16},
+      {"staticSize":16, "exclusiveSize":3, "size":16}
     ]}]'''
   [cases.bool_empty]
     skip = true # https://github.com/facebookexperimental/object-introspection/issues/14
@@ -46,7 +46,7 @@ definitions = '''
     param_types = ["const std::vector<std::vector<int>>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":24}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size":24, "length":0, "capacity":0, "members":[]}]'
   [cases.vector_int_some]
     param_types = ["const std::vector<std::vector<int>>&"]
     setup = "return {{{1,2,3},{4},{5,6}}};"
@@ -62,10 +62,10 @@ definitions = '''
         {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4},
         {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
       ]}]'''
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[
-      {"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[]},
-      {"staticSize":24, "exclusiveSize":24, "length":1, "capacity": 1, "members":[]},
-      {"staticSize":24, "exclusiveSize":24, "length":2, "capacity": 2, "members":[]}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":120, "length":3, "capacity": 3, "members":[
+      {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity": 3, "members":[]},
+      {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity": 1, "members":[]},
+      {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity": 2, "members":[]}
     ]}]'''
   [cases.reserve]
     param_types = ["const std::vector<int>&"]
@@ -75,8 +75,8 @@ definitions = '''
       return ret;
     '''
     expect_json = '[{"staticSize":24, "dynamicSize":40, "exclusiveSize":64, "length":3, "capacity":10, "elementStaticSize":4}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":52, "length":3, "capacity":10, "members":[
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":52, "size":64, "length":3, "capacity":10, "members":[
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4}
     ]}]'''
diff --git a/test/integration/std_vector_del_allocator.toml b/test/integration/std_vector_del_allocator.toml
index 934bd34..32edcc2 100644
--- a/test/integration/std_vector_del_allocator.toml
+++ b/test/integration/std_vector_del_allocator.toml
@@ -60,18 +60,19 @@ includes = ["vector"]
     expect_json_v2 = '''[{
       "staticSize":48,
       "exclusiveSize":0,
+      "size":60,
       "members":[
-        {"name":"v1", "staticSize":24, "exclusiveSize":24, "length":1, "capacity":1, "members":[
-          {"name":"[]", "staticSize":4, "exclusiveSize":0, "members":[
-            {"name":"a", "staticSize":4, "exclusiveSize":4, "members":[]}
+        {"name":"v1", "staticSize":24, "exclusiveSize":24, "size": 28, "length":1, "capacity":1, "members":[
+          {"name":"[]", "staticSize":4, "exclusiveSize":0, "size": 4, "members":[
+            {"name":"a", "staticSize":4, "exclusiveSize":4, "size": 4, "members":[]}
           ]}
         ]},
-        {"name":"v2", "staticSize":24, "exclusiveSize":24, "length":2, "capacity":2, "members":[
-          {"name":"[]", "staticSize":4, "exclusiveSize":0, "members":[
-            {"name":"b", "staticSize":4, "exclusiveSize":4, "members":[]}
+        {"name":"v2", "staticSize":24, "exclusiveSize":24, "size": 32, "length":2, "capacity":2, "members":[
+          {"name":"[]", "staticSize":4, "exclusiveSize":0, "size": 4, "members":[
+            {"name":"b", "staticSize":4, "exclusiveSize":4, "size": 4, "members":[]}
           ]},
-          {"name":"[]", "staticSize":4, "exclusiveSize":0, "members":[
-            {"name":"b", "staticSize":4, "exclusiveSize":4, "members":[]}
+          {"name":"[]", "staticSize":4, "exclusiveSize":0, "size": 4, "members":[
+            {"name":"b", "staticSize":4, "exclusiveSize":4, "size": 4, "members":[]}
           ]}
         ]}
       ]
diff --git a/test/integration/templates.toml b/test/integration/templates.toml
index b02cfc8..8be4dc9 100644
--- a/test/integration/templates.toml
+++ b/test/integration/templates.toml
@@ -53,10 +53,12 @@ definitions = '''
       "typeName":"ns_templates::TemplatedClass1<std::vector<int, std::allocator<int> > >",
       "staticSize":24,
       "exclusiveSize":0,
+      "size":24,
       "members":[{
         "typeName":"std::vector<int32_t, std::allocator<int32_t>>",
         "staticSize":24,
         "exclusiveSize":24,
+        "size":24,
         "length":0,
         "capacity":0
     }]}]'''
@@ -90,9 +92,11 @@ definitions = '''
       "typeName":"ns_templates::TemplatedClassVal<3>",
       "staticSize":12,
       "exclusiveSize":0,
+      "size":12,
       "members":[{
         "staticSize":12,
         "exclusiveSize":0,
+        "size":12,
         "length":3,
         "capacity":3
     }]}]'''
diff --git a/test/integration/thrift_unions.toml b/test/integration/thrift_unions.toml
index 5f0ba1f..651869b 100644
--- a/test/integration/thrift_unions.toml
+++ b/test/integration/thrift_unions.toml
@@ -37,9 +37,10 @@ namespace cpp2 {
     expect_json_v2 = '''[{
       "staticSize":8,
       "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"typeNames":["storage_type"], "name":"value_", "staticSize":4, "exclusiveSize":4},
-        {"typeNames":["underlying_type_t<cpp2::StaticUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4}
+        {"typeNames":["storage_type"], "name":"value_", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"typeNames":["underlying_type_t<cpp2::StaticUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}]'''
   [cases.dynamic_int]
     param_types = ["const cpp2::DynamicUnion&"]
@@ -58,9 +59,10 @@ namespace cpp2 {
     expect_json_v2 = '''[{
       "staticSize":32,
       "exclusiveSize":4,
+      "size":32,
       "members":[
-        {"typeNames":["storage_type"], "name":"value_", "staticSize":24, "exclusiveSize":24},
-        {"typeNames":["underlying_type_t<cpp2::DynamicUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4}
+        {"typeNames":["storage_type"], "name":"value_", "staticSize":24, "exclusiveSize":24, "size":24},
+        {"typeNames":["underlying_type_t<cpp2::DynamicUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}]'''
   [cases.dynamic_vec]
     param_types = ["const cpp2::DynamicUnion&"]
@@ -79,7 +81,8 @@ namespace cpp2 {
     expect_json_v2 = '''[{
       "staticSize":32,
       "exclusiveSize":4,
+      "size":32,
       "members":[
-        {"typeNames":["storage_type"], "name":"value_", "staticSize":24, "exclusiveSize":24},
-        {"typeNames":["underlying_type_t<cpp2::DynamicUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4}
+        {"typeNames":["storage_type"], "name":"value_", "staticSize":24, "exclusiveSize":24, "size":24},
+        {"typeNames":["underlying_type_t<cpp2::DynamicUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}]'''
diff --git a/test/integration/typedefed_parent.toml b/test/integration/typedefed_parent.toml
index 72c4a57..25631f9 100644
--- a/test/integration/typedefed_parent.toml
+++ b/test/integration/typedefed_parent.toml
@@ -30,9 +30,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":48,
       "exclusiveSize":0,
+      "size":80,
       "members":[
-        {"name":"a", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3},
-        {"name":"b", "staticSize":24, "exclusiveSize":24, "length":5, "capacity":5}
+        {"name":"a", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "size":36},
+        {"name":"b", "staticSize":24, "exclusiveSize":24, "length":5, "capacity":5, "size":44}
       ]}]'''
   [cases.multilevel_typedef_parent]
     param_types = ["const Bar_2&"]
@@ -50,7 +51,8 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":48,
       "exclusiveSize":0,
+      "size":80,
       "members":[
-        {"name":"a", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3},
-        {"name":"c", "staticSize":24, "exclusiveSize":24, "length":5, "capacity":5}
+        {"name":"a", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "size":36},
+        {"name":"c", "staticSize":24, "exclusiveSize":24, "length":5, "capacity":5, "size":44}
       ]}]'''
diff --git a/test/integration/unions.toml b/test/integration/unions.toml
index 42341de..66df45e 100644
--- a/test/integration/unions.toml
+++ b/test/integration/unions.toml
@@ -45,17 +45,17 @@ definitions = '''
     param_types = ["const MyUnion&"]
     setup = "return 123;"
     expect_json = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}]'
-    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]}]'
   [cases.vector]
     param_types = ["const MyUnion&"]
     setup = "return std::vector{1,2,3};"
     expect_json = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}]'
-    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]}]'
   [cases.unordered_map]
     param_types = ["const MyUnion&"]
     setup = 'return std::unordered_map<std::string, std::string>{{"a", "b"}, {"c","d"}};'
     expect_json = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}]'
-    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]}]'
 
   [cases.alignment]
     # Wrap the union in a pair as a way of inferring its alignment
@@ -67,9 +67,9 @@ definitions = '''
         {"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}
       ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[
-        {"staticSize":1, "dynamicSize":0, "exclusiveSize":1},
-        {"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}
+      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "size":64, "members":[
+        {"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1},
+        {"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]}
       ]}]'''
 
   [cases.tagged_int]
@@ -81,9 +81,9 @@ definitions = '''
         {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "NOT":"members"}
       ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[
-        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]},
-        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "members":[]}
+      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "size":64, "members":[
+        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]},
+        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1, "members":[]}
       ]}]'''
   [cases.tagged_vector]
     param_types = ["const TaggedUnion&"]
@@ -94,9 +94,9 @@ definitions = '''
         {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "NOT":"members"}
       ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[
-        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]},
-        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "members":[]}
+      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "size":64, "members":[
+        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]},
+        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1, "members":[]}
       ]}]'''
   [cases.tagged_unordered_map]
     param_types = ["const TaggedUnion&"]
@@ -107,7 +107,7 @@ definitions = '''
         {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "NOT":"members"}
       ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[
-        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]},
-        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "members":[]}
+      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "size":64, "members":[
+        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]},
+        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1, "members":[]}
       ]}]'''

From 9a0c67266491a9c2df183fbf35aaf23adf2c0095 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 2 Jan 2024 21:33:51 +0000
Subject: [PATCH 099/188] tbv2: calculate total memory footprint

TODO: Review and clean up any todos.

Add the option to calculate total size (inclusive size) by wrapping the
existing iterator. This change provides a new iterator, `SizedIterator`, which
wraps an existing iterator and adds a new field `size` to the output element.

This is achieved with a two pass algorithm on the existing iterator:
1. Gather metadata for each element. This includes the total size up until that
   element and the range of elements that should be included in the size.
2. Return the result from the underlying iterator with the additional
   field.

This algorithm is `O(N)` time on the number of elements in the iterator and
`O(N)` time, storing 16 bytes per element. This isn't super expensive but is a
lot more than the current algorithm which requires close to constant space.
Because of this I've implemented it as a wrapper on the iterator rather than on
by default, though it is now on in every one of our integration test cases.

Test plan:
- Added to the integration tests for full coverage.
---
 include/oi/exporters/Json.h                   | 162 +++++++++++++++++-
 include/oi/result/SizedResult-inl.h           | 144 ++++++++++++++++
 include/oi/result/SizedResult.h               |  84 +++++++++
 oi/CMakeLists.txt                             |   4 -
 oi/exporters/Json.cpp                         | 146 ----------------
 test/integration/CMakeLists.txt               |   2 +-
 test/integration/anonymous.toml               |  15 +-
 test/integration/arrays.toml                  |  14 +-
 test/integration/enums.toml                   |  16 +-
 test/integration/enums_params.toml            |  10 +-
 test/integration/fbstring.toml                |   7 +
 test/integration/folly_f14_fast_map.toml      |  26 +--
 test/integration/folly_f14_fast_set.toml      |   9 +-
 test/integration/folly_f14_node_map.toml      |  26 +--
 test/integration/folly_f14_node_set.toml      |   9 +-
 test/integration/folly_f14_value_map.toml     |  26 +--
 test/integration/folly_f14_value_set.toml     |   9 +-
 test/integration/folly_f14_vector_map.toml    |  26 +--
 test/integration/folly_f14_vector_set.toml    |   9 +-
 test/integration/folly_small_vector.toml      |  28 +--
 test/integration/folly_sorted_vector_map.toml |  30 ++--
 test/integration/gen_tests.py                 |   5 +-
 test/integration/inheritance_access.toml      |  22 ++-
 test/integration/inheritance_multiple.toml    |  13 +-
 test/integration/packed.toml                  |   8 +-
 test/integration/padding.toml                 |  26 +--
 test/integration/pointers.toml                |  32 ++--
 test/integration/primitives.toml              |  38 ++--
 test/integration/runner_common.cpp            |   2 +
 test/integration/simple.toml                  |  20 ++-
 test/integration/sorted_vector_set.toml       |   2 +
 test/integration/std_list.toml                |  28 +--
 test/integration/std_list_del_allocator.toml  |   5 +-
 .../std_map_custom_comparator.toml            |   8 +-
 .../std_multimap_custom_comparator.toml       |   6 +-
 .../std_multiset_custom_comparator.toml       |   9 +-
 test/integration/std_optional.toml            |   5 +
 test/integration/std_pair.toml                |  24 +--
 .../std_set_custom_comparator.toml            |   9 +-
 test/integration/std_smart_ptr.toml           |  33 ++--
 .../std_unordered_map_custom_operator.toml    |  10 +-
 ...td_unordered_multimap_custom_operator.toml |  10 +-
 ...td_unordered_multiset_custom_operator.toml |  10 +-
 .../std_unordered_set_custom_operator.toml    |  10 +-
 test/integration/std_vector.toml              |  36 ++--
 .../integration/std_vector_del_allocator.toml |  17 +-
 test/integration/templates.toml               |   4 +
 test/integration/thrift_unions.toml           |  15 +-
 test/integration/typedefed_parent.toml        |  10 +-
 test/integration/unions.toml                  |  30 ++--
 50 files changed, 795 insertions(+), 454 deletions(-)
 create mode 100644 include/oi/result/SizedResult-inl.h
 create mode 100644 include/oi/result/SizedResult.h
 delete mode 100644 oi/exporters/Json.cpp

diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h
index 31aa808..dd88ade 100644
--- a/include/oi/exporters/Json.h
+++ b/include/oi/exporters/Json.h
@@ -17,8 +17,10 @@
 #define INCLUDED_OI_EXPORTERS_JSON_H 1
 
 #include <oi/IntrospectionResult.h>
+#include <oi/result/SizedResult.h>
 
 #include <ostream>
+#include <string_view>
 
 namespace oi::exporters {
 
@@ -26,19 +28,173 @@ class Json {
  public:
   Json(std::ostream& out);
 
-  void print(const IntrospectionResult&);
-  void print(IntrospectionResult::const_iterator& it,
-             IntrospectionResult::const_iterator& end);
+  template <typename Res>
+  void print(const Res& r) {
+    auto begin = r.begin();
+    auto end = r.end();
+    return print(begin, end);
+  }
+  template <typename It>
+  void print(It& it, const It& end);
 
   void setPretty(bool pretty) {
     pretty_ = pretty;
   }
 
  private:
+  std::string_view tab() const;
+  std::string_view space() const;
+  std::string_view endl() const;
+  static std::string makeIndent(size_t depth);
+
+  void printStringField(std::string_view name,
+                        std::string_view value,
+                        std::string_view indent);
+  void printBoolField(std::string_view name,
+                      bool value,
+                      std::string_view indent);
+  void printUnsignedField(std::string_view name,
+                          uint64_t value,
+                          std::string_view indent);
+  void printPointerField(std::string_view name,
+                         uintptr_t value,
+                         std::string_view indent);
+  template <typename Rng>
+  void printListField(std::string_view name,
+                      const Rng& range,
+                      std::string_view indent);
+
+  void printFields(const result::Element&, std::string_view indent);
+  template <typename El>
+  void printFields(const result::SizedElement<El>&, std::string_view indent);
+
   bool pretty_ = false;
   std::ostream& out_;
 };
 
+inline Json::Json(std::ostream& out) : out_(out) {
+}
+
+inline std::string_view Json::tab() const {
+  return pretty_ ? "  " : "";
+}
+inline std::string_view Json::space() const {
+  return pretty_ ? " " : "";
+}
+inline std::string_view Json::endl() const {
+  return pretty_ ? "\n" : "";
+}
+inline std::string Json::makeIndent(size_t depth) {
+  depth = std::max(depth, 1UL);
+  return std::string((depth - 1) * 4, ' ');
+}
+
+inline void Json::printStringField(std::string_view name,
+                                   std::string_view value,
+                                   std::string_view indent) {
+  out_ << tab() << '"' << name << '"' << ':' << space() << "\"" << value
+       << "\"," << endl() << indent;
+}
+inline void Json::printBoolField(std::string_view name,
+                                 bool value,
+                                 std::string_view indent) {
+  out_ << tab() << '"' << name << "\":" << space() << value << ',' << endl()
+       << indent;
+}
+inline void Json::printUnsignedField(std::string_view name,
+                                     uint64_t value,
+                                     std::string_view indent) {
+  out_ << tab() << '"' << name << "\":" << space() << value << ',' << endl()
+       << indent;
+}
+inline void Json::printPointerField(std::string_view name,
+                                    uintptr_t value,
+                                    std::string_view indent) {
+  out_ << tab() << '"' << name << "\":" << space() << "\"0x" << std::hex
+       << value << std::dec << "\"," << endl() << indent;
+}
+template <typename Rng>
+void Json::printListField(std::string_view name,
+                          const Rng& range,
+                          std::string_view indent) {
+  out_ << tab() << '"' << name << '"' << ':' << space() << '[';
+  bool first = true;
+  for (const auto& el : range) {
+    if (!std::exchange(first, false))
+      out_ << ',' << space();
+    out_ << '"' << el << '"';
+  }
+  out_ << "]," << endl() << indent;
+}
+
+template <typename El>
+void Json::printFields(const result::SizedElement<El>& el,
+                       std::string_view indent) {
+  printUnsignedField("size", el.size, indent);
+
+  printFields(el.inner(), indent);
+}
+
+inline void Json::printFields(const result::Element& el,
+                              std::string_view indent) {
+  printStringField("name", el.name, indent);
+  printListField("typePath", el.type_path, indent);
+  printListField("typeNames", el.type_names, indent);
+  printUnsignedField("staticSize", el.static_size, indent);
+  printUnsignedField("exclusiveSize", el.exclusive_size, indent);
+  if (el.pointer.has_value())
+    printUnsignedField("pointer", *el.pointer, indent);
+
+  if (const auto* s = std::get_if<result::Element::Scalar>(&el.data)) {
+    printUnsignedField("data", s->n, indent);
+  } else if (const auto* p = std::get_if<result::Element::Pointer>(&el.data)) {
+    printPointerField("data", p->p, indent);
+  } else if (const auto* str = std::get_if<std::string>(&el.data)) {
+    printStringField("data", *str, indent);
+  }
+
+  if (el.container_stats.has_value()) {
+    printUnsignedField("length", el.container_stats->length, indent);
+    printUnsignedField("capacity", el.container_stats->capacity, indent);
+  }
+  if (el.is_set_stats.has_value())
+    printUnsignedField("is_set", el.is_set_stats->is_set, indent);
+}
+
+template <typename It>
+void Json::print(It& it, const It& end) {
+  const auto depth = it->type_path.size();
+
+  const auto thisIndent = pretty_ ? makeIndent(depth) : "";
+  const auto lastIndent = pretty_ ? makeIndent(depth - 1) : "";
+
+  out_ << '[' << endl() << thisIndent;
+
+  bool first = true;
+  while (it != end && it->type_path.size() >= depth) {
+    if (!std::exchange(first, false))
+      out_ << ',' << endl() << thisIndent;
+
+    out_ << '{' << endl() << thisIndent;
+
+    printFields(*it, thisIndent);
+
+    out_ << tab() << "\"members\":" << space();
+    if (++it != end && it->type_path.size() > depth) {
+      print(it, end);
+    } else {
+      out_ << "[]" << endl();
+    }
+
+    out_ << thisIndent << "}";
+  }
+  if (depth == 1) {
+    out_ << endl() << ']' << endl();
+  } else {
+    out_ << endl() << lastIndent << tab() << ']' << endl();
+  }
+}
+
 }  // namespace oi::exporters
 
 #endif
diff --git a/include/oi/result/SizedResult-inl.h b/include/oi/result/SizedResult-inl.h
new file mode 100644
index 0000000..f86ff0c
--- /dev/null
+++ b/include/oi/result/SizedResult-inl.h
@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#if defined(INCLUDED_OI_RESULT_SIZED_ELEMENT_INL_H) || \
+    !defined(INCLUDED_OI_RESULT_SIZED_ELEMENT_H)
+static_assert(false,
+              "SizedResult-inl.h provides inline declarations for "
+              "SizedResult.h and should only "
+              "be included by SizedResult.h");
+#endif
+#define INCLUDED_OI_RESULT_SIZED_ELEMENT_INL_H 1
+
+#include <utility>
+#include <vector>
+
+#include "SizedResult.h"
+
+namespace oi::result {
+
+template <typename Res>
+SizedResult<Res>::SizedResult(Res res) : res_{std::move(res)} {
+}
+
+template <typename Res>
+typename SizedResult<Res>::const_iterator SizedResult<Res>::begin() const {
+  return ++const_iterator{res_.begin(), res_.end()};
+}
+template <typename Res>
+typename SizedResult<Res>::const_iterator SizedResult<Res>::end() const {
+  return res_.end();
+}
+
+template <typename Res>
+SizedResult<Res>::const_iterator::const_iterator(It it, const It& end)
+    : data_{it} {
+  struct StackEntry {
+    size_t index;
+    size_t depth;
+  };
+  std::vector<StackEntry> stack;
+
+  size_t size = 0;
+  size_t count = -1;
+  for (; it != end; ++it) {
+    ++count;
+
+    auto depth = it->type_path.size();
+    while (!stack.empty() && stack.back().depth >= depth) {
+      auto i = stack.back().index;
+      stack.pop_back();
+      helpers_[i].last_child = count - 1;
+    }
+
+    size += it->exclusive_size;
+    helpers_.emplace_back(SizeHelper{.size = size});
+    stack.emplace_back(StackEntry{.index = count, .depth = depth});
+  }
+  while (!stack.empty()) {
+    auto i = stack.back().index;
+    stack.pop_back();
+    helpers_[i].last_child = count;
+  }
+}
+
+template <typename Res>
+SizedResult<Res>::const_iterator::const_iterator(It end) : data_{end} {
+}
+
+template <typename Res>
+typename SizedResult<Res>::const_iterator
+SizedResult<Res>::const_iterator::operator++(int) {
+  auto old = *this;
+  operator++();
+  return old;
+}
+
+template <typename Res>
+typename SizedResult<Res>::const_iterator&
+SizedResult<Res>::const_iterator::operator++() {
+  // The below iterator is already pointing at the first element while this
+  // iterator is not. Skip incrementing it the first time around.
+  if (count_ != 0)
+    ++data_;
+
+  if (count_ == helpers_.size()) {
+    next_.reset();
+    return *this;
+  }
+
+  size_t size = helpers_[helpers_[count_].last_child].size;
+  if (count_ != 0)
+    size -= helpers_[count_ - 1].size;
+
+  next_.emplace(*data_, size);
+  ++count_;
+  return *this;
+}
+
+template <typename Res>
+const typename SizedResult<Res>::Element&
+SizedResult<Res>::const_iterator::operator*() const {
+  return *next_;
+}
+template <typename Res>
+const typename SizedResult<Res>::Element*
+SizedResult<Res>::const_iterator::operator->() const {
+  return &*next_;
+}
+
+template <typename Res>
+bool SizedResult<Res>::const_iterator::operator==(
+    const SizedResult<Res>::const_iterator& that) const {
+  return this->data_ == that.data_;
+}
+
+template <typename Res>
+bool SizedResult<Res>::const_iterator::operator!=(
+    const SizedResult<Res>::const_iterator& that) const {
+  return !(*this == that);
+}
+
+template <typename El>
+SizedElement<El>::SizedElement(const El& el, size_t size_)
+    : El{el}, size{size_} {
+}
+
+template <typename El>
+const El& SizedElement<El>::inner() const {
+  return static_cast<const El&>(*this);
+}
+
+}  // namespace oi::result
diff --git a/include/oi/result/SizedResult.h b/include/oi/result/SizedResult.h
new file mode 100644
index 0000000..983b804
--- /dev/null
+++ b/include/oi/result/SizedResult.h
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef INCLUDED_OI_RESULT_SIZED_ELEMENT_H
+#define INCLUDED_OI_RESULT_SIZED_ELEMENT_H 1
+
+#include <cstddef>
+#include <optional>
+#include <type_traits>
+#include <vector>
+
+namespace oi::result {
+
+template <typename El>
+struct SizedElement : public El {
+  SizedElement(const El& el, size_t size);
+  const El& inner() const;
+
+  size_t size;
+};
+
+template <typename Res>
+class SizedResult {
+ private:
+  using It = std::decay_t<decltype(std::declval<Res&>().begin())>;
+  using ParentEl = std::decay_t<decltype(std::declval<It&>().operator*())>;
+
+ public:
+  using Element = SizedElement<ParentEl>;
+
+ private:
+  struct SizeHelper {
+    size_t size = -1;
+    size_t last_child = -1;
+  };
+
+ public:
+  class const_iterator {
+    friend SizedResult;
+
+   public:
+    bool operator==(const const_iterator& that) const;
+    bool operator!=(const const_iterator& that) const;
+    const Element& operator*() const;
+    const Element* operator->() const;
+    const_iterator& operator++();
+    const_iterator operator++(int);
+
+   private:
+    const_iterator(It start, const It& end);
+    const_iterator(It end);
+
+    It data_;
+
+    std::vector<SizeHelper> helpers_;
+    size_t count_ = 0;
+    std::optional<Element> next_;
+  };
+
+  SizedResult(Res res);
+
+  const_iterator begin() const;
+  const_iterator end() const;
+
+ private:
+  Res res_;
+};
+
+}  // namespace oi::result
+
+#include "SizedResult-inl.h"
+#endif
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index 3754f41..64dbcae 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -55,10 +55,6 @@ target_link_libraries(codegen
   glog::glog
 )
 
-add_library(exporters_json exporters/Json.cpp)
-target_include_directories(exporters_json PUBLIC ${CMAKE_SOURCE_DIR}/include)
-target_link_libraries(exporters_json oil)
-
 add_library(exporters_csv exporters/CSV.cpp)
 target_include_directories(exporters_csv PUBLIC ${CMAKE_SOURCE_DIR}/include)
 target_link_libraries(exporters_csv oil)
diff --git a/oi/exporters/Json.cpp b/oi/exporters/Json.cpp
deleted file mode 100644
index 8a4d33c..0000000
--- a/oi/exporters/Json.cpp
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include <oi/exporters/Json.h>
-
-#include <algorithm>
-#include <stdexcept>
-#include <variant>
-
-template <class>
-inline constexpr bool always_false_v = false;
-
-namespace oi::exporters {
-namespace {
-
-template <typename It>
-void printStringList(std::ostream& out, It it, It end, bool pretty) {
-  out << '[';
-  for (; it != end; ++it) {
-    out << '"' << (*it) << '"';
-    if (it != end - 1)
-      out << (pretty ? ", " : ",");
-  }
-  out << ']';
-}
-
-std::string makeIndent(size_t depth) {
-  depth = std::max(depth, 1UL);
-  return std::string((depth - 1) * 4, ' ');
-}
-
-}  // namespace
-
-Json::Json(std::ostream& out) : out_(out) {
-}
-
-void Json::print(const IntrospectionResult& r) {
-  auto begin = r.cbegin();
-  auto end = r.cend();
-  return print(begin, end);
-}
-
-void Json::print(IntrospectionResult::const_iterator& it,
-                 IntrospectionResult::const_iterator& end) {
-  const auto firstTypePathSize = it->type_path.size();
-
-  const auto indent = pretty_ ? makeIndent(firstTypePathSize) : "";
-  const auto lastIndent =
-      pretty_ ? makeIndent(std::max(firstTypePathSize, 1UL) - 1) : "";
-  const auto* tab = pretty_ ? "  " : "";
-  const auto* space = pretty_ ? " " : "";
-  const auto* endl = pretty_ ? "\n" : "";
-
-  out_ << '[' << endl << indent;
-
-  bool first = true;
-  while (it != end) {
-    if (it->type_path.size() < firstTypePathSize) {
-      // no longer a sibling, must be a sibling of the type we're printing
-      break;
-    }
-
-    if (!first)
-      out_ << ',' << endl << indent;
-    first = false;
-
-    out_ << '{' << endl << indent;
-
-    out_ << tab << "\"name\"" << space << ':' << space << "\"" << it->name
-         << "\"," << endl
-         << indent;
-
-    out_ << tab << "\"typePath\"" << space << ':' << space << "";
-    printStringList(out_, it->type_path.begin(), it->type_path.end(), pretty_);
-    out_ << (pretty_ ? ",\n" : ",") << indent;
-
-    out_ << tab << "\"typeNames\"" << space << ':' << space;
-    printStringList(
-        out_, it->type_names.begin(), it->type_names.end(), pretty_);
-    out_ << ',' << endl << indent;
-
-    out_ << tab << "\"staticSize\":" << space << it->static_size << ',' << endl
-         << indent;
-    out_ << tab << "\"exclusiveSize\":" << space << it->exclusive_size << ','
-         << endl
-         << indent;
-
-    if (it->pointer.has_value()) {
-      out_ << tab << "\"pointer\":" << space << *(it->pointer) << ',' << endl
-           << indent;
-    }
-
-    if (auto* s = std::get_if<result::Element::Scalar>(&it->data)) {
-      out_ << tab << "\"data\":" << space << s->n << ',' << endl << indent;
-    } else if (auto* p = std::get_if<result::Element::Pointer>(&it->data)) {
-      out_ << tab << "\"data\":" << space << "\"0x" << std::hex << p->p
-           << std::dec << "\"," << endl
-           << indent;
-    } else if (auto* str = std::get_if<std::string>(&it->data)) {
-      out_ << tab << "\"data\":" << space << "\"" << *str << "\"," << endl
-           << indent;
-    }
-
-    if (it->container_stats.has_value()) {
-      out_ << tab << "\"length\":" << space << it->container_stats->length
-           << ',' << endl
-           << indent;
-      out_ << tab << "\"capacity\":" << space << it->container_stats->capacity
-           << ',' << endl
-           << indent;
-    }
-    if (it->is_set_stats.has_value()) {
-      out_ << tab << "\"is_set\":" << space << it->is_set_stats->is_set << ','
-           << endl
-           << indent;
-    }
-
-    out_ << tab << "\"members\":" << space;
-    if (++it != end && it->type_path.size() > firstTypePathSize) {
-      print(it, end);
-    } else {
-      out_ << "[]" << endl;
-    }
-
-    out_ << indent << "}";
-  }
-  if (firstTypePathSize == 1) {
-    out_ << endl << ']' << endl;
-  } else {
-    out_ << endl << lastIndent << tab << ']' << endl;
-  }
-}
-
-}  // namespace oi::exporters
diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 12cb239..4d235b5 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -50,7 +50,7 @@ add_executable(integration_test_target
   ${INTEGRATION_TEST_TARGET_SRC}
   folly_shims.cpp)
 target_compile_options(integration_test_target PRIVATE -O1)
-target_link_libraries(integration_test_target PRIVATE oil_jit exporters_json Boost::headers ${Boost_LIBRARIES})
+target_link_libraries(integration_test_target PRIVATE oil_jit Boost::headers ${Boost_LIBRARIES})
 
 add_executable(integration_test_runner ${INTEGRATION_TEST_RUNNER_SRC} runner_common.cpp)
 target_include_directories(integration_test_runner PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
diff --git a/test/integration/anonymous.toml b/test/integration/anonymous.toml
index 6a91293..15b9f10 100644
--- a/test/integration/anonymous.toml
+++ b/test/integration/anonymous.toml
@@ -88,10 +88,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":12,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size": 12,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":4, "dynamicSize":0},
-        {"name":"c", "staticSize":4, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"c", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}
       ]}]'''
 
   [cases.anon_struct]
@@ -207,10 +209,11 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize": 24,
       "exclusiveSize": 10,
+      "size": 24,
       "members": [
-        {"name":"__oi_anon_0", "staticSize":2, "exclusiveSize":2},
-        {"name":"__oi_anon_2", "staticSize":8, "exclusiveSize":8},
-        {"name":"e", "staticSize":4, "exclusiveSize":4, "typeNames":["int32_t"]}
+        {"name":"__oi_anon_0", "staticSize":2, "exclusiveSize":2, "size":2},
+        {"name":"__oi_anon_2", "staticSize":8, "exclusiveSize":8, "size":8},
+        {"name":"e", "staticSize":4, "exclusiveSize":4, "size":4, "typeNames":["int32_t"]}
       ]
     }]'''
 
diff --git a/test/integration/arrays.toml b/test/integration/arrays.toml
index 4a06b13..5074b62 100644
--- a/test/integration/arrays.toml
+++ b/test/integration/arrays.toml
@@ -33,9 +33,11 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":40,
       "exclusiveSize":0,
+      "size":40,
       "members":[{
         "staticSize":40,
         "exclusiveSize":0,
+        "size":40,
         "length":10,
         "capacity":10
       }]}]'''
@@ -56,9 +58,11 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":1,
       "exclusiveSize":1,
+      "size":1,
       "members":[{
         "staticSize":0,
-        "exclusiveSize":0
+        "exclusiveSize":0,
+        "size":1
       }]}]'''
   [cases.multidim_legacy] # Test for legacy behaviour. Remove with OICodeGen
     oil_disable = 'oil only runs on codegen v2'
@@ -86,10 +90,10 @@ definitions = '''
           {"staticSize":12, "dynamicSize":0, "exclusiveSize":12, "length":3, "capacity":3, "elementStaticSize":4}]
       }]}]'''
     expect_json_v2 = '''[
-      {"staticSize":24, "exclusiveSize":0, "members":[
-        {"staticSize":24, "exclusiveSize":0, "length":2, "capacity":2, "members":[
-          {"staticSize":12, "exclusiveSize":0, "length":3, "capacity":3},
-          {"staticSize":12, "exclusiveSize":0, "length":3, "capacity":3}]
+      {"staticSize":24, "exclusiveSize":0, "size":24, "members":[
+        {"staticSize":24, "exclusiveSize":0, "size":24, "length":2, "capacity":2, "members":[
+          {"staticSize":12, "exclusiveSize":0, "size":12, "length":3, "capacity":3},
+          {"staticSize":12, "exclusiveSize":0, "size":12, "length":3, "capacity":3}]
       }]}]'''
   [cases.direct_int10]
     skip = "Direct array arguments don't work"
diff --git a/test/integration/enums.toml b/test/integration/enums.toml
index f112b66..fa91cbc 100644
--- a/test/integration/enums.toml
+++ b/test/integration/enums.toml
@@ -36,31 +36,31 @@ definitions = '''
     param_types = ["ScopedEnum"]
     setup = "return {};"
     expect_json = '[{"staticSize":4, "dynamicSize":0}]'
-    expect_json_v2 = '[{"typeNames": ["ScopedEnum"], "staticSize":4, "exclusiveSize":4}]'
+    expect_json_v2 = '[{"typeNames": ["ScopedEnum"], "staticSize":4, "exclusiveSize":4, "size":4}]'
   [cases.scoped_uint8]
     param_types = ["ScopedEnumUint8"]
     setup = "return {};"
     expect_json = '[{"staticSize":1, "dynamicSize":0}]'
-    expect_json_v2 = '[{"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1}]'
+    expect_json_v2 = '[{"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1}]'
   [cases.unscoped]
     param_types = ["UNSCOPED_ENUM"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.anonymous]
     skip = "TreeBuilder crashes" # https://github.com/facebookexperimental/object-introspection/issues/232
     param_types = ["Holder&"]
     setup = "return {};"
     expect_json = '''[
-      {"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "members":[
-        {"name":"e", "staticSize":4, "dynamicSize":0, "exclusiveSize":4}
+      {"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4, "members":[
+        {"name":"e", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}
     ]}]'''
   [cases.paired_with_element]
     param_types = ["Pair<ScopedEnumUint8, uint8_t>"]
     setup = "return {};"
     expect_json = '[{"staticSize":2, "dynamicSize":0}]'
     expect_json_v2 = '''[
-      {"staticSize": 2, "exclusiveSize": 0, "members": [
-        {"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1},
-        {"typeNames": ["uint8_t"], "staticSize":1, "exclusiveSize":1}
+      {"staticSize": 2, "exclusiveSize": 0, "size":2, "members": [
+        {"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1},
+        {"typeNames": ["uint8_t"], "staticSize":1, "exclusiveSize":1, "size":1}
       ]}
     ]'''
diff --git a/test/integration/enums_params.toml b/test/integration/enums_params.toml
index cf61d6f..dd66f31 100644
--- a/test/integration/enums_params.toml
+++ b/test/integration/enums_params.toml
@@ -39,20 +39,20 @@ definitions = '''
     param_types = ["const std::array<int, static_cast<size_t>(MyNS::ScopedEnum::Two)>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":8, "length":2, "capacity":2, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":8, "dynamicSize":0, "length":2, "capacity":2}]'
+    expect_json_v2 = '[{"staticSize":8, "exclusiveSize":0, "size":8, "length":2, "capacity":2}]'
 
   [cases.scoped_enum_val]
     param_types = ["const MyClass<MyNS::ScopedEnum::One>&"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4}]'
   [cases.scoped_enum_val_gaps]
     param_types = ["const ClassGaps<MyNS::EnumWithGaps::Twenty>&"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4}]'
   [cases.scoped_enum_val_negative]
     param_types = ["const ClassGaps<MyNS::EnumWithGaps::MinusTwo>&"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4}]'
 
   [cases.unscoped_enum_type]
     param_types = ["const std::vector<MyNS::UNSCOPED_ENUM>&"]
@@ -61,4 +61,4 @@ definitions = '''
     param_types = ["const std::array<int, MyNS::ONE>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":4, "length":1, "capacity":1, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":4, "length":1, "capacity":1}]'
+    expect_json_v2 = '[{"staticSize":4, "exclusiveSize":0, "size":4, "length":1, "capacity":1}]'
diff --git a/test/integration/fbstring.toml b/test/integration/fbstring.toml
index 6e68d99..cec73ca 100644
--- a/test/integration/fbstring.toml
+++ b/test/integration/fbstring.toml
@@ -28,6 +28,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 24,
+      "size": 24,
       "length": 0,
       "capacity": 23
     }]'''
@@ -60,6 +61,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 24,
+      "size": 24,
       "length": 6,
       "capacity": 23
     }]'''
@@ -92,6 +94,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 50,
+      "size": 50,
       "length": 26,
       "capacity": 26
     }]'''
@@ -124,6 +127,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 1056,
+      "size": 1056,
       "length": 1024,
       "capacity": 1024
     }]'''
@@ -137,17 +141,20 @@ includes = ["folly/FBString.h", "utility"]
     expect_json_v2 = '''[{
       "staticSize": 48,
       "exclusiveSize": 0,
+      "size": 1080,
       "members": [
         {
           "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
           "staticSize": 24,
           "exclusiveSize": 1056,
+          "size": 1056,
           "length": 1024,
           "capacity": 1024
         }, {
           "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
           "staticSize": 24,
           "exclusiveSize": 24,
+          "size": 24,
           "length": 1024,
           "capacity": 1024
         }
diff --git a/test/integration/folly_f14_fast_map.toml b/test/integration/folly_f14_fast_map.toml
index 5f9e509..273c222 100644
--- a/test/integration/folly_f14_fast_map.toml
+++ b/test/integration/folly_f14_fast_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 464,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "size": 72, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "size": 104, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "size": 104,"length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 40,
+          "size": 184,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_fast_set.toml b/test/integration/folly_f14_fast_set.toml
index 486b9ba..24d8abe 100644
--- a/test/integration/folly_f14_fast_set.toml
+++ b/test/integration/folly_f14_fast_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size":304,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "size":56, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "size":88, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "size":72, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "size":88, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_f14_node_map.toml b/test/integration/folly_f14_node_map.toml
index 78d414d..07b5b89 100644
--- a/test/integration/folly_f14_node_map.toml
+++ b/test/integration/folly_f14_node_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size":668,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "size": 96, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "size": 148, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "size": 160, "length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 120,
+          "size": 264,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_node_set.toml b/test/integration/folly_f14_node_set.toml
index 497976c..a68cc7b 100644
--- a/test/integration/folly_f14_node_set.toml
+++ b/test/integration/folly_f14_node_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 500,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 120, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "size": 84, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "size": 128, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "size": 132, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 120, "size": 156, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_f14_value_map.toml b/test/integration/folly_f14_value_map.toml
index fac9829..a1a09e8 100644
--- a/test/integration/folly_f14_value_map.toml
+++ b/test/integration/folly_f14_value_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 464,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "size": 72, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "size": 104, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "size": 104, "length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 40,
+          "size": 184,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_value_set.toml b/test/integration/folly_f14_value_set.toml
index 3f19b31..2cf03df 100644
--- a/test/integration/folly_f14_value_set.toml
+++ b/test/integration/folly_f14_value_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 304,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "size": 56, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "size": 88, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "size": 72, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "size": 88, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_f14_vector_map.toml b/test/integration/folly_f14_vector_map.toml
index 9912141..4126243 100644
--- a/test/integration/folly_f14_vector_map.toml
+++ b/test/integration/folly_f14_vector_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 576,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 64, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 60, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 80, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 64, "size": 88, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 60, "size": 120, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 80, "size": 136, "length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 88,
+          "size": 232,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_vector_set.toml b/test/integration/folly_f14_vector_set.toml
index 3ffad84..c67f649 100644
--- a/test/integration/folly_f14_vector_set.toml
+++ b/test/integration/folly_f14_vector_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 400,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 60, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 64, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 76, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 84, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 60, "size": 72, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 64, "size": 104, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 76, "size": 104, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 84, "size": 120, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_small_vector.toml b/test/integration/folly_small_vector.toml
index 5ae53f3..b97b5ff 100644
--- a/test/integration/folly_small_vector.toml
+++ b/test/integration/folly_small_vector.toml
@@ -4,23 +4,23 @@ includes = ["folly/small_vector.h", "vector"]
     param_types = ["const folly::small_vector<int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "length":0, "capacity":2, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "length":0, "capacity":2}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "size":16, "length":0, "capacity":2}]'
   [cases.int_default_inlined]
     param_types = ["const folly::small_vector<int>&"]
     setup = "return {{1,2}};"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "length":2, "capacity":2, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":8, "length":2, "capacity":2}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":8, "size":16, "length":2, "capacity":2}]'
   [cases.int_default_overflow]
     param_types = ["const folly::small_vector<int>&"]
     setup = "return {{1,2,3,4}};"
     expect_json = '[{"staticSize":16, "dynamicSize":24, "exclusiveSize":40, "length":4, "capacity":6, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":24, "length":4, "capacity":6}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":24, "size":40, "length":4, "capacity":6}]'
 
   [cases.vector_3_empty]
     param_types = ["const folly::small_vector<std::vector<int>, 3>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":80, "dynamicSize":0, "exclusiveSize":80, "length":0, "capacity":3, "elementStaticSize":24}]'
-    expect_json_v2 = '[{"staticSize":80, "exclusiveSize":80, "length":0, "capacity":3}]'
+    expect_json_v2 = '[{"staticSize":80, "exclusiveSize":80, "size":80, "length":0, "capacity":3}]'
   [cases.vector_3_inlined]
     param_types = ["const folly::small_vector<std::vector<int>, 3>&"]
     setup = "return {{ {1,2,3}, {4}, {5,6} }};"
@@ -31,10 +31,10 @@ includes = ["folly/small_vector.h", "vector"]
         {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
     ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":80, "length":3, "exclusiveSize":8, "capacity":3, "members":[
-        {"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3},
-        {"staticSize":24, "exclusiveSize":24, "length":1, "capacity":1},
-        {"staticSize":24, "exclusiveSize":24, "length":2, "capacity":2}
+      {"staticSize":80, "length":3, "exclusiveSize":8, "size":104, "capacity":3, "members":[
+        {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity":3},
+        {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity":1},
+        {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity":2}
     ]}]'''
   [cases.vector_3_overflow]
     param_types = ["const folly::small_vector<std::vector<int>, 3>&"]
@@ -47,15 +47,15 @@ includes = ["folly/small_vector.h", "vector"]
         {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4}
     ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":80, "exclusiveSize":104, "length":4, "capacity":5, "members":[
-        {"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3},
-        {"staticSize":24, "exclusiveSize":24, "length":1, "capacity":1},
-        {"staticSize":24, "exclusiveSize":24, "length":2, "capacity":2},
-        {"staticSize":24, "exclusiveSize":24, "length":1, "capacity":1}
+      {"staticSize":80, "exclusiveSize":104, "size":228, "length":4, "capacity":5, "members":[
+        {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity":3},
+        {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity":1},
+        {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity":2},
+        {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity":1}
     ]}]'''
 
   [cases.int_always_heap]
     param_types = ["const folly::small_vector<int, 0>&"]
     setup = "return {{1}};"
     expect_json = '[{"staticSize":16, "dynamicSize":4, "exclusiveSize":20, "length":1, "capacity":1, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "length":1, "capacity":1}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "size":20, "length":1, "capacity":1}]'
diff --git a/test/integration/folly_sorted_vector_map.toml b/test/integration/folly_sorted_vector_map.toml
index 6d3a2a5..44f91e0 100644
--- a/test/integration/folly_sorted_vector_map.toml
+++ b/test/integration/folly_sorted_vector_map.toml
@@ -4,19 +4,19 @@ includes = ["folly/sorted_vector_types.h", "vector"]
     param_types = ["const folly::sorted_vector_map<int, int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "exclusiveSize":24, "length":0, "capacity":0, "elementStaticSize":8}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size":24, "length":0, "capacity":0, "members":[]}]'
   [cases.int_int_some]
     param_types = ["const folly::sorted_vector_map<int, int>&"]
     setup = "return {{ {1,2}, {3,4} }};"
     expect_json = '[{"staticSize":24, "dynamicSize":16, "exclusiveSize":40, "length":2, "capacity":2, "elementStaticSize":8}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":2, "capacity":2, "members":[
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":40, "length":2, "capacity":2, "members":[
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]},
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}
     ]}]'''
   [cases.int_int_reserve]
@@ -27,14 +27,14 @@ includes = ["folly/sorted_vector_types.h", "vector"]
       return m;
     '''
     expect_json = '[{"staticSize":24, "dynamicSize":80, "exclusiveSize":104, "length":2, "capacity":10, "elementStaticSize":8}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":88, "length":2, "capacity":10, "members":[
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":88, "size":104, "length":2, "capacity":10, "members":[
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]},
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}
     ]}]'''
 
diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py
index 2dbc3ff..b72e371 100644
--- a/test/integration/gen_tests.py
+++ b/test/integration/gen_tests.py
@@ -40,6 +40,7 @@ def add_headers(f, custom_headers, thrift_headers):
 
 #include <oi/oi.h>
 #include <oi/exporters/Json.h>
+#include <oi/result/SizedResult.h>
 
 """
     )
@@ -145,8 +146,8 @@ def add_test_setup(f, config):
         oil_func_body += "    auto pr = oi::exporters::Json(std::cout);\n"
         oil_func_body += "    pr.setPretty(true);\n"
         for i in range(len(case["param_types"])):
-            oil_func_body += f"    auto ret{i} = oi::setupAndIntrospect(a{i}, opts);\n"
-            oil_func_body += f"    pr.print(*ret{i});\n"
+            oil_func_body += f"    auto ret{i} = oi::result::SizedResult(*oi::setupAndIntrospect(a{i}, opts));\n"
+            oil_func_body += f"    pr.print(ret{i});\n"
 
         f.write(
             define_traceable_func(
diff --git a/test/integration/inheritance_access.toml b/test/integration/inheritance_access.toml
index faeb35c..9751052 100644
--- a/test/integration/inheritance_access.toml
+++ b/test/integration/inheritance_access.toml
@@ -21,27 +21,33 @@ definitions = '''
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"},
-        {"name":"public_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"},
+        {"name":"public_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
   [cases.protected]
     param_types = ["const Protected&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"},
-        {"name":"protected_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"},
+        {"name":"protected_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
   [cases.private]
     param_types = ["const Private&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"},
-        {"name":"private_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"},
+        {"name":"private_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
   [cases.public_as_base]
     param_types = ["const Base&"]
@@ -49,6 +55,8 @@ definitions = '''
     setup = "return {};"
     expect_json = '''[{
       "staticSize":4,
+      "exclusiveSize":0,
+      "size":4,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
diff --git a/test/integration/inheritance_multiple.toml b/test/integration/inheritance_multiple.toml
index 61d725f..53f956f 100644
--- a/test/integration/inheritance_multiple.toml
+++ b/test/integration/inheritance_multiple.toml
@@ -37,11 +37,12 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":24,
       "exclusiveSize":0,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "exclusiveSize":4},
-        {"name":"b", "staticSize":4, "exclusiveSize":4},
-        {"name":"c", "staticSize":4, "exclusiveSize":4},
-        {"name":"d", "staticSize":4, "exclusiveSize":4},
-        {"name":"e", "staticSize":4, "exclusiveSize":4},
-        {"name":"f", "staticSize":4, "exclusiveSize":4}
+        {"name":"a", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"c", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"d", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"e", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"f", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}]'''
diff --git a/test/integration/packed.toml b/test/integration/packed.toml
index 2bbef02..0e45479 100644
--- a/test/integration/packed.toml
+++ b/test/integration/packed.toml
@@ -12,8 +12,10 @@ definitions = '''
     expect_json = '''[{
       "staticSize":17,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size":17,
       "members":[
-        {"name":"p", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":1, "dynamicSize":0},
-        {"name":"x", "staticSize":8, "dynamicSize":0}
+        {"name":"p", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1},
+        {"name":"x", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
diff --git a/test/integration/padding.toml b/test/integration/padding.toml
index 6cc17d1..5001be6 100644
--- a/test/integration/padding.toml
+++ b/test/integration/padding.toml
@@ -65,10 +65,11 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize": 7,
       "members":[
-        { "name":"a", "staticSize":8, "dynamicSize":0 },
-        { "name":"b", "staticSize":1, "dynamicSize":0 },
-        { "name":"c", "staticSize":8, "dynamicSize":0 }
+        { "name":"a", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
+        { "name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1 },
+        { "name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 }
       ]}]'''
 
   [cases.nested_padding]
@@ -77,18 +78,22 @@ definitions = '''
     expect_json = '''[{
       "staticSize":48,
       "dynamicSize":0,
+      "exclusiveSize":7,
+      "size":48,
       "members":[
-        { "name":"a", "staticSize":8, "dynamicSize":0 },
-        { "name":"b", "staticSize":1, "dynamicSize":0 },
-        { "name":"c", "staticSize":8, "dynamicSize":0 },
+        { "name":"a", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
+        { "name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1 },
+        { "name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
         {
           "name":"d",
           "staticSize":24,
           "dynamicSize":0,
+          "exclusiveSize":7,
+          "size":24,
           "members": [
-            { "name":"a", "staticSize":8, "dynamicSize":0 },
-            { "name":"b", "staticSize":1, "dynamicSize":0 },
-            { "name":"c", "staticSize":8, "dynamicSize":0 }
+            { "name":"a", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
+            { "name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1 },
+            { "name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 }
         ]}
       ]}]'''
 
@@ -103,5 +108,6 @@ definitions = '''
     }]'''
     expect_json_v2 = '''[{
       "staticSize": 104,
-      "exclusiveSize": 32
+      "exclusiveSize": 32,
+      "size": 104
     }]'''
diff --git a/test/integration/pointers.toml b/test/integration/pointers.toml
index 389ba29..71bf2ad 100644
--- a/test/integration/pointers.toml
+++ b/test/integration/pointers.toml
@@ -153,10 +153,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize":4,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_primitive_ptrs_null]
     param_types = ["const PrimitivePtrs&"]
@@ -165,10 +167,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize":4,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
 
 
@@ -189,8 +193,10 @@ definitions = '''
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"vec", "staticSize":8, "dynamicSize":0}
+        {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_vector_ptr_null]
     param_types = ["const VectorPtr&"]
@@ -199,8 +205,10 @@ definitions = '''
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"vec", "staticSize":8, "dynamicSize":0}
+        {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
 
 
@@ -242,10 +250,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize":4,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.feature_config]
     oil_disable = "oil can't chase raw pointers safely"
diff --git a/test/integration/primitives.toml b/test/integration/primitives.toml
index 429712f..94784fb 100644
--- a/test/integration/primitives.toml
+++ b/test/integration/primitives.toml
@@ -2,76 +2,76 @@
   [cases.short]
     param_types = ["short"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":2, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":2, "dynamicSize":0, "exclusiveSize":2, "size":2}]'
   [cases.ushort]
     param_types = ["unsigned short"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":2, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":2, "dynamicSize":0, "exclusiveSize":2, "size":2}]'
   [cases.int]
     param_types = ["int"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.uint]
     param_types = ["unsigned int"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.long]
     param_types = ["long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.ulong]
     param_types = ["unsigned long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.longlong]
     param_types = ["long long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.ulonglong]
     param_types = ["unsigned long long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.bool]
     param_types = ["bool"]
     setup = "return true;"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.char]
     param_types = ["char"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.uchar]
     param_types = ["unsigned char"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.schar]
     param_types = ["signed char"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.wchar_t]
     param_types = ["wchar_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.char8_t]
     param_types = ["char8_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.char16_t]
     param_types = ["char16_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":2, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":2, "dynamicSize":0, "exclusiveSize":2, "size":2}]'
   [cases.char32_t]
     param_types = ["char32_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.float]
     param_types = ["float"]
     setup = "return 3.14;"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.double]
     param_types = ["double"]
     setup = "return 3.14;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.long_double]
     param_types = ["long double"]
     setup = "return 3.14;"
-    expect_json = '[{"staticSize":16, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "size":16}]'
diff --git a/test/integration/runner_common.cpp b/test/integration/runner_common.cpp
index c86bd59..3c8525a 100644
--- a/test/integration/runner_common.cpp
+++ b/test/integration/runner_common.cpp
@@ -365,6 +365,8 @@ void IntegrationBase::compare_json(const bpt::ptree& expected_json,
         }
       } else if (key == "dynamicSize" && val.get_value<size_t>() == 0) {
         continue;
+      } else if (key == "size") {
+        continue;
       }
 
       ADD_FAILURE() << "Expected key not found in output: " << curr_key;
diff --git a/test/integration/simple.toml b/test/integration/simple.toml
index 71a215a..0945f70 100644
--- a/test/integration/simple.toml
+++ b/test/integration/simple.toml
@@ -23,10 +23,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
+      "exclusiveSize": 3,
+      "size": 16,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":1, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
+        {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
   [cases.class]
     param_types = ["const SimpleClass&"]
@@ -34,15 +36,19 @@ definitions = '''
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
+      "exclusiveSize": 3,
+      "size": 16,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":1, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
+        {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
   [cases.union]
     param_types = ["const SimpleUnion&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
-      "dynamicSize":0
+      "dynamicSize":0,
+      "exclusiveSize":8,
+      "size":8
     }]'''
diff --git a/test/integration/sorted_vector_set.toml b/test/integration/sorted_vector_set.toml
index 8e37abd..0a4e416 100644
--- a/test/integration/sorted_vector_set.toml
+++ b/test/integration/sorted_vector_set.toml
@@ -23,6 +23,7 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize": 24,
       "exclusiveSize": 24,
+      "size": 24,
       "length": 0,
       "capacity": 0
     }]'''
@@ -56,6 +57,7 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize": 24,
       "exclusiveSize": 28,
+      "size": 40,
       "length": 3,
       "capacity": 4
     }]'''
diff --git a/test/integration/std_list.toml b/test/integration/std_list.toml
index dfe2d8f..92c039a 100644
--- a/test/integration/std_list.toml
+++ b/test/integration/std_list.toml
@@ -13,30 +13,30 @@ definitions = '''
     param_types = ["const std::list<int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size":24, "length":0, "capacity":0, "members":[]}]'
   [cases.int_some]
     param_types = ["const std::list<int>&"]
     setup = "return {{1,2,3}};"
     expect_json = '[{"staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity":3, "members":[
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4}
     ]}]'''
   [cases.struct_some]
     param_types = ["const std::list<SimpleStruct>&"]
     setup = "return {{{}, {}, {}}};"
     expect_json = '[{"staticSize":24, "dynamicSize":48, "length":3, "capacity":3, "elementStaticSize":16}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
-      {"staticSize":16, "exclusiveSize":3},
-      {"staticSize":16, "exclusiveSize":3},
-      {"staticSize":16, "exclusiveSize":3}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":72, "length":3, "capacity":3, "members":[
+      {"staticSize":16, "exclusiveSize":3, "size":16},
+      {"staticSize":16, "exclusiveSize":3, "size":16},
+      {"staticSize":16, "exclusiveSize":3, "size":16}
     ]}]'''
   [cases.list_int_empty]
     param_types = ["const std::list<std::list<int>>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":24}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size":24, "length":0, "capacity":0, "members":[]}]'
   [cases.list_int_some]
     param_types = ["const std::list<std::list<int>>&"]
     setup = "return {{{1,2,3},{4},{5,6}}};"
@@ -52,8 +52,8 @@ definitions = '''
         {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4},
         {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
       ]}]'''
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[
-      {"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[]},
-      {"staticSize":24, "exclusiveSize":24, "length":1, "capacity": 1, "members":[]},
-      {"staticSize":24, "exclusiveSize":24, "length":2, "capacity": 2, "members":[]}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":120, "length":3, "capacity": 3, "members":[
+      {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity": 3},
+      {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity": 1},
+      {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity": 2}
     ]}]'''
diff --git a/test/integration/std_list_del_allocator.toml b/test/integration/std_list_del_allocator.toml
index b986996..c85d3ab 100644
--- a/test/integration/std_list_del_allocator.toml
+++ b/test/integration/std_list_del_allocator.toml
@@ -60,7 +60,8 @@ includes = ["list"]
     expect_json_v2 = '''[{
       "staticSize": 48,
       "exclusiveSize": 0,
+      "size": 60,
       "members": [
-        {"name": "v1", "staticSize": 24, "exclusiveSize": 24, "length": 1, "capacity": 1},
-        {"name": "v2", "staticSize": 24, "exclusiveSize": 24, "length": 2, "capacity": 2}
+        {"name": "v1", "staticSize": 24, "exclusiveSize": 24, "size": 28, "length": 1, "capacity": 1},
+        {"name": "v2", "staticSize": 24, "exclusiveSize": 24, "size": 32, "length": 2, "capacity": 2}
       ]}]'''
diff --git a/test/integration/std_map_custom_comparator.toml b/test/integration/std_map_custom_comparator.toml
index f8edd24..5f30b36 100644
--- a/test/integration/std_map_custom_comparator.toml
+++ b/test/integration/std_map_custom_comparator.toml
@@ -70,10 +70,12 @@ includes = ["map", "functional"]
     expect_json_v2 = '''[{
       "staticSize":8184,
       "exclusiveSize":0,
+      "size":9168,
       "members":[
         {"name":"m1",
          "staticSize":48,
          "exclusiveSize":48,
+         "size":192,
          "length":3,
          "capacity":3,
          "members": [
@@ -81,7 +83,7 @@ includes = ["map", "functional"]
           {},
           {}
          ]},
-        {"name":"m2", "staticSize":48, "exclusiveSize":48, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":48, "exclusiveSize":48, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":8040, "exclusiveSize":8040, "length":9, "capacity":9}
+        {"name":"m2", "staticSize":48, "exclusiveSize":48, "size":248, "length":5, "capacity":5},
+        {"name":"m3", "staticSize":48, "exclusiveSize":48, "size":328, "length":7, "capacity":7},
+        {"name":"m4", "staticSize":8040, "exclusiveSize":8040, "size":8400, "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_multimap_custom_comparator.toml b/test/integration/std_multimap_custom_comparator.toml
index 7f8171f..883c4cd 100644
--- a/test/integration/std_multimap_custom_comparator.toml
+++ b/test/integration/std_multimap_custom_comparator.toml
@@ -38,16 +38,18 @@ includes = ["map"]
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize":0,
+      "size":440,
       "members":[
         {"name":"m1",
          "staticSize":48,
          "exclusiveSize":48,
+         "size":192,
          "length":3,
          "capacity":3,
          "members": [
-          {"name":"[]", "staticSize":48, "exclusiveSize":36},
+          {"name":"[]", "staticSize":48, "exclusiveSize":36, "size":48},
           {},
           {}
          ]},
-        {"name":"m2", "staticSize":48, "exclusiveSize":48, "length":5, "capacity":5}
+        {"name":"m2", "staticSize":48, "exclusiveSize":48, "size":248, "length":5, "capacity":5}
       ]}]'''
diff --git a/test/integration/std_multiset_custom_comparator.toml b/test/integration/std_multiset_custom_comparator.toml
index 04d5da8..e8ef1aa 100644
--- a/test/integration/std_multiset_custom_comparator.toml
+++ b/test/integration/std_multiset_custom_comparator.toml
@@ -65,9 +65,10 @@ includes = ["set", "functional"]
     expect_json_v2 = '''[{
       "staticSize":8184,
       "exclusiveSize": 0,
+      "size":9144,
       "members":[
-        {"name":"m1", "staticSize":48, "exclusiveSize": 156, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":48, "exclusiveSize": 228, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":48, "exclusiveSize": 300, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":8040, "exclusiveSize": 8364, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":48, "exclusiveSize": 156, "size": 168, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":48, "exclusiveSize": 228, "size": 248, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":48, "exclusiveSize": 300, "size": 328, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":8040, "exclusiveSize": 8364, "size": 8400, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/std_optional.toml b/test/integration/std_optional.toml
index 7475e28..9da0271 100644
--- a/test/integration/std_optional.toml
+++ b/test/integration/std_optional.toml
@@ -20,6 +20,7 @@ includes = ["optional", "cstdint", "vector"]
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 16,
         "length": 0,
         "capacity": 1,
         "members": []
@@ -46,6 +47,7 @@ includes = ["optional", "cstdint", "vector"]
       {
         "staticSize": 16,
         "exclusiveSize": 8,
+        "size": 16,
         "length": 1,
         "capacity": 1,
         "members": [
@@ -77,6 +79,7 @@ includes = ["optional", "cstdint", "vector"]
       {
         "staticSize": 32,
         "exclusiveSize": 32,
+        "size": 32,
         "length": 0,
         "capacity": 1,
         "members": []
@@ -111,12 +114,14 @@ includes = ["optional", "cstdint", "vector"]
       {
         "staticSize": 32,
         "exclusiveSize": 8,
+        "size": 72,
         "length": 1,
         "capacity": 1,
         "members": [
           {
             "staticSize": 24,
             "exclusiveSize": 24,
+            "size": 64,
             "length": 5,
             "capacity": 5
           }
diff --git a/test/integration/std_pair.toml b/test/integration/std_pair.toml
index c60e784..1780951 100644
--- a/test/integration/std_pair.toml
+++ b/test/integration/std_pair.toml
@@ -14,9 +14,9 @@ includes = ["vector", "utility", "cstdint"]
     ]
     '''
     expect_json_v2 = '''[
-      {"staticSize": 16, "exclusiveSize": 0, "members": [
-        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8},
-        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8}
+      {"staticSize": 16, "exclusiveSize": 0, "size": 16, "members": [
+        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8, "size": 8},
+        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8, "size": 8}
       ]}
     ]'''
   [cases.uint64_uint32]
@@ -34,9 +34,9 @@ includes = ["vector", "utility", "cstdint"]
     ]
     '''
     expect_json_v2 = '''[
-      {"staticSize": 16, "exclusiveSize": 4, "members": [
-        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8},
-        {"typeNames": ["uint32_t"], "staticSize": 4, "exclusiveSize": 4}
+      {"staticSize": 16, "exclusiveSize": 4, "size": 16, "members": [
+        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8, "size": 8},
+        {"typeNames": ["uint32_t"], "staticSize": 4, "exclusiveSize": 4, "size": 4}
       ]}
     ]'''
 
@@ -48,9 +48,9 @@ includes = ["vector", "utility", "cstdint"]
     param_types = ["std::pair<uint64_t, uint64_t*>&"]
     setup = "return {{0, nullptr}};"
     expect_json_v2 = '''[
-      {"staticSize": 16, "exclusiveSize": 0, "members": [
-        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8},
-        {"typeNames": ["uintptr_t (stubbed)"], "staticSize": 8, "exclusiveSize": 8}
+      {"staticSize": 16, "exclusiveSize": 0, "size": 16, "members": [
+        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8, "size": 8},
+        {"typeNames": ["uintptr_t (stubbed)"], "staticSize": 8, "exclusiveSize": 8, "size": 8}
       ]}
     ]'''
 
@@ -82,8 +82,8 @@ includes = ["vector", "utility", "cstdint"]
     ]
     '''
     expect_json_v2 = '''[
-      {"staticSize": 48, "exclusiveSize": 0, "members": [
-        {"typeNames": ["std::vector<uint64_t, std::allocator<uint64_t>>"], "staticSize": 24, "exclusiveSize": 24},
-        {"typeNames": ["std::vector<uint64_t, std::allocator<uint64_t>>"], "staticSize": 24, "exclusiveSize": 24}
+      {"staticSize": 48, "exclusiveSize": 0, "size": 104, "members": [
+        {"typeNames": ["std::vector<uint64_t, std::allocator<uint64_t>>"], "staticSize": 24, "exclusiveSize": 24, "size": 48},
+        {"typeNames": ["std::vector<uint64_t, std::allocator<uint64_t>>"], "staticSize": 24, "exclusiveSize": 24, "size": 56}
       ]}
     ]'''
diff --git a/test/integration/std_set_custom_comparator.toml b/test/integration/std_set_custom_comparator.toml
index 66031f7..49d07f8 100644
--- a/test/integration/std_set_custom_comparator.toml
+++ b/test/integration/std_set_custom_comparator.toml
@@ -65,9 +65,10 @@ includes = ["set", "functional"]
     expect_json_v2 = '''[{
       "staticSize":8184,
       "exclusiveSize": 0,
+      "size": 9144,
       "members":[
-        {"name":"m1", "staticSize":48, "exclusiveSize": 156, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":48, "exclusiveSize": 228, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":48, "exclusiveSize": 300, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":8040, "exclusiveSize": 8364, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":48, "exclusiveSize": 156, "size": 168, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":48, "exclusiveSize": 228, "size": 248, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":48, "exclusiveSize": 300, "size": 328, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":8040, "exclusiveSize": 8364, "size": 8400, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/std_smart_ptr.toml b/test/integration/std_smart_ptr.toml
index 8fc66bf..dfb86ae 100644
--- a/test/integration/std_smart_ptr.toml
+++ b/test/integration/std_smart_ptr.toml
@@ -26,6 +26,7 @@ definitions = '''
       {
         "staticSize": 8,
         "exclusiveSize": 8,
+        "size": 8,
         "length": 0,
         "capacity": 1
       }
@@ -50,6 +51,7 @@ definitions = '''
       {
         "staticSize": 8,
         "exclusiveSize": 8,
+        "size": 16,
         "length": 1,
         "capacity": 1
       }
@@ -74,6 +76,7 @@ definitions = '''
       {
         "staticSize": 8,
         "exclusiveSize": 8,
+        "size": 8,
         "length": 0,
         "capacity": 1
       }
@@ -106,6 +109,7 @@ definitions = '''
       {
         "staticSize": 8,
         "exclusiveSize": 8,
+        "size": 72,
         "length": 1,
         "capacity": 1,
         "members": [
@@ -134,7 +138,8 @@ definitions = '''
     [
       {
         "staticSize": 16,
-        "exclusiveSize": 16
+        "exclusiveSize": 16,
+        "size": 16
       }
     ]
     '''
@@ -153,7 +158,8 @@ definitions = '''
     [
       {
         "staticSize": 16,
-        "exclusiveSize": 16
+        "exclusiveSize": 16,
+        "size": 16
       }
     ]
     '''
@@ -176,6 +182,7 @@ definitions = '''
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 16,
         "length": 0,
         "capacity": 1
       }
@@ -200,6 +207,7 @@ definitions = '''
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 24,
         "length": 1,
         "capacity": 1,
         "members": [
@@ -227,6 +235,7 @@ definitions = '''
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 16,
         "length": 0,
         "capacity": 1
       }
@@ -257,12 +266,14 @@ definitions = '''
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 80,
         "length": 1,
         "capacity": 1,
         "members": [
           {
             "staticSize": 24,
             "exclusiveSize": 24,
+            "size": 64,
             "length": 5,
             "capacity": 5
           }
@@ -278,7 +289,8 @@ definitions = '''
       {
         "staticSize": 16,
         "dynamicSize": 0,
-        "exclusiveSize": 16
+        "exclusiveSize": 16,
+        "size": 16
       }
     ]
     '''
@@ -290,7 +302,8 @@ definitions = '''
       {
         "staticSize": 16,
         "dynamicSize": 0,
-        "exclusiveSize": 16
+        "exclusiveSize": 16,
+        "size": 16
       }
     ]
     '''
@@ -307,7 +320,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_void_empty]
     param_types = ["std::weak_ptr<void>&"]
     setup = "return std::weak_ptr<void>();"
@@ -321,7 +334,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_present]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
     setup = '''
@@ -339,7 +352,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_expired]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
     setup = '''
@@ -357,7 +370,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_present_chase]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
     cli_options = ["-fchase-raw-pointers"]
@@ -376,7 +389,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_expired_chase]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
     cli_options = ["-fchase-raw-pointers"]
@@ -393,4 +406,4 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
diff --git a/test/integration/std_unordered_map_custom_operator.toml b/test/integration/std_unordered_map_custom_operator.toml
index 3dcf55d..ea78bff 100644
--- a/test/integration/std_unordered_map_custom_operator.toml
+++ b/test/integration/std_unordered_map_custom_operator.toml
@@ -66,9 +66,11 @@ includes = ["unordered_map"]
       ]}]'''
     expect_json_v2 = '''[{
       "staticSize":480,
+      "exclusiveSize": 0,
+      "size": 1952,
       "members":[
-        {"name":"m1", "staticSize":56, "exclusiveSize":220, "length":3, "capacity":3},
-        {"name":"m2", "staticSize":120, "exclusiveSize":324, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":120, "exclusiveSize":364, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":184, "exclusiveSize":468, "length":9, "capacity":9}
+        {"name":"m1", "staticSize":56, "exclusiveSize":220, "size":292, "length":3, "capacity":3},
+        {"name":"m2", "staticSize":120, "exclusiveSize":324, "size":444,  "length":5, "capacity":5},
+        {"name":"m3", "staticSize":120, "exclusiveSize":364, "size":532,  "length":7, "capacity":7},
+        {"name":"m4", "staticSize":184, "exclusiveSize":468, "size":684,  "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_unordered_multimap_custom_operator.toml b/test/integration/std_unordered_multimap_custom_operator.toml
index 37fe213..f36a296 100644
--- a/test/integration/std_unordered_multimap_custom_operator.toml
+++ b/test/integration/std_unordered_multimap_custom_operator.toml
@@ -66,9 +66,11 @@ includes = ["unordered_map"]
       ]}]'''
     expect_json_v2 = '''[{
       "staticSize":480,
+      "exclusiveSize":0,
+      "size":1952,
       "members":[
-        {"name":"m1", "staticSize":56, "exclusiveSize":220, "length":3, "capacity":3},
-        {"name":"m2", "staticSize":120, "exclusiveSize":324, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":120, "exclusiveSize":364, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":184, "exclusiveSize":468, "length":9, "capacity":9}
+        {"name":"m1", "staticSize":56, "exclusiveSize":220, "size":292, "length":3, "capacity":3},
+        {"name":"m2", "staticSize":120, "exclusiveSize":324, "size":444,  "length":5, "capacity":5},
+        {"name":"m3", "staticSize":120, "exclusiveSize":364, "size":532,  "length":7, "capacity":7},
+        {"name":"m4", "staticSize":184, "exclusiveSize":468, "size":684,  "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_unordered_multiset_custom_operator.toml b/test/integration/std_unordered_multiset_custom_operator.toml
index ef08dcb..d9ce1cc 100644
--- a/test/integration/std_unordered_multiset_custom_operator.toml
+++ b/test/integration/std_unordered_multiset_custom_operator.toml
@@ -66,9 +66,11 @@ includes = ["unordered_set"]
       ]}]'''
     expect_json_v2 = '''[{
       "staticSize":480,
+      "exclusiveSize":0,
+      "size":1472,
       "members":[
-        {"name":"m1", "staticSize":56, "exclusiveSize":220, "length":3, "capacity":3},
-        {"name":"m2", "staticSize":120, "exclusiveSize":324, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":120, "exclusiveSize":364, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":184, "exclusiveSize":468, "length":9, "capacity":9}
+        {"name":"m1", "staticSize":56, "exclusiveSize":220, "size":232, "length":3, "capacity":3},
+        {"name":"m2", "staticSize":120, "exclusiveSize":324, "size":344, "length":5, "capacity":5},
+        {"name":"m3", "staticSize":120, "exclusiveSize":364, "size":392, "length":7, "capacity":7},
+        {"name":"m4", "staticSize":184, "exclusiveSize":468, "size":504, "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_unordered_set_custom_operator.toml b/test/integration/std_unordered_set_custom_operator.toml
index 3c01f6c..797c5ea 100644
--- a/test/integration/std_unordered_set_custom_operator.toml
+++ b/test/integration/std_unordered_set_custom_operator.toml
@@ -66,9 +66,11 @@ includes = ["unordered_set"]
       ]}]'''
     expect_json_v2 = '''[{
       "staticSize":480,
+      "exclusiveSize":0,
+      "size":1472,
       "members":[
-        {"name":"m1", "staticSize":56, "exclusiveSize":220, "length":3, "capacity":3},
-        {"name":"m2", "staticSize":120, "exclusiveSize":324, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":120, "exclusiveSize":364, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":184, "exclusiveSize":468, "length":9, "capacity":9}
+        {"name":"m1", "staticSize":56, "exclusiveSize":220, "size":232,"length":3, "capacity":3},
+        {"name":"m2", "staticSize":120, "exclusiveSize":324, "size":344, "length":5, "capacity":5},
+        {"name":"m3", "staticSize":120, "exclusiveSize":364, "size":392, "length":7, "capacity":7},
+        {"name":"m4", "staticSize":184, "exclusiveSize":468, "size":504, "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_vector.toml b/test/integration/std_vector.toml
index d466891..20e00d8 100644
--- a/test/integration/std_vector.toml
+++ b/test/integration/std_vector.toml
@@ -13,24 +13,24 @@ definitions = '''
     param_types = ["const std::vector<int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size": 24, "length":0, "capacity":0, "members":[]}]'
   [cases.int_some]
     param_types = ["const std::vector<int>&"]
     setup = "return {{1,2,3}};"
     expect_json = '[{"staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":36,"length":3, "capacity":3, "members":[
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4}
     ]}]'''
   [cases.struct_some]
     param_types = ["const std::vector<SimpleStruct>&"]
     setup = "return {{{}, {}, {}}};"
     expect_json = '[{"staticSize":24, "dynamicSize":48, "length":3, "capacity":3, "elementStaticSize":16}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
-      {"staticSize":16, "exclusiveSize":3},
-      {"staticSize":16, "exclusiveSize":3},
-      {"staticSize":16, "exclusiveSize":3}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":72, "length":3, "capacity":3, "members":[
+      {"staticSize":16, "exclusiveSize":3, "size":16},
+      {"staticSize":16, "exclusiveSize":3, "size":16},
+      {"staticSize":16, "exclusiveSize":3, "size":16}
     ]}]'''
   [cases.bool_empty]
     skip = true # https://github.com/facebookexperimental/object-introspection/issues/14
@@ -46,7 +46,7 @@ definitions = '''
     param_types = ["const std::vector<std::vector<int>>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":24}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size":24, "length":0, "capacity":0, "members":[]}]'
   [cases.vector_int_some]
     param_types = ["const std::vector<std::vector<int>>&"]
     setup = "return {{{1,2,3},{4},{5,6}}};"
@@ -62,10 +62,10 @@ definitions = '''
         {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4},
         {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
       ]}]'''
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[
-      {"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[]},
-      {"staticSize":24, "exclusiveSize":24, "length":1, "capacity": 1, "members":[]},
-      {"staticSize":24, "exclusiveSize":24, "length":2, "capacity": 2, "members":[]}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":120, "length":3, "capacity": 3, "members":[
+      {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity": 3, "members":[]},
+      {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity": 1, "members":[]},
+      {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity": 2, "members":[]}
     ]}]'''
   [cases.reserve]
     param_types = ["const std::vector<int>&"]
@@ -75,8 +75,8 @@ definitions = '''
       return ret;
     '''
     expect_json = '[{"staticSize":24, "dynamicSize":40, "exclusiveSize":64, "length":3, "capacity":10, "elementStaticSize":4}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":52, "length":3, "capacity":10, "members":[
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":52, "size":64, "length":3, "capacity":10, "members":[
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4}
     ]}]'''
diff --git a/test/integration/std_vector_del_allocator.toml b/test/integration/std_vector_del_allocator.toml
index 934bd34..32edcc2 100644
--- a/test/integration/std_vector_del_allocator.toml
+++ b/test/integration/std_vector_del_allocator.toml
@@ -60,18 +60,19 @@ includes = ["vector"]
     expect_json_v2 = '''[{
       "staticSize":48,
       "exclusiveSize":0,
+      "size":60,
       "members":[
-        {"name":"v1", "staticSize":24, "exclusiveSize":24, "length":1, "capacity":1, "members":[
-          {"name":"[]", "staticSize":4, "exclusiveSize":0, "members":[
-            {"name":"a", "staticSize":4, "exclusiveSize":4, "members":[]}
+        {"name":"v1", "staticSize":24, "exclusiveSize":24, "size": 28, "length":1, "capacity":1, "members":[
+          {"name":"[]", "staticSize":4, "exclusiveSize":0, "size": 4, "members":[
+            {"name":"a", "staticSize":4, "exclusiveSize":4, "size": 4, "members":[]}
           ]}
         ]},
-        {"name":"v2", "staticSize":24, "exclusiveSize":24, "length":2, "capacity":2, "members":[
-          {"name":"[]", "staticSize":4, "exclusiveSize":0, "members":[
-            {"name":"b", "staticSize":4, "exclusiveSize":4, "members":[]}
+        {"name":"v2", "staticSize":24, "exclusiveSize":24, "size": 32, "length":2, "capacity":2, "members":[
+          {"name":"[]", "staticSize":4, "exclusiveSize":0, "size": 4, "members":[
+            {"name":"b", "staticSize":4, "exclusiveSize":4, "size": 4, "members":[]}
           ]},
-          {"name":"[]", "staticSize":4, "exclusiveSize":0, "members":[
-            {"name":"b", "staticSize":4, "exclusiveSize":4, "members":[]}
+          {"name":"[]", "staticSize":4, "exclusiveSize":0, "size": 4, "members":[
+            {"name":"b", "staticSize":4, "exclusiveSize":4, "size": 4, "members":[]}
           ]}
         ]}
       ]
diff --git a/test/integration/templates.toml b/test/integration/templates.toml
index b02cfc8..8be4dc9 100644
--- a/test/integration/templates.toml
+++ b/test/integration/templates.toml
@@ -53,10 +53,12 @@ definitions = '''
       "typeName":"ns_templates::TemplatedClass1<std::vector<int, std::allocator<int> > >",
       "staticSize":24,
       "exclusiveSize":0,
+      "size":24,
       "members":[{
         "typeName":"std::vector<int32_t, std::allocator<int32_t>>",
         "staticSize":24,
         "exclusiveSize":24,
+        "size":24,
         "length":0,
         "capacity":0
     }]}]'''
@@ -90,9 +92,11 @@ definitions = '''
       "typeName":"ns_templates::TemplatedClassVal<3>",
       "staticSize":12,
       "exclusiveSize":0,
+      "size":12,
       "members":[{
         "staticSize":12,
         "exclusiveSize":0,
+        "size":12,
         "length":3,
         "capacity":3
     }]}]'''
diff --git a/test/integration/thrift_unions.toml b/test/integration/thrift_unions.toml
index 5f0ba1f..651869b 100644
--- a/test/integration/thrift_unions.toml
+++ b/test/integration/thrift_unions.toml
@@ -37,9 +37,10 @@ namespace cpp2 {
     expect_json_v2 = '''[{
       "staticSize":8,
       "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"typeNames":["storage_type"], "name":"value_", "staticSize":4, "exclusiveSize":4},
-        {"typeNames":["underlying_type_t<cpp2::StaticUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4}
+        {"typeNames":["storage_type"], "name":"value_", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"typeNames":["underlying_type_t<cpp2::StaticUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}]'''
   [cases.dynamic_int]
     param_types = ["const cpp2::DynamicUnion&"]
@@ -58,9 +59,10 @@ namespace cpp2 {
     expect_json_v2 = '''[{
       "staticSize":32,
       "exclusiveSize":4,
+      "size":32,
       "members":[
-        {"typeNames":["storage_type"], "name":"value_", "staticSize":24, "exclusiveSize":24},
-        {"typeNames":["underlying_type_t<cpp2::DynamicUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4}
+        {"typeNames":["storage_type"], "name":"value_", "staticSize":24, "exclusiveSize":24, "size":24},
+        {"typeNames":["underlying_type_t<cpp2::DynamicUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}]'''
   [cases.dynamic_vec]
     param_types = ["const cpp2::DynamicUnion&"]
@@ -79,7 +81,8 @@ namespace cpp2 {
     expect_json_v2 = '''[{
       "staticSize":32,
       "exclusiveSize":4,
+      "size":32,
       "members":[
-        {"typeNames":["storage_type"], "name":"value_", "staticSize":24, "exclusiveSize":24},
-        {"typeNames":["underlying_type_t<cpp2::DynamicUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4}
+        {"typeNames":["storage_type"], "name":"value_", "staticSize":24, "exclusiveSize":24, "size":24},
+        {"typeNames":["underlying_type_t<cpp2::DynamicUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}]'''
diff --git a/test/integration/typedefed_parent.toml b/test/integration/typedefed_parent.toml
index 72c4a57..25631f9 100644
--- a/test/integration/typedefed_parent.toml
+++ b/test/integration/typedefed_parent.toml
@@ -30,9 +30,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":48,
       "exclusiveSize":0,
+      "size":80,
       "members":[
-        {"name":"a", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3},
-        {"name":"b", "staticSize":24, "exclusiveSize":24, "length":5, "capacity":5}
+        {"name":"a", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "size":36},
+        {"name":"b", "staticSize":24, "exclusiveSize":24, "length":5, "capacity":5, "size":44}
       ]}]'''
   [cases.multilevel_typedef_parent]
     param_types = ["const Bar_2&"]
@@ -50,7 +51,8 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":48,
       "exclusiveSize":0,
+      "size":80,
       "members":[
-        {"name":"a", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3},
-        {"name":"c", "staticSize":24, "exclusiveSize":24, "length":5, "capacity":5}
+        {"name":"a", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "size":36},
+        {"name":"c", "staticSize":24, "exclusiveSize":24, "length":5, "capacity":5, "size":44}
       ]}]'''
diff --git a/test/integration/unions.toml b/test/integration/unions.toml
index 42341de..66df45e 100644
--- a/test/integration/unions.toml
+++ b/test/integration/unions.toml
@@ -45,17 +45,17 @@ definitions = '''
     param_types = ["const MyUnion&"]
     setup = "return 123;"
     expect_json = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}]'
-    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]}]'
   [cases.vector]
     param_types = ["const MyUnion&"]
     setup = "return std::vector{1,2,3};"
     expect_json = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}]'
-    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]}]'
   [cases.unordered_map]
     param_types = ["const MyUnion&"]
     setup = 'return std::unordered_map<std::string, std::string>{{"a", "b"}, {"c","d"}};'
     expect_json = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}]'
-    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]}]'
 
   [cases.alignment]
     # Wrap the union in a pair as a way of inferring its alignment
@@ -67,9 +67,9 @@ definitions = '''
         {"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}
       ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[
-        {"staticSize":1, "dynamicSize":0, "exclusiveSize":1},
-        {"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}
+      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "size":64, "members":[
+        {"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1},
+        {"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]}
       ]}]'''
 
   [cases.tagged_int]
@@ -81,9 +81,9 @@ definitions = '''
         {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "NOT":"members"}
       ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[
-        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]},
-        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "members":[]}
+      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "size":64, "members":[
+        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]},
+        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1, "members":[]}
       ]}]'''
   [cases.tagged_vector]
     param_types = ["const TaggedUnion&"]
@@ -94,9 +94,9 @@ definitions = '''
         {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "NOT":"members"}
       ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[
-        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]},
-        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "members":[]}
+      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "size":64, "members":[
+        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]},
+        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1, "members":[]}
       ]}]'''
   [cases.tagged_unordered_map]
     param_types = ["const TaggedUnion&"]
@@ -107,7 +107,7 @@ definitions = '''
         {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "NOT":"members"}
       ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[
-        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]},
-        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "members":[]}
+      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "size":64, "members":[
+        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]},
+        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1, "members":[]}
       ]}]'''

From e7adc0611a647c6f99febbf589cf62e89fa52b5b Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 2 Jan 2024 21:44:00 +0000
Subject: [PATCH 100/188] tbv2: calculate total memory footprint

Add the option to calculate total size (inclusive size) by wrapping the
existing iterator. This change provides a new iterator, `SizedIterator`, which
wraps an existing iterator and adds a new field `size` to the output element.

This is achieved with a two pass algorithm on the existing iterator:
1. Gather metadata for each element. This includes the total size up until that
   element and the range of elements that should be included in the size.
2. Return the result from the underlying iterator with the additional
   field.

This algorithm is `O(N)` time on the number of elements in the iterator and
`O(N)` time, storing 16 bytes per element. This isn't super expensive but is a
lot more than the current algorithm which requires close to constant space.
Because of this I've implemented it as a wrapper on the iterator rather than on
by default, though it is now on in every one of our integration test cases.

Test plan:
- Added to the integration tests for full coverage.
---
 include/oi/exporters/Json.h                   | 162 +++++++++++++++++-
 include/oi/result/SizedResult-inl.h           | 144 ++++++++++++++++
 include/oi/result/SizedResult.h               |  84 +++++++++
 oi/CMakeLists.txt                             |   4 -
 oi/exporters/Json.cpp                         | 146 ----------------
 test/integration/CMakeLists.txt               |   2 +-
 test/integration/anonymous.toml               |  15 +-
 test/integration/arrays.toml                  |  14 +-
 test/integration/enums.toml                   |  16 +-
 test/integration/enums_params.toml            |  10 +-
 test/integration/fbstring.toml                |   7 +
 test/integration/folly_f14_fast_map.toml      |  26 +--
 test/integration/folly_f14_fast_set.toml      |   9 +-
 test/integration/folly_f14_node_map.toml      |  26 +--
 test/integration/folly_f14_node_set.toml      |   9 +-
 test/integration/folly_f14_value_map.toml     |  26 +--
 test/integration/folly_f14_value_set.toml     |   9 +-
 test/integration/folly_f14_vector_map.toml    |  26 +--
 test/integration/folly_f14_vector_set.toml    |   9 +-
 test/integration/folly_small_vector.toml      |  28 +--
 test/integration/folly_sorted_vector_map.toml |  30 ++--
 test/integration/gen_tests.py                 |   5 +-
 test/integration/inheritance_access.toml      |  22 ++-
 test/integration/inheritance_multiple.toml    |  13 +-
 test/integration/packed.toml                  |   8 +-
 test/integration/padding.toml                 |  26 +--
 test/integration/pointers.toml                |  32 ++--
 test/integration/primitives.toml              |  38 ++--
 test/integration/runner_common.cpp            |   2 +
 test/integration/simple.toml                  |  20 ++-
 test/integration/sorted_vector_set.toml       |   2 +
 test/integration/std_list.toml                |  28 +--
 test/integration/std_list_del_allocator.toml  |   5 +-
 .../std_map_custom_comparator.toml            |   8 +-
 .../std_multimap_custom_comparator.toml       |   6 +-
 .../std_multiset_custom_comparator.toml       |   9 +-
 test/integration/std_optional.toml            |   5 +
 test/integration/std_pair.toml                |  24 +--
 .../std_set_custom_comparator.toml            |   9 +-
 test/integration/std_smart_ptr.toml           |  33 ++--
 .../std_unordered_map_custom_operator.toml    |  10 +-
 ...td_unordered_multimap_custom_operator.toml |  10 +-
 ...td_unordered_multiset_custom_operator.toml |  10 +-
 .../std_unordered_set_custom_operator.toml    |  10 +-
 test/integration/std_vector.toml              |  36 ++--
 .../integration/std_vector_del_allocator.toml |  17 +-
 test/integration/templates.toml               |   4 +
 test/integration/thrift_unions.toml           |  15 +-
 test/integration/typedefed_parent.toml        |  10 +-
 test/integration/unions.toml                  |  30 ++--
 50 files changed, 795 insertions(+), 454 deletions(-)
 create mode 100644 include/oi/result/SizedResult-inl.h
 create mode 100644 include/oi/result/SizedResult.h
 delete mode 100644 oi/exporters/Json.cpp

diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h
index 31aa808..dd88ade 100644
--- a/include/oi/exporters/Json.h
+++ b/include/oi/exporters/Json.h
@@ -17,8 +17,10 @@
 #define INCLUDED_OI_EXPORTERS_JSON_H 1
 
 #include <oi/IntrospectionResult.h>
+#include <oi/result/SizedResult.h>
 
 #include <ostream>
+#include <string_view>
 
 namespace oi::exporters {
 
@@ -26,19 +28,173 @@ class Json {
  public:
   Json(std::ostream& out);
 
-  void print(const IntrospectionResult&);
-  void print(IntrospectionResult::const_iterator& it,
-             IntrospectionResult::const_iterator& end);
+  template <typename Res>
+  void print(const Res& r) {
+    auto begin = r.begin();
+    auto end = r.end();
+    return print(begin, end);
+  }
+  template <typename It>
+  void print(It& it, const It& end);
 
   void setPretty(bool pretty) {
     pretty_ = pretty;
   }
 
  private:
+  std::string_view tab() const;
+  std::string_view space() const;
+  std::string_view endl() const;
+  static std::string makeIndent(size_t depth);
+
+  void printStringField(std::string_view name,
+                        std::string_view value,
+                        std::string_view indent);
+  void printBoolField(std::string_view name,
+                      bool value,
+                      std::string_view indent);
+  void printUnsignedField(std::string_view name,
+                          uint64_t value,
+                          std::string_view indent);
+  void printPointerField(std::string_view name,
+                         uintptr_t value,
+                         std::string_view indent);
+  template <typename Rng>
+  void printListField(std::string_view name,
+                      const Rng& range,
+                      std::string_view indent);
+
+  void printFields(const result::Element&, std::string_view indent);
+  template <typename El>
+  void printFields(const result::SizedElement<El>&, std::string_view indent);
+
   bool pretty_ = false;
   std::ostream& out_;
 };
 
+inline Json::Json(std::ostream& out) : out_(out) {
+}
+
+inline std::string_view Json::tab() const {
+  return pretty_ ? "  " : "";
+}
+inline std::string_view Json::space() const {
+  return pretty_ ? " " : "";
+}
+inline std::string_view Json::endl() const {
+  return pretty_ ? "\n" : "";
+}
+inline std::string Json::makeIndent(size_t depth) {
+  depth = std::max(depth, 1UL);
+  return std::string((depth - 1) * 4, ' ');
+}
+
+inline void Json::printStringField(std::string_view name,
+                                   std::string_view value,
+                                   std::string_view indent) {
+  out_ << tab() << '"' << name << '"' << ':' << space() << "\"" << value
+       << "\"," << endl() << indent;
+}
+inline void Json::printBoolField(std::string_view name,
+                                 bool value,
+                                 std::string_view indent) {
+  out_ << tab() << '"' << name << "\":" << space() << value << ',' << endl()
+       << indent;
+}
+inline void Json::printUnsignedField(std::string_view name,
+                                     uint64_t value,
+                                     std::string_view indent) {
+  out_ << tab() << '"' << name << "\":" << space() << value << ',' << endl()
+       << indent;
+}
+inline void Json::printPointerField(std::string_view name,
+                                    uintptr_t value,
+                                    std::string_view indent) {
+  out_ << tab() << '"' << name << "\":" << space() << "\"0x" << std::hex
+       << value << std::dec << "\"," << endl() << indent;
+}
+template <typename Rng>
+void Json::printListField(std::string_view name,
+                          const Rng& range,
+                          std::string_view indent) {
+  out_ << tab() << '"' << name << '"' << ':' << space() << '[';
+  bool first = true;
+  for (const auto& el : range) {
+    if (!std::exchange(first, false))
+      out_ << ',' << space();
+    out_ << '"' << el << '"';
+  }
+  out_ << "]," << endl() << indent;
+}
+
+template <typename El>
+void Json::printFields(const result::SizedElement<El>& el,
+                       std::string_view indent) {
+  printUnsignedField("size", el.size, indent);
+
+  printFields(el.inner(), indent);
+}
+
+inline void Json::printFields(const result::Element& el,
+                              std::string_view indent) {
+  printStringField("name", el.name, indent);
+  printListField("typePath", el.type_path, indent);
+  printListField("typeNames", el.type_names, indent);
+  printUnsignedField("staticSize", el.static_size, indent);
+  printUnsignedField("exclusiveSize", el.exclusive_size, indent);
+  if (el.pointer.has_value())
+    printUnsignedField("pointer", *el.pointer, indent);
+
+  if (const auto* s = std::get_if<result::Element::Scalar>(&el.data)) {
+    printUnsignedField("data", s->n, indent);
+  } else if (const auto* p = std::get_if<result::Element::Pointer>(&el.data)) {
+    printPointerField("data", p->p, indent);
+  } else if (const auto* str = std::get_if<std::string>(&el.data)) {
+    printStringField("data", *str, indent);
+  }
+
+  if (el.container_stats.has_value()) {
+    printUnsignedField("length", el.container_stats->length, indent);
+    printUnsignedField("capacity", el.container_stats->capacity, indent);
+  }
+  if (el.is_set_stats.has_value())
+    printUnsignedField("is_set", el.is_set_stats->is_set, indent);
+}
+
+template <typename It>
+void Json::print(It& it, const It& end) {
+  const auto depth = it->type_path.size();
+
+  const auto thisIndent = pretty_ ? makeIndent(depth) : "";
+  const auto lastIndent = pretty_ ? makeIndent(depth - 1) : "";
+
+  out_ << '[' << endl() << thisIndent;
+
+  bool first = true;
+  while (it != end && it->type_path.size() >= depth) {
+    if (!std::exchange(first, false))
+      out_ << ',' << endl() << thisIndent;
+
+    out_ << '{' << endl() << thisIndent;
+
+    printFields(*it, thisIndent);
+
+    out_ << tab() << "\"members\":" << space();
+    if (++it != end && it->type_path.size() > depth) {
+      print(it, end);
+    } else {
+      out_ << "[]" << endl();
+    }
+
+    out_ << thisIndent << "}";
+  }
+  if (depth == 1) {
+    out_ << endl() << ']' << endl();
+  } else {
+    out_ << endl() << lastIndent << tab() << ']' << endl();
+  }
+}
+
 }  // namespace oi::exporters
 
 #endif
diff --git a/include/oi/result/SizedResult-inl.h b/include/oi/result/SizedResult-inl.h
new file mode 100644
index 0000000..f86ff0c
--- /dev/null
+++ b/include/oi/result/SizedResult-inl.h
@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#if defined(INCLUDED_OI_RESULT_SIZED_ELEMENT_INL_H) || \
+    !defined(INCLUDED_OI_RESULT_SIZED_ELEMENT_H)
+static_assert(false,
+              "SizedResult-inl.h provides inline declarations for "
+              "SizedResult.h and should only "
+              "be included by SizedResult.h");
+#endif
+#define INCLUDED_OI_RESULT_SIZED_ELEMENT_INL_H 1
+
+#include <utility>
+#include <vector>
+
+#include "SizedResult.h"
+
+namespace oi::result {
+
+template <typename Res>
+SizedResult<Res>::SizedResult(Res res) : res_{std::move(res)} {
+}
+
+template <typename Res>
+typename SizedResult<Res>::const_iterator SizedResult<Res>::begin() const {
+  return ++const_iterator{res_.begin(), res_.end()};
+}
+template <typename Res>
+typename SizedResult<Res>::const_iterator SizedResult<Res>::end() const {
+  return res_.end();
+}
+
+template <typename Res>
+SizedResult<Res>::const_iterator::const_iterator(It it, const It& end)
+    : data_{it} {
+  struct StackEntry {
+    size_t index;
+    size_t depth;
+  };
+  std::vector<StackEntry> stack;
+
+  size_t size = 0;
+  size_t count = -1;
+  for (; it != end; ++it) {
+    ++count;
+
+    auto depth = it->type_path.size();
+    while (!stack.empty() && stack.back().depth >= depth) {
+      auto i = stack.back().index;
+      stack.pop_back();
+      helpers_[i].last_child = count - 1;
+    }
+
+    size += it->exclusive_size;
+    helpers_.emplace_back(SizeHelper{.size = size});
+    stack.emplace_back(StackEntry{.index = count, .depth = depth});
+  }
+  while (!stack.empty()) {
+    auto i = stack.back().index;
+    stack.pop_back();
+    helpers_[i].last_child = count;
+  }
+}
+
+template <typename Res>
+SizedResult<Res>::const_iterator::const_iterator(It end) : data_{end} {
+}
+
+template <typename Res>
+typename SizedResult<Res>::const_iterator
+SizedResult<Res>::const_iterator::operator++(int) {
+  auto old = *this;
+  operator++();
+  return old;
+}
+
+template <typename Res>
+typename SizedResult<Res>::const_iterator&
+SizedResult<Res>::const_iterator::operator++() {
+  // The below iterator is already pointing at the first element while this
+  // iterator is not. Skip incrementing it the first time around.
+  if (count_ != 0)
+    ++data_;
+
+  if (count_ == helpers_.size()) {
+    next_.reset();
+    return *this;
+  }
+
+  size_t size = helpers_[helpers_[count_].last_child].size;
+  if (count_ != 0)
+    size -= helpers_[count_ - 1].size;
+
+  next_.emplace(*data_, size);
+  ++count_;
+  return *this;
+}
+
+template <typename Res>
+const typename SizedResult<Res>::Element&
+SizedResult<Res>::const_iterator::operator*() const {
+  return *next_;
+}
+template <typename Res>
+const typename SizedResult<Res>::Element*
+SizedResult<Res>::const_iterator::operator->() const {
+  return &*next_;
+}
+
+template <typename Res>
+bool SizedResult<Res>::const_iterator::operator==(
+    const SizedResult<Res>::const_iterator& that) const {
+  return this->data_ == that.data_;
+}
+
+template <typename Res>
+bool SizedResult<Res>::const_iterator::operator!=(
+    const SizedResult<Res>::const_iterator& that) const {
+  return !(*this == that);
+}
+
+template <typename El>
+SizedElement<El>::SizedElement(const El& el, size_t size_)
+    : El{el}, size{size_} {
+}
+
+template <typename El>
+const El& SizedElement<El>::inner() const {
+  return static_cast<const El&>(*this);
+}
+
+}  // namespace oi::result
diff --git a/include/oi/result/SizedResult.h b/include/oi/result/SizedResult.h
new file mode 100644
index 0000000..983b804
--- /dev/null
+++ b/include/oi/result/SizedResult.h
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef INCLUDED_OI_RESULT_SIZED_ELEMENT_H
+#define INCLUDED_OI_RESULT_SIZED_ELEMENT_H 1
+
+#include <cstddef>
+#include <optional>
+#include <type_traits>
+#include <vector>
+
+namespace oi::result {
+
+template <typename El>
+struct SizedElement : public El {
+  SizedElement(const El& el, size_t size);
+  const El& inner() const;
+
+  size_t size;
+};
+
+template <typename Res>
+class SizedResult {
+ private:
+  using It = std::decay_t<decltype(std::declval<Res&>().begin())>;
+  using ParentEl = std::decay_t<decltype(std::declval<It&>().operator*())>;
+
+ public:
+  using Element = SizedElement<ParentEl>;
+
+ private:
+  struct SizeHelper {
+    size_t size = -1;
+    size_t last_child = -1;
+  };
+
+ public:
+  class const_iterator {
+    friend SizedResult;
+
+   public:
+    bool operator==(const const_iterator& that) const;
+    bool operator!=(const const_iterator& that) const;
+    const Element& operator*() const;
+    const Element* operator->() const;
+    const_iterator& operator++();
+    const_iterator operator++(int);
+
+   private:
+    const_iterator(It start, const It& end);
+    const_iterator(It end);
+
+    It data_;
+
+    std::vector<SizeHelper> helpers_;
+    size_t count_ = 0;
+    std::optional<Element> next_;
+  };
+
+  SizedResult(Res res);
+
+  const_iterator begin() const;
+  const_iterator end() const;
+
+ private:
+  Res res_;
+};
+
+}  // namespace oi::result
+
+#include "SizedResult-inl.h"
+#endif
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index 3754f41..64dbcae 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -55,10 +55,6 @@ target_link_libraries(codegen
   glog::glog
 )
 
-add_library(exporters_json exporters/Json.cpp)
-target_include_directories(exporters_json PUBLIC ${CMAKE_SOURCE_DIR}/include)
-target_link_libraries(exporters_json oil)
-
 add_library(exporters_csv exporters/CSV.cpp)
 target_include_directories(exporters_csv PUBLIC ${CMAKE_SOURCE_DIR}/include)
 target_link_libraries(exporters_csv oil)
diff --git a/oi/exporters/Json.cpp b/oi/exporters/Json.cpp
deleted file mode 100644
index 8a4d33c..0000000
--- a/oi/exporters/Json.cpp
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include <oi/exporters/Json.h>
-
-#include <algorithm>
-#include <stdexcept>
-#include <variant>
-
-template <class>
-inline constexpr bool always_false_v = false;
-
-namespace oi::exporters {
-namespace {
-
-template <typename It>
-void printStringList(std::ostream& out, It it, It end, bool pretty) {
-  out << '[';
-  for (; it != end; ++it) {
-    out << '"' << (*it) << '"';
-    if (it != end - 1)
-      out << (pretty ? ", " : ",");
-  }
-  out << ']';
-}
-
-std::string makeIndent(size_t depth) {
-  depth = std::max(depth, 1UL);
-  return std::string((depth - 1) * 4, ' ');
-}
-
-}  // namespace
-
-Json::Json(std::ostream& out) : out_(out) {
-}
-
-void Json::print(const IntrospectionResult& r) {
-  auto begin = r.cbegin();
-  auto end = r.cend();
-  return print(begin, end);
-}
-
-void Json::print(IntrospectionResult::const_iterator& it,
-                 IntrospectionResult::const_iterator& end) {
-  const auto firstTypePathSize = it->type_path.size();
-
-  const auto indent = pretty_ ? makeIndent(firstTypePathSize) : "";
-  const auto lastIndent =
-      pretty_ ? makeIndent(std::max(firstTypePathSize, 1UL) - 1) : "";
-  const auto* tab = pretty_ ? "  " : "";
-  const auto* space = pretty_ ? " " : "";
-  const auto* endl = pretty_ ? "\n" : "";
-
-  out_ << '[' << endl << indent;
-
-  bool first = true;
-  while (it != end) {
-    if (it->type_path.size() < firstTypePathSize) {
-      // no longer a sibling, must be a sibling of the type we're printing
-      break;
-    }
-
-    if (!first)
-      out_ << ',' << endl << indent;
-    first = false;
-
-    out_ << '{' << endl << indent;
-
-    out_ << tab << "\"name\"" << space << ':' << space << "\"" << it->name
-         << "\"," << endl
-         << indent;
-
-    out_ << tab << "\"typePath\"" << space << ':' << space << "";
-    printStringList(out_, it->type_path.begin(), it->type_path.end(), pretty_);
-    out_ << (pretty_ ? ",\n" : ",") << indent;
-
-    out_ << tab << "\"typeNames\"" << space << ':' << space;
-    printStringList(
-        out_, it->type_names.begin(), it->type_names.end(), pretty_);
-    out_ << ',' << endl << indent;
-
-    out_ << tab << "\"staticSize\":" << space << it->static_size << ',' << endl
-         << indent;
-    out_ << tab << "\"exclusiveSize\":" << space << it->exclusive_size << ','
-         << endl
-         << indent;
-
-    if (it->pointer.has_value()) {
-      out_ << tab << "\"pointer\":" << space << *(it->pointer) << ',' << endl
-           << indent;
-    }
-
-    if (auto* s = std::get_if<result::Element::Scalar>(&it->data)) {
-      out_ << tab << "\"data\":" << space << s->n << ',' << endl << indent;
-    } else if (auto* p = std::get_if<result::Element::Pointer>(&it->data)) {
-      out_ << tab << "\"data\":" << space << "\"0x" << std::hex << p->p
-           << std::dec << "\"," << endl
-           << indent;
-    } else if (auto* str = std::get_if<std::string>(&it->data)) {
-      out_ << tab << "\"data\":" << space << "\"" << *str << "\"," << endl
-           << indent;
-    }
-
-    if (it->container_stats.has_value()) {
-      out_ << tab << "\"length\":" << space << it->container_stats->length
-           << ',' << endl
-           << indent;
-      out_ << tab << "\"capacity\":" << space << it->container_stats->capacity
-           << ',' << endl
-           << indent;
-    }
-    if (it->is_set_stats.has_value()) {
-      out_ << tab << "\"is_set\":" << space << it->is_set_stats->is_set << ','
-           << endl
-           << indent;
-    }
-
-    out_ << tab << "\"members\":" << space;
-    if (++it != end && it->type_path.size() > firstTypePathSize) {
-      print(it, end);
-    } else {
-      out_ << "[]" << endl;
-    }
-
-    out_ << indent << "}";
-  }
-  if (firstTypePathSize == 1) {
-    out_ << endl << ']' << endl;
-  } else {
-    out_ << endl << lastIndent << tab << ']' << endl;
-  }
-}
-
-}  // namespace oi::exporters
diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 12cb239..4d235b5 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -50,7 +50,7 @@ add_executable(integration_test_target
   ${INTEGRATION_TEST_TARGET_SRC}
   folly_shims.cpp)
 target_compile_options(integration_test_target PRIVATE -O1)
-target_link_libraries(integration_test_target PRIVATE oil_jit exporters_json Boost::headers ${Boost_LIBRARIES})
+target_link_libraries(integration_test_target PRIVATE oil_jit Boost::headers ${Boost_LIBRARIES})
 
 add_executable(integration_test_runner ${INTEGRATION_TEST_RUNNER_SRC} runner_common.cpp)
 target_include_directories(integration_test_runner PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
diff --git a/test/integration/anonymous.toml b/test/integration/anonymous.toml
index 6a91293..15b9f10 100644
--- a/test/integration/anonymous.toml
+++ b/test/integration/anonymous.toml
@@ -88,10 +88,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":12,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size": 12,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":4, "dynamicSize":0},
-        {"name":"c", "staticSize":4, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"c", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}
       ]}]'''
 
   [cases.anon_struct]
@@ -207,10 +209,11 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize": 24,
       "exclusiveSize": 10,
+      "size": 24,
       "members": [
-        {"name":"__oi_anon_0", "staticSize":2, "exclusiveSize":2},
-        {"name":"__oi_anon_2", "staticSize":8, "exclusiveSize":8},
-        {"name":"e", "staticSize":4, "exclusiveSize":4, "typeNames":["int32_t"]}
+        {"name":"__oi_anon_0", "staticSize":2, "exclusiveSize":2, "size":2},
+        {"name":"__oi_anon_2", "staticSize":8, "exclusiveSize":8, "size":8},
+        {"name":"e", "staticSize":4, "exclusiveSize":4, "size":4, "typeNames":["int32_t"]}
       ]
     }]'''
 
diff --git a/test/integration/arrays.toml b/test/integration/arrays.toml
index 4a06b13..5074b62 100644
--- a/test/integration/arrays.toml
+++ b/test/integration/arrays.toml
@@ -33,9 +33,11 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":40,
       "exclusiveSize":0,
+      "size":40,
       "members":[{
         "staticSize":40,
         "exclusiveSize":0,
+        "size":40,
         "length":10,
         "capacity":10
       }]}]'''
@@ -56,9 +58,11 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":1,
       "exclusiveSize":1,
+      "size":1,
       "members":[{
         "staticSize":0,
-        "exclusiveSize":0
+        "exclusiveSize":0,
+        "size":1
       }]}]'''
   [cases.multidim_legacy] # Test for legacy behaviour. Remove with OICodeGen
     oil_disable = 'oil only runs on codegen v2'
@@ -86,10 +90,10 @@ definitions = '''
           {"staticSize":12, "dynamicSize":0, "exclusiveSize":12, "length":3, "capacity":3, "elementStaticSize":4}]
       }]}]'''
     expect_json_v2 = '''[
-      {"staticSize":24, "exclusiveSize":0, "members":[
-        {"staticSize":24, "exclusiveSize":0, "length":2, "capacity":2, "members":[
-          {"staticSize":12, "exclusiveSize":0, "length":3, "capacity":3},
-          {"staticSize":12, "exclusiveSize":0, "length":3, "capacity":3}]
+      {"staticSize":24, "exclusiveSize":0, "size":24, "members":[
+        {"staticSize":24, "exclusiveSize":0, "size":24, "length":2, "capacity":2, "members":[
+          {"staticSize":12, "exclusiveSize":0, "size":12, "length":3, "capacity":3},
+          {"staticSize":12, "exclusiveSize":0, "size":12, "length":3, "capacity":3}]
       }]}]'''
   [cases.direct_int10]
     skip = "Direct array arguments don't work"
diff --git a/test/integration/enums.toml b/test/integration/enums.toml
index f112b66..fa91cbc 100644
--- a/test/integration/enums.toml
+++ b/test/integration/enums.toml
@@ -36,31 +36,31 @@ definitions = '''
     param_types = ["ScopedEnum"]
     setup = "return {};"
     expect_json = '[{"staticSize":4, "dynamicSize":0}]'
-    expect_json_v2 = '[{"typeNames": ["ScopedEnum"], "staticSize":4, "exclusiveSize":4}]'
+    expect_json_v2 = '[{"typeNames": ["ScopedEnum"], "staticSize":4, "exclusiveSize":4, "size":4}]'
   [cases.scoped_uint8]
     param_types = ["ScopedEnumUint8"]
     setup = "return {};"
     expect_json = '[{"staticSize":1, "dynamicSize":0}]'
-    expect_json_v2 = '[{"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1}]'
+    expect_json_v2 = '[{"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1}]'
   [cases.unscoped]
     param_types = ["UNSCOPED_ENUM"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.anonymous]
     skip = "TreeBuilder crashes" # https://github.com/facebookexperimental/object-introspection/issues/232
     param_types = ["Holder&"]
     setup = "return {};"
     expect_json = '''[
-      {"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "members":[
-        {"name":"e", "staticSize":4, "dynamicSize":0, "exclusiveSize":4}
+      {"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4, "members":[
+        {"name":"e", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}
     ]}]'''
   [cases.paired_with_element]
     param_types = ["Pair<ScopedEnumUint8, uint8_t>"]
     setup = "return {};"
     expect_json = '[{"staticSize":2, "dynamicSize":0}]'
     expect_json_v2 = '''[
-      {"staticSize": 2, "exclusiveSize": 0, "members": [
-        {"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1},
-        {"typeNames": ["uint8_t"], "staticSize":1, "exclusiveSize":1}
+      {"staticSize": 2, "exclusiveSize": 0, "size":2, "members": [
+        {"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1},
+        {"typeNames": ["uint8_t"], "staticSize":1, "exclusiveSize":1, "size":1}
       ]}
     ]'''
diff --git a/test/integration/enums_params.toml b/test/integration/enums_params.toml
index cf61d6f..dd66f31 100644
--- a/test/integration/enums_params.toml
+++ b/test/integration/enums_params.toml
@@ -39,20 +39,20 @@ definitions = '''
     param_types = ["const std::array<int, static_cast<size_t>(MyNS::ScopedEnum::Two)>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":8, "length":2, "capacity":2, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":8, "dynamicSize":0, "length":2, "capacity":2}]'
+    expect_json_v2 = '[{"staticSize":8, "exclusiveSize":0, "size":8, "length":2, "capacity":2}]'
 
   [cases.scoped_enum_val]
     param_types = ["const MyClass<MyNS::ScopedEnum::One>&"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4}]'
   [cases.scoped_enum_val_gaps]
     param_types = ["const ClassGaps<MyNS::EnumWithGaps::Twenty>&"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4}]'
   [cases.scoped_enum_val_negative]
     param_types = ["const ClassGaps<MyNS::EnumWithGaps::MinusTwo>&"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4}]'
 
   [cases.unscoped_enum_type]
     param_types = ["const std::vector<MyNS::UNSCOPED_ENUM>&"]
@@ -61,4 +61,4 @@ definitions = '''
     param_types = ["const std::array<int, MyNS::ONE>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":4, "length":1, "capacity":1, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":4, "length":1, "capacity":1}]'
+    expect_json_v2 = '[{"staticSize":4, "exclusiveSize":0, "size":4, "length":1, "capacity":1}]'
diff --git a/test/integration/fbstring.toml b/test/integration/fbstring.toml
index 6e68d99..cec73ca 100644
--- a/test/integration/fbstring.toml
+++ b/test/integration/fbstring.toml
@@ -28,6 +28,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 24,
+      "size": 24,
       "length": 0,
       "capacity": 23
     }]'''
@@ -60,6 +61,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 24,
+      "size": 24,
       "length": 6,
       "capacity": 23
     }]'''
@@ -92,6 +94,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 50,
+      "size": 50,
       "length": 26,
       "capacity": 26
     }]'''
@@ -124,6 +127,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 1056,
+      "size": 1056,
       "length": 1024,
       "capacity": 1024
     }]'''
@@ -137,17 +141,20 @@ includes = ["folly/FBString.h", "utility"]
     expect_json_v2 = '''[{
       "staticSize": 48,
       "exclusiveSize": 0,
+      "size": 1080,
       "members": [
         {
           "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
           "staticSize": 24,
           "exclusiveSize": 1056,
+          "size": 1056,
           "length": 1024,
           "capacity": 1024
         }, {
           "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
           "staticSize": 24,
           "exclusiveSize": 24,
+          "size": 24,
           "length": 1024,
           "capacity": 1024
         }
diff --git a/test/integration/folly_f14_fast_map.toml b/test/integration/folly_f14_fast_map.toml
index 5f9e509..273c222 100644
--- a/test/integration/folly_f14_fast_map.toml
+++ b/test/integration/folly_f14_fast_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 464,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "size": 72, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "size": 104, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "size": 104,"length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 40,
+          "size": 184,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_fast_set.toml b/test/integration/folly_f14_fast_set.toml
index 486b9ba..24d8abe 100644
--- a/test/integration/folly_f14_fast_set.toml
+++ b/test/integration/folly_f14_fast_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size":304,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "size":56, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "size":88, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "size":72, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "size":88, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_f14_node_map.toml b/test/integration/folly_f14_node_map.toml
index 78d414d..07b5b89 100644
--- a/test/integration/folly_f14_node_map.toml
+++ b/test/integration/folly_f14_node_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size":668,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "size": 96, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "size": 148, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "size": 160, "length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 120,
+          "size": 264,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_node_set.toml b/test/integration/folly_f14_node_set.toml
index 497976c..a68cc7b 100644
--- a/test/integration/folly_f14_node_set.toml
+++ b/test/integration/folly_f14_node_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 500,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 120, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "size": 84, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "size": 128, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "size": 132, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 120, "size": 156, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_f14_value_map.toml b/test/integration/folly_f14_value_map.toml
index fac9829..a1a09e8 100644
--- a/test/integration/folly_f14_value_map.toml
+++ b/test/integration/folly_f14_value_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 464,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "size": 72, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "size": 104, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "size": 104, "length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 40,
+          "size": 184,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_value_set.toml b/test/integration/folly_f14_value_set.toml
index 3f19b31..2cf03df 100644
--- a/test/integration/folly_f14_value_set.toml
+++ b/test/integration/folly_f14_value_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 304,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "size": 56, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "size": 88, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "size": 72, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "size": 88, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_f14_vector_map.toml b/test/integration/folly_f14_vector_map.toml
index 9912141..4126243 100644
--- a/test/integration/folly_f14_vector_map.toml
+++ b/test/integration/folly_f14_vector_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 576,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 64, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 60, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 80, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 64, "size": 88, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 60, "size": 120, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 80, "size": 136, "length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 88,
+          "size": 232,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_vector_set.toml b/test/integration/folly_f14_vector_set.toml
index 3ffad84..c67f649 100644
--- a/test/integration/folly_f14_vector_set.toml
+++ b/test/integration/folly_f14_vector_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 400,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 60, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 64, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 76, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 84, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 60, "size": 72, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 64, "size": 104, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 76, "size": 104, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 84, "size": 120, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_small_vector.toml b/test/integration/folly_small_vector.toml
index 5ae53f3..b97b5ff 100644
--- a/test/integration/folly_small_vector.toml
+++ b/test/integration/folly_small_vector.toml
@@ -4,23 +4,23 @@ includes = ["folly/small_vector.h", "vector"]
     param_types = ["const folly::small_vector<int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "length":0, "capacity":2, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "length":0, "capacity":2}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "size":16, "length":0, "capacity":2}]'
   [cases.int_default_inlined]
     param_types = ["const folly::small_vector<int>&"]
     setup = "return {{1,2}};"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "length":2, "capacity":2, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":8, "length":2, "capacity":2}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":8, "size":16, "length":2, "capacity":2}]'
   [cases.int_default_overflow]
     param_types = ["const folly::small_vector<int>&"]
     setup = "return {{1,2,3,4}};"
     expect_json = '[{"staticSize":16, "dynamicSize":24, "exclusiveSize":40, "length":4, "capacity":6, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":24, "length":4, "capacity":6}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":24, "size":40, "length":4, "capacity":6}]'
 
   [cases.vector_3_empty]
     param_types = ["const folly::small_vector<std::vector<int>, 3>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":80, "dynamicSize":0, "exclusiveSize":80, "length":0, "capacity":3, "elementStaticSize":24}]'
-    expect_json_v2 = '[{"staticSize":80, "exclusiveSize":80, "length":0, "capacity":3}]'
+    expect_json_v2 = '[{"staticSize":80, "exclusiveSize":80, "size":80, "length":0, "capacity":3}]'
   [cases.vector_3_inlined]
     param_types = ["const folly::small_vector<std::vector<int>, 3>&"]
     setup = "return {{ {1,2,3}, {4}, {5,6} }};"
@@ -31,10 +31,10 @@ includes = ["folly/small_vector.h", "vector"]
         {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
     ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":80, "length":3, "exclusiveSize":8, "capacity":3, "members":[
-        {"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3},
-        {"staticSize":24, "exclusiveSize":24, "length":1, "capacity":1},
-        {"staticSize":24, "exclusiveSize":24, "length":2, "capacity":2}
+      {"staticSize":80, "length":3, "exclusiveSize":8, "size":104, "capacity":3, "members":[
+        {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity":3},
+        {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity":1},
+        {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity":2}
     ]}]'''
   [cases.vector_3_overflow]
     param_types = ["const folly::small_vector<std::vector<int>, 3>&"]
@@ -47,15 +47,15 @@ includes = ["folly/small_vector.h", "vector"]
         {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4}
     ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":80, "exclusiveSize":104, "length":4, "capacity":5, "members":[
-        {"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3},
-        {"staticSize":24, "exclusiveSize":24, "length":1, "capacity":1},
-        {"staticSize":24, "exclusiveSize":24, "length":2, "capacity":2},
-        {"staticSize":24, "exclusiveSize":24, "length":1, "capacity":1}
+      {"staticSize":80, "exclusiveSize":104, "size":228, "length":4, "capacity":5, "members":[
+        {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity":3},
+        {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity":1},
+        {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity":2},
+        {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity":1}
     ]}]'''
 
   [cases.int_always_heap]
     param_types = ["const folly::small_vector<int, 0>&"]
     setup = "return {{1}};"
     expect_json = '[{"staticSize":16, "dynamicSize":4, "exclusiveSize":20, "length":1, "capacity":1, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "length":1, "capacity":1}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "size":20, "length":1, "capacity":1}]'
diff --git a/test/integration/folly_sorted_vector_map.toml b/test/integration/folly_sorted_vector_map.toml
index 6d3a2a5..44f91e0 100644
--- a/test/integration/folly_sorted_vector_map.toml
+++ b/test/integration/folly_sorted_vector_map.toml
@@ -4,19 +4,19 @@ includes = ["folly/sorted_vector_types.h", "vector"]
     param_types = ["const folly::sorted_vector_map<int, int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "exclusiveSize":24, "length":0, "capacity":0, "elementStaticSize":8}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size":24, "length":0, "capacity":0, "members":[]}]'
   [cases.int_int_some]
     param_types = ["const folly::sorted_vector_map<int, int>&"]
     setup = "return {{ {1,2}, {3,4} }};"
     expect_json = '[{"staticSize":24, "dynamicSize":16, "exclusiveSize":40, "length":2, "capacity":2, "elementStaticSize":8}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":2, "capacity":2, "members":[
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":40, "length":2, "capacity":2, "members":[
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]},
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}
     ]}]'''
   [cases.int_int_reserve]
@@ -27,14 +27,14 @@ includes = ["folly/sorted_vector_types.h", "vector"]
       return m;
     '''
     expect_json = '[{"staticSize":24, "dynamicSize":80, "exclusiveSize":104, "length":2, "capacity":10, "elementStaticSize":8}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":88, "length":2, "capacity":10, "members":[
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":88, "size":104, "length":2, "capacity":10, "members":[
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]},
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}
     ]}]'''
 
diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py
index 2dbc3ff..b72e371 100644
--- a/test/integration/gen_tests.py
+++ b/test/integration/gen_tests.py
@@ -40,6 +40,7 @@ def add_headers(f, custom_headers, thrift_headers):
 
 #include <oi/oi.h>
 #include <oi/exporters/Json.h>
+#include <oi/result/SizedResult.h>
 
 """
     )
@@ -145,8 +146,8 @@ def add_test_setup(f, config):
         oil_func_body += "    auto pr = oi::exporters::Json(std::cout);\n"
         oil_func_body += "    pr.setPretty(true);\n"
         for i in range(len(case["param_types"])):
-            oil_func_body += f"    auto ret{i} = oi::setupAndIntrospect(a{i}, opts);\n"
-            oil_func_body += f"    pr.print(*ret{i});\n"
+            oil_func_body += f"    auto ret{i} = oi::result::SizedResult(*oi::setupAndIntrospect(a{i}, opts));\n"
+            oil_func_body += f"    pr.print(ret{i});\n"
 
         f.write(
             define_traceable_func(
diff --git a/test/integration/inheritance_access.toml b/test/integration/inheritance_access.toml
index faeb35c..9751052 100644
--- a/test/integration/inheritance_access.toml
+++ b/test/integration/inheritance_access.toml
@@ -21,27 +21,33 @@ definitions = '''
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"},
-        {"name":"public_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"},
+        {"name":"public_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
   [cases.protected]
     param_types = ["const Protected&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"},
-        {"name":"protected_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"},
+        {"name":"protected_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
   [cases.private]
     param_types = ["const Private&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"},
-        {"name":"private_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"},
+        {"name":"private_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
   [cases.public_as_base]
     param_types = ["const Base&"]
@@ -49,6 +55,8 @@ definitions = '''
     setup = "return {};"
     expect_json = '''[{
       "staticSize":4,
+      "exclusiveSize":0,
+      "size":4,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
diff --git a/test/integration/inheritance_multiple.toml b/test/integration/inheritance_multiple.toml
index 61d725f..53f956f 100644
--- a/test/integration/inheritance_multiple.toml
+++ b/test/integration/inheritance_multiple.toml
@@ -37,11 +37,12 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":24,
       "exclusiveSize":0,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "exclusiveSize":4},
-        {"name":"b", "staticSize":4, "exclusiveSize":4},
-        {"name":"c", "staticSize":4, "exclusiveSize":4},
-        {"name":"d", "staticSize":4, "exclusiveSize":4},
-        {"name":"e", "staticSize":4, "exclusiveSize":4},
-        {"name":"f", "staticSize":4, "exclusiveSize":4}
+        {"name":"a", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"c", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"d", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"e", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"f", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}]'''
diff --git a/test/integration/packed.toml b/test/integration/packed.toml
index 2bbef02..0e45479 100644
--- a/test/integration/packed.toml
+++ b/test/integration/packed.toml
@@ -12,8 +12,10 @@ definitions = '''
     expect_json = '''[{
       "staticSize":17,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size":17,
       "members":[
-        {"name":"p", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":1, "dynamicSize":0},
-        {"name":"x", "staticSize":8, "dynamicSize":0}
+        {"name":"p", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1},
+        {"name":"x", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
diff --git a/test/integration/padding.toml b/test/integration/padding.toml
index 6cc17d1..5001be6 100644
--- a/test/integration/padding.toml
+++ b/test/integration/padding.toml
@@ -65,10 +65,11 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize": 7,
       "members":[
-        { "name":"a", "staticSize":8, "dynamicSize":0 },
-        { "name":"b", "staticSize":1, "dynamicSize":0 },
-        { "name":"c", "staticSize":8, "dynamicSize":0 }
+        { "name":"a", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
+        { "name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1 },
+        { "name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 }
       ]}]'''
 
   [cases.nested_padding]
@@ -77,18 +78,22 @@ definitions = '''
     expect_json = '''[{
       "staticSize":48,
       "dynamicSize":0,
+      "exclusiveSize":7,
+      "size":48,
       "members":[
-        { "name":"a", "staticSize":8, "dynamicSize":0 },
-        { "name":"b", "staticSize":1, "dynamicSize":0 },
-        { "name":"c", "staticSize":8, "dynamicSize":0 },
+        { "name":"a", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
+        { "name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1 },
+        { "name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
         {
           "name":"d",
           "staticSize":24,
           "dynamicSize":0,
+          "exclusiveSize":7,
+          "size":24,
           "members": [
-            { "name":"a", "staticSize":8, "dynamicSize":0 },
-            { "name":"b", "staticSize":1, "dynamicSize":0 },
-            { "name":"c", "staticSize":8, "dynamicSize":0 }
+            { "name":"a", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
+            { "name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1 },
+            { "name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 }
         ]}
       ]}]'''
 
@@ -103,5 +108,6 @@ definitions = '''
     }]'''
     expect_json_v2 = '''[{
       "staticSize": 104,
-      "exclusiveSize": 32
+      "exclusiveSize": 32,
+      "size": 104
     }]'''
diff --git a/test/integration/pointers.toml b/test/integration/pointers.toml
index 389ba29..71bf2ad 100644
--- a/test/integration/pointers.toml
+++ b/test/integration/pointers.toml
@@ -153,10 +153,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize":4,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_primitive_ptrs_null]
     param_types = ["const PrimitivePtrs&"]
@@ -165,10 +167,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize":4,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
 
 
@@ -189,8 +193,10 @@ definitions = '''
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"vec", "staticSize":8, "dynamicSize":0}
+        {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_vector_ptr_null]
     param_types = ["const VectorPtr&"]
@@ -199,8 +205,10 @@ definitions = '''
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"vec", "staticSize":8, "dynamicSize":0}
+        {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
 
 
@@ -242,10 +250,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize":4,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.feature_config]
     oil_disable = "oil can't chase raw pointers safely"
diff --git a/test/integration/primitives.toml b/test/integration/primitives.toml
index 429712f..94784fb 100644
--- a/test/integration/primitives.toml
+++ b/test/integration/primitives.toml
@@ -2,76 +2,76 @@
   [cases.short]
     param_types = ["short"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":2, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":2, "dynamicSize":0, "exclusiveSize":2, "size":2}]'
   [cases.ushort]
     param_types = ["unsigned short"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":2, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":2, "dynamicSize":0, "exclusiveSize":2, "size":2}]'
   [cases.int]
     param_types = ["int"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.uint]
     param_types = ["unsigned int"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.long]
     param_types = ["long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.ulong]
     param_types = ["unsigned long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.longlong]
     param_types = ["long long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.ulonglong]
     param_types = ["unsigned long long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.bool]
     param_types = ["bool"]
     setup = "return true;"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.char]
     param_types = ["char"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.uchar]
     param_types = ["unsigned char"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.schar]
     param_types = ["signed char"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.wchar_t]
     param_types = ["wchar_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.char8_t]
     param_types = ["char8_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.char16_t]
     param_types = ["char16_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":2, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":2, "dynamicSize":0, "exclusiveSize":2, "size":2}]'
   [cases.char32_t]
     param_types = ["char32_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.float]
     param_types = ["float"]
     setup = "return 3.14;"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.double]
     param_types = ["double"]
     setup = "return 3.14;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.long_double]
     param_types = ["long double"]
     setup = "return 3.14;"
-    expect_json = '[{"staticSize":16, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "size":16}]'
diff --git a/test/integration/runner_common.cpp b/test/integration/runner_common.cpp
index c86bd59..3c8525a 100644
--- a/test/integration/runner_common.cpp
+++ b/test/integration/runner_common.cpp
@@ -365,6 +365,8 @@ void IntegrationBase::compare_json(const bpt::ptree& expected_json,
         }
       } else if (key == "dynamicSize" && val.get_value<size_t>() == 0) {
         continue;
+      } else if (key == "size") {
+        continue;
       }
 
       ADD_FAILURE() << "Expected key not found in output: " << curr_key;
diff --git a/test/integration/simple.toml b/test/integration/simple.toml
index 71a215a..0945f70 100644
--- a/test/integration/simple.toml
+++ b/test/integration/simple.toml
@@ -23,10 +23,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
+      "exclusiveSize": 3,
+      "size": 16,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":1, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
+        {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
   [cases.class]
     param_types = ["const SimpleClass&"]
@@ -34,15 +36,19 @@ definitions = '''
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
+      "exclusiveSize": 3,
+      "size": 16,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":1, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
+        {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
   [cases.union]
     param_types = ["const SimpleUnion&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
-      "dynamicSize":0
+      "dynamicSize":0,
+      "exclusiveSize":8,
+      "size":8
     }]'''
diff --git a/test/integration/sorted_vector_set.toml b/test/integration/sorted_vector_set.toml
index 8e37abd..0a4e416 100644
--- a/test/integration/sorted_vector_set.toml
+++ b/test/integration/sorted_vector_set.toml
@@ -23,6 +23,7 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize": 24,
       "exclusiveSize": 24,
+      "size": 24,
       "length": 0,
       "capacity": 0
     }]'''
@@ -56,6 +57,7 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize": 24,
       "exclusiveSize": 28,
+      "size": 40,
       "length": 3,
       "capacity": 4
     }]'''
diff --git a/test/integration/std_list.toml b/test/integration/std_list.toml
index dfe2d8f..92c039a 100644
--- a/test/integration/std_list.toml
+++ b/test/integration/std_list.toml
@@ -13,30 +13,30 @@ definitions = '''
     param_types = ["const std::list<int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size":24, "length":0, "capacity":0, "members":[]}]'
   [cases.int_some]
     param_types = ["const std::list<int>&"]
     setup = "return {{1,2,3}};"
     expect_json = '[{"staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity":3, "members":[
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4}
     ]}]'''
   [cases.struct_some]
     param_types = ["const std::list<SimpleStruct>&"]
     setup = "return {{{}, {}, {}}};"
     expect_json = '[{"staticSize":24, "dynamicSize":48, "length":3, "capacity":3, "elementStaticSize":16}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
-      {"staticSize":16, "exclusiveSize":3},
-      {"staticSize":16, "exclusiveSize":3},
-      {"staticSize":16, "exclusiveSize":3}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":72, "length":3, "capacity":3, "members":[
+      {"staticSize":16, "exclusiveSize":3, "size":16},
+      {"staticSize":16, "exclusiveSize":3, "size":16},
+      {"staticSize":16, "exclusiveSize":3, "size":16}
     ]}]'''
   [cases.list_int_empty]
     param_types = ["const std::list<std::list<int>>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":24}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size":24, "length":0, "capacity":0, "members":[]}]'
   [cases.list_int_some]
     param_types = ["const std::list<std::list<int>>&"]
     setup = "return {{{1,2,3},{4},{5,6}}};"
@@ -52,8 +52,8 @@ definitions = '''
         {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4},
         {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
       ]}]'''
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[
-      {"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[]},
-      {"staticSize":24, "exclusiveSize":24, "length":1, "capacity": 1, "members":[]},
-      {"staticSize":24, "exclusiveSize":24, "length":2, "capacity": 2, "members":[]}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":120, "length":3, "capacity": 3, "members":[
+      {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity": 3},
+      {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity": 1},
+      {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity": 2}
     ]}]'''
diff --git a/test/integration/std_list_del_allocator.toml b/test/integration/std_list_del_allocator.toml
index b986996..c85d3ab 100644
--- a/test/integration/std_list_del_allocator.toml
+++ b/test/integration/std_list_del_allocator.toml
@@ -60,7 +60,8 @@ includes = ["list"]
     expect_json_v2 = '''[{
       "staticSize": 48,
       "exclusiveSize": 0,
+      "size": 60,
       "members": [
-        {"name": "v1", "staticSize": 24, "exclusiveSize": 24, "length": 1, "capacity": 1},
-        {"name": "v2", "staticSize": 24, "exclusiveSize": 24, "length": 2, "capacity": 2}
+        {"name": "v1", "staticSize": 24, "exclusiveSize": 24, "size": 28, "length": 1, "capacity": 1},
+        {"name": "v2", "staticSize": 24, "exclusiveSize": 24, "size": 32, "length": 2, "capacity": 2}
       ]}]'''
diff --git a/test/integration/std_map_custom_comparator.toml b/test/integration/std_map_custom_comparator.toml
index f8edd24..5f30b36 100644
--- a/test/integration/std_map_custom_comparator.toml
+++ b/test/integration/std_map_custom_comparator.toml
@@ -70,10 +70,12 @@ includes = ["map", "functional"]
     expect_json_v2 = '''[{
       "staticSize":8184,
       "exclusiveSize":0,
+      "size":9168,
       "members":[
         {"name":"m1",
          "staticSize":48,
          "exclusiveSize":48,
+         "size":192,
          "length":3,
          "capacity":3,
          "members": [
@@ -81,7 +83,7 @@ includes = ["map", "functional"]
           {},
           {}
          ]},
-        {"name":"m2", "staticSize":48, "exclusiveSize":48, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":48, "exclusiveSize":48, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":8040, "exclusiveSize":8040, "length":9, "capacity":9}
+        {"name":"m2", "staticSize":48, "exclusiveSize":48, "size":248, "length":5, "capacity":5},
+        {"name":"m3", "staticSize":48, "exclusiveSize":48, "size":328, "length":7, "capacity":7},
+        {"name":"m4", "staticSize":8040, "exclusiveSize":8040, "size":8400, "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_multimap_custom_comparator.toml b/test/integration/std_multimap_custom_comparator.toml
index 7f8171f..883c4cd 100644
--- a/test/integration/std_multimap_custom_comparator.toml
+++ b/test/integration/std_multimap_custom_comparator.toml
@@ -38,16 +38,18 @@ includes = ["map"]
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize":0,
+      "size":440,
       "members":[
         {"name":"m1",
          "staticSize":48,
          "exclusiveSize":48,
+         "size":192,
          "length":3,
          "capacity":3,
          "members": [
-          {"name":"[]", "staticSize":48, "exclusiveSize":36},
+          {"name":"[]", "staticSize":48, "exclusiveSize":36, "size":48},
           {},
           {}
          ]},
-        {"name":"m2", "staticSize":48, "exclusiveSize":48, "length":5, "capacity":5}
+        {"name":"m2", "staticSize":48, "exclusiveSize":48, "size":248, "length":5, "capacity":5}
       ]}]'''
diff --git a/test/integration/std_multiset_custom_comparator.toml b/test/integration/std_multiset_custom_comparator.toml
index 04d5da8..e8ef1aa 100644
--- a/test/integration/std_multiset_custom_comparator.toml
+++ b/test/integration/std_multiset_custom_comparator.toml
@@ -65,9 +65,10 @@ includes = ["set", "functional"]
     expect_json_v2 = '''[{
       "staticSize":8184,
       "exclusiveSize": 0,
+      "size":9144,
       "members":[
-        {"name":"m1", "staticSize":48, "exclusiveSize": 156, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":48, "exclusiveSize": 228, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":48, "exclusiveSize": 300, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":8040, "exclusiveSize": 8364, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":48, "exclusiveSize": 156, "size": 168, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":48, "exclusiveSize": 228, "size": 248, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":48, "exclusiveSize": 300, "size": 328, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":8040, "exclusiveSize": 8364, "size": 8400, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/std_optional.toml b/test/integration/std_optional.toml
index 7475e28..9da0271 100644
--- a/test/integration/std_optional.toml
+++ b/test/integration/std_optional.toml
@@ -20,6 +20,7 @@ includes = ["optional", "cstdint", "vector"]
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 16,
         "length": 0,
         "capacity": 1,
         "members": []
@@ -46,6 +47,7 @@ includes = ["optional", "cstdint", "vector"]
       {
         "staticSize": 16,
         "exclusiveSize": 8,
+        "size": 16,
         "length": 1,
         "capacity": 1,
         "members": [
@@ -77,6 +79,7 @@ includes = ["optional", "cstdint", "vector"]
       {
         "staticSize": 32,
         "exclusiveSize": 32,
+        "size": 32,
         "length": 0,
         "capacity": 1,
         "members": []
@@ -111,12 +114,14 @@ includes = ["optional", "cstdint", "vector"]
       {
         "staticSize": 32,
         "exclusiveSize": 8,
+        "size": 72,
         "length": 1,
         "capacity": 1,
         "members": [
           {
             "staticSize": 24,
             "exclusiveSize": 24,
+            "size": 64,
             "length": 5,
             "capacity": 5
           }
diff --git a/test/integration/std_pair.toml b/test/integration/std_pair.toml
index c60e784..1780951 100644
--- a/test/integration/std_pair.toml
+++ b/test/integration/std_pair.toml
@@ -14,9 +14,9 @@ includes = ["vector", "utility", "cstdint"]
     ]
     '''
     expect_json_v2 = '''[
-      {"staticSize": 16, "exclusiveSize": 0, "members": [
-        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8},
-        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8}
+      {"staticSize": 16, "exclusiveSize": 0, "size": 16, "members": [
+        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8, "size": 8},
+        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8, "size": 8}
       ]}
     ]'''
   [cases.uint64_uint32]
@@ -34,9 +34,9 @@ includes = ["vector", "utility", "cstdint"]
     ]
     '''
     expect_json_v2 = '''[
-      {"staticSize": 16, "exclusiveSize": 4, "members": [
-        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8},
-        {"typeNames": ["uint32_t"], "staticSize": 4, "exclusiveSize": 4}
+      {"staticSize": 16, "exclusiveSize": 4, "size": 16, "members": [
+        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8, "size": 8},
+        {"typeNames": ["uint32_t"], "staticSize": 4, "exclusiveSize": 4, "size": 4}
       ]}
     ]'''
 
@@ -48,9 +48,9 @@ includes = ["vector", "utility", "cstdint"]
     param_types = ["std::pair<uint64_t, uint64_t*>&"]
     setup = "return {{0, nullptr}};"
     expect_json_v2 = '''[
-      {"staticSize": 16, "exclusiveSize": 0, "members": [
-        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8},
-        {"typeNames": ["uintptr_t (stubbed)"], "staticSize": 8, "exclusiveSize": 8}
+      {"staticSize": 16, "exclusiveSize": 0, "size": 16, "members": [
+        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8, "size": 8},
+        {"typeNames": ["uintptr_t (stubbed)"], "staticSize": 8, "exclusiveSize": 8, "size": 8}
       ]}
     ]'''
 
@@ -82,8 +82,8 @@ includes = ["vector", "utility", "cstdint"]
     ]
     '''
     expect_json_v2 = '''[
-      {"staticSize": 48, "exclusiveSize": 0, "members": [
-        {"typeNames": ["std::vector<uint64_t, std::allocator<uint64_t>>"], "staticSize": 24, "exclusiveSize": 24},
-        {"typeNames": ["std::vector<uint64_t, std::allocator<uint64_t>>"], "staticSize": 24, "exclusiveSize": 24}
+      {"staticSize": 48, "exclusiveSize": 0, "size": 104, "members": [
+        {"typeNames": ["std::vector<uint64_t, std::allocator<uint64_t>>"], "staticSize": 24, "exclusiveSize": 24, "size": 48},
+        {"typeNames": ["std::vector<uint64_t, std::allocator<uint64_t>>"], "staticSize": 24, "exclusiveSize": 24, "size": 56}
       ]}
     ]'''
diff --git a/test/integration/std_set_custom_comparator.toml b/test/integration/std_set_custom_comparator.toml
index 66031f7..49d07f8 100644
--- a/test/integration/std_set_custom_comparator.toml
+++ b/test/integration/std_set_custom_comparator.toml
@@ -65,9 +65,10 @@ includes = ["set", "functional"]
     expect_json_v2 = '''[{
       "staticSize":8184,
       "exclusiveSize": 0,
+      "size": 9144,
       "members":[
-        {"name":"m1", "staticSize":48, "exclusiveSize": 156, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":48, "exclusiveSize": 228, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":48, "exclusiveSize": 300, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":8040, "exclusiveSize": 8364, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":48, "exclusiveSize": 156, "size": 168, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":48, "exclusiveSize": 228, "size": 248, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":48, "exclusiveSize": 300, "size": 328, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":8040, "exclusiveSize": 8364, "size": 8400, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/std_smart_ptr.toml b/test/integration/std_smart_ptr.toml
index 8fc66bf..dfb86ae 100644
--- a/test/integration/std_smart_ptr.toml
+++ b/test/integration/std_smart_ptr.toml
@@ -26,6 +26,7 @@ definitions = '''
       {
         "staticSize": 8,
         "exclusiveSize": 8,
+        "size": 8,
         "length": 0,
         "capacity": 1
       }
@@ -50,6 +51,7 @@ definitions = '''
       {
         "staticSize": 8,
         "exclusiveSize": 8,
+        "size": 16,
         "length": 1,
         "capacity": 1
       }
@@ -74,6 +76,7 @@ definitions = '''
       {
         "staticSize": 8,
         "exclusiveSize": 8,
+        "size": 8,
         "length": 0,
         "capacity": 1
       }
@@ -106,6 +109,7 @@ definitions = '''
       {
         "staticSize": 8,
         "exclusiveSize": 8,
+        "size": 72,
         "length": 1,
         "capacity": 1,
         "members": [
@@ -134,7 +138,8 @@ definitions = '''
     [
       {
         "staticSize": 16,
-        "exclusiveSize": 16
+        "exclusiveSize": 16,
+        "size": 16
       }
     ]
     '''
@@ -153,7 +158,8 @@ definitions = '''
     [
       {
         "staticSize": 16,
-        "exclusiveSize": 16
+        "exclusiveSize": 16,
+        "size": 16
       }
     ]
     '''
@@ -176,6 +182,7 @@ definitions = '''
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 16,
         "length": 0,
         "capacity": 1
       }
@@ -200,6 +207,7 @@ definitions = '''
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 24,
         "length": 1,
         "capacity": 1,
         "members": [
@@ -227,6 +235,7 @@ definitions = '''
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 16,
         "length": 0,
         "capacity": 1
       }
@@ -257,12 +266,14 @@ definitions = '''
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 80,
         "length": 1,
         "capacity": 1,
         "members": [
           {
             "staticSize": 24,
             "exclusiveSize": 24,
+            "size": 64,
             "length": 5,
             "capacity": 5
           }
@@ -278,7 +289,8 @@ definitions = '''
       {
         "staticSize": 16,
         "dynamicSize": 0,
-        "exclusiveSize": 16
+        "exclusiveSize": 16,
+        "size": 16
       }
     ]
     '''
@@ -290,7 +302,8 @@ definitions = '''
       {
         "staticSize": 16,
         "dynamicSize": 0,
-        "exclusiveSize": 16
+        "exclusiveSize": 16,
+        "size": 16
       }
     ]
     '''
@@ -307,7 +320,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_void_empty]
     param_types = ["std::weak_ptr<void>&"]
     setup = "return std::weak_ptr<void>();"
@@ -321,7 +334,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_present]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
     setup = '''
@@ -339,7 +352,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_expired]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
     setup = '''
@@ -357,7 +370,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_present_chase]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
     cli_options = ["-fchase-raw-pointers"]
@@ -376,7 +389,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_expired_chase]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
     cli_options = ["-fchase-raw-pointers"]
@@ -393,4 +406,4 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
diff --git a/test/integration/std_unordered_map_custom_operator.toml b/test/integration/std_unordered_map_custom_operator.toml
index 3dcf55d..ea78bff 100644
--- a/test/integration/std_unordered_map_custom_operator.toml
+++ b/test/integration/std_unordered_map_custom_operator.toml
@@ -66,9 +66,11 @@ includes = ["unordered_map"]
       ]}]'''
     expect_json_v2 = '''[{
       "staticSize":480,
+      "exclusiveSize": 0,
+      "size": 1952,
       "members":[
-        {"name":"m1", "staticSize":56, "exclusiveSize":220, "length":3, "capacity":3},
-        {"name":"m2", "staticSize":120, "exclusiveSize":324, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":120, "exclusiveSize":364, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":184, "exclusiveSize":468, "length":9, "capacity":9}
+        {"name":"m1", "staticSize":56, "exclusiveSize":220, "size":292, "length":3, "capacity":3},
+        {"name":"m2", "staticSize":120, "exclusiveSize":324, "size":444,  "length":5, "capacity":5},
+        {"name":"m3", "staticSize":120, "exclusiveSize":364, "size":532,  "length":7, "capacity":7},
+        {"name":"m4", "staticSize":184, "exclusiveSize":468, "size":684,  "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_unordered_multimap_custom_operator.toml b/test/integration/std_unordered_multimap_custom_operator.toml
index 37fe213..f36a296 100644
--- a/test/integration/std_unordered_multimap_custom_operator.toml
+++ b/test/integration/std_unordered_multimap_custom_operator.toml
@@ -66,9 +66,11 @@ includes = ["unordered_map"]
       ]}]'''
     expect_json_v2 = '''[{
       "staticSize":480,
+      "exclusiveSize":0,
+      "size":1952,
       "members":[
-        {"name":"m1", "staticSize":56, "exclusiveSize":220, "length":3, "capacity":3},
-        {"name":"m2", "staticSize":120, "exclusiveSize":324, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":120, "exclusiveSize":364, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":184, "exclusiveSize":468, "length":9, "capacity":9}
+        {"name":"m1", "staticSize":56, "exclusiveSize":220, "size":292, "length":3, "capacity":3},
+        {"name":"m2", "staticSize":120, "exclusiveSize":324, "size":444,  "length":5, "capacity":5},
+        {"name":"m3", "staticSize":120, "exclusiveSize":364, "size":532,  "length":7, "capacity":7},
+        {"name":"m4", "staticSize":184, "exclusiveSize":468, "size":684,  "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_unordered_multiset_custom_operator.toml b/test/integration/std_unordered_multiset_custom_operator.toml
index ef08dcb..d9ce1cc 100644
--- a/test/integration/std_unordered_multiset_custom_operator.toml
+++ b/test/integration/std_unordered_multiset_custom_operator.toml
@@ -66,9 +66,11 @@ includes = ["unordered_set"]
       ]}]'''
     expect_json_v2 = '''[{
       "staticSize":480,
+      "exclusiveSize":0,
+      "size":1472,
       "members":[
-        {"name":"m1", "staticSize":56, "exclusiveSize":220, "length":3, "capacity":3},
-        {"name":"m2", "staticSize":120, "exclusiveSize":324, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":120, "exclusiveSize":364, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":184, "exclusiveSize":468, "length":9, "capacity":9}
+        {"name":"m1", "staticSize":56, "exclusiveSize":220, "size":232, "length":3, "capacity":3},
+        {"name":"m2", "staticSize":120, "exclusiveSize":324, "size":344, "length":5, "capacity":5},
+        {"name":"m3", "staticSize":120, "exclusiveSize":364, "size":392, "length":7, "capacity":7},
+        {"name":"m4", "staticSize":184, "exclusiveSize":468, "size":504, "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_unordered_set_custom_operator.toml b/test/integration/std_unordered_set_custom_operator.toml
index 3c01f6c..797c5ea 100644
--- a/test/integration/std_unordered_set_custom_operator.toml
+++ b/test/integration/std_unordered_set_custom_operator.toml
@@ -66,9 +66,11 @@ includes = ["unordered_set"]
       ]}]'''
     expect_json_v2 = '''[{
       "staticSize":480,
+      "exclusiveSize":0,
+      "size":1472,
       "members":[
-        {"name":"m1", "staticSize":56, "exclusiveSize":220, "length":3, "capacity":3},
-        {"name":"m2", "staticSize":120, "exclusiveSize":324, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":120, "exclusiveSize":364, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":184, "exclusiveSize":468, "length":9, "capacity":9}
+        {"name":"m1", "staticSize":56, "exclusiveSize":220, "size":232,"length":3, "capacity":3},
+        {"name":"m2", "staticSize":120, "exclusiveSize":324, "size":344, "length":5, "capacity":5},
+        {"name":"m3", "staticSize":120, "exclusiveSize":364, "size":392, "length":7, "capacity":7},
+        {"name":"m4", "staticSize":184, "exclusiveSize":468, "size":504, "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_vector.toml b/test/integration/std_vector.toml
index d466891..20e00d8 100644
--- a/test/integration/std_vector.toml
+++ b/test/integration/std_vector.toml
@@ -13,24 +13,24 @@ definitions = '''
     param_types = ["const std::vector<int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size": 24, "length":0, "capacity":0, "members":[]}]'
   [cases.int_some]
     param_types = ["const std::vector<int>&"]
     setup = "return {{1,2,3}};"
     expect_json = '[{"staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":36,"length":3, "capacity":3, "members":[
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4}
     ]}]'''
   [cases.struct_some]
     param_types = ["const std::vector<SimpleStruct>&"]
     setup = "return {{{}, {}, {}}};"
     expect_json = '[{"staticSize":24, "dynamicSize":48, "length":3, "capacity":3, "elementStaticSize":16}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
-      {"staticSize":16, "exclusiveSize":3},
-      {"staticSize":16, "exclusiveSize":3},
-      {"staticSize":16, "exclusiveSize":3}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":72, "length":3, "capacity":3, "members":[
+      {"staticSize":16, "exclusiveSize":3, "size":16},
+      {"staticSize":16, "exclusiveSize":3, "size":16},
+      {"staticSize":16, "exclusiveSize":3, "size":16}
     ]}]'''
   [cases.bool_empty]
     skip = true # https://github.com/facebookexperimental/object-introspection/issues/14
@@ -46,7 +46,7 @@ definitions = '''
     param_types = ["const std::vector<std::vector<int>>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":24}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size":24, "length":0, "capacity":0, "members":[]}]'
   [cases.vector_int_some]
     param_types = ["const std::vector<std::vector<int>>&"]
     setup = "return {{{1,2,3},{4},{5,6}}};"
@@ -62,10 +62,10 @@ definitions = '''
         {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4},
         {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
       ]}]'''
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[
-      {"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[]},
-      {"staticSize":24, "exclusiveSize":24, "length":1, "capacity": 1, "members":[]},
-      {"staticSize":24, "exclusiveSize":24, "length":2, "capacity": 2, "members":[]}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":120, "length":3, "capacity": 3, "members":[
+      {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity": 3, "members":[]},
+      {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity": 1, "members":[]},
+      {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity": 2, "members":[]}
     ]}]'''
   [cases.reserve]
     param_types = ["const std::vector<int>&"]
@@ -75,8 +75,8 @@ definitions = '''
       return ret;
     '''
     expect_json = '[{"staticSize":24, "dynamicSize":40, "exclusiveSize":64, "length":3, "capacity":10, "elementStaticSize":4}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":52, "length":3, "capacity":10, "members":[
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":52, "size":64, "length":3, "capacity":10, "members":[
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4}
     ]}]'''
diff --git a/test/integration/std_vector_del_allocator.toml b/test/integration/std_vector_del_allocator.toml
index 934bd34..32edcc2 100644
--- a/test/integration/std_vector_del_allocator.toml
+++ b/test/integration/std_vector_del_allocator.toml
@@ -60,18 +60,19 @@ includes = ["vector"]
     expect_json_v2 = '''[{
       "staticSize":48,
       "exclusiveSize":0,
+      "size":60,
       "members":[
-        {"name":"v1", "staticSize":24, "exclusiveSize":24, "length":1, "capacity":1, "members":[
-          {"name":"[]", "staticSize":4, "exclusiveSize":0, "members":[
-            {"name":"a", "staticSize":4, "exclusiveSize":4, "members":[]}
+        {"name":"v1", "staticSize":24, "exclusiveSize":24, "size": 28, "length":1, "capacity":1, "members":[
+          {"name":"[]", "staticSize":4, "exclusiveSize":0, "size": 4, "members":[
+            {"name":"a", "staticSize":4, "exclusiveSize":4, "size": 4, "members":[]}
           ]}
         ]},
-        {"name":"v2", "staticSize":24, "exclusiveSize":24, "length":2, "capacity":2, "members":[
-          {"name":"[]", "staticSize":4, "exclusiveSize":0, "members":[
-            {"name":"b", "staticSize":4, "exclusiveSize":4, "members":[]}
+        {"name":"v2", "staticSize":24, "exclusiveSize":24, "size": 32, "length":2, "capacity":2, "members":[
+          {"name":"[]", "staticSize":4, "exclusiveSize":0, "size": 4, "members":[
+            {"name":"b", "staticSize":4, "exclusiveSize":4, "size": 4, "members":[]}
           ]},
-          {"name":"[]", "staticSize":4, "exclusiveSize":0, "members":[
-            {"name":"b", "staticSize":4, "exclusiveSize":4, "members":[]}
+          {"name":"[]", "staticSize":4, "exclusiveSize":0, "size": 4, "members":[
+            {"name":"b", "staticSize":4, "exclusiveSize":4, "size": 4, "members":[]}
           ]}
         ]}
       ]
diff --git a/test/integration/templates.toml b/test/integration/templates.toml
index b02cfc8..8be4dc9 100644
--- a/test/integration/templates.toml
+++ b/test/integration/templates.toml
@@ -53,10 +53,12 @@ definitions = '''
       "typeName":"ns_templates::TemplatedClass1<std::vector<int, std::allocator<int> > >",
       "staticSize":24,
       "exclusiveSize":0,
+      "size":24,
       "members":[{
         "typeName":"std::vector<int32_t, std::allocator<int32_t>>",
         "staticSize":24,
         "exclusiveSize":24,
+        "size":24,
         "length":0,
         "capacity":0
     }]}]'''
@@ -90,9 +92,11 @@ definitions = '''
       "typeName":"ns_templates::TemplatedClassVal<3>",
       "staticSize":12,
       "exclusiveSize":0,
+      "size":12,
       "members":[{
         "staticSize":12,
         "exclusiveSize":0,
+        "size":12,
         "length":3,
         "capacity":3
     }]}]'''
diff --git a/test/integration/thrift_unions.toml b/test/integration/thrift_unions.toml
index 5f0ba1f..651869b 100644
--- a/test/integration/thrift_unions.toml
+++ b/test/integration/thrift_unions.toml
@@ -37,9 +37,10 @@ namespace cpp2 {
     expect_json_v2 = '''[{
       "staticSize":8,
       "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"typeNames":["storage_type"], "name":"value_", "staticSize":4, "exclusiveSize":4},
-        {"typeNames":["underlying_type_t<cpp2::StaticUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4}
+        {"typeNames":["storage_type"], "name":"value_", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"typeNames":["underlying_type_t<cpp2::StaticUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}]'''
   [cases.dynamic_int]
     param_types = ["const cpp2::DynamicUnion&"]
@@ -58,9 +59,10 @@ namespace cpp2 {
     expect_json_v2 = '''[{
       "staticSize":32,
       "exclusiveSize":4,
+      "size":32,
       "members":[
-        {"typeNames":["storage_type"], "name":"value_", "staticSize":24, "exclusiveSize":24},
-        {"typeNames":["underlying_type_t<cpp2::DynamicUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4}
+        {"typeNames":["storage_type"], "name":"value_", "staticSize":24, "exclusiveSize":24, "size":24},
+        {"typeNames":["underlying_type_t<cpp2::DynamicUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}]'''
   [cases.dynamic_vec]
     param_types = ["const cpp2::DynamicUnion&"]
@@ -79,7 +81,8 @@ namespace cpp2 {
     expect_json_v2 = '''[{
       "staticSize":32,
       "exclusiveSize":4,
+      "size":32,
       "members":[
-        {"typeNames":["storage_type"], "name":"value_", "staticSize":24, "exclusiveSize":24},
-        {"typeNames":["underlying_type_t<cpp2::DynamicUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4}
+        {"typeNames":["storage_type"], "name":"value_", "staticSize":24, "exclusiveSize":24, "size":24},
+        {"typeNames":["underlying_type_t<cpp2::DynamicUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}]'''
diff --git a/test/integration/typedefed_parent.toml b/test/integration/typedefed_parent.toml
index 72c4a57..25631f9 100644
--- a/test/integration/typedefed_parent.toml
+++ b/test/integration/typedefed_parent.toml
@@ -30,9 +30,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":48,
       "exclusiveSize":0,
+      "size":80,
       "members":[
-        {"name":"a", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3},
-        {"name":"b", "staticSize":24, "exclusiveSize":24, "length":5, "capacity":5}
+        {"name":"a", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "size":36},
+        {"name":"b", "staticSize":24, "exclusiveSize":24, "length":5, "capacity":5, "size":44}
       ]}]'''
   [cases.multilevel_typedef_parent]
     param_types = ["const Bar_2&"]
@@ -50,7 +51,8 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":48,
       "exclusiveSize":0,
+      "size":80,
       "members":[
-        {"name":"a", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3},
-        {"name":"c", "staticSize":24, "exclusiveSize":24, "length":5, "capacity":5}
+        {"name":"a", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "size":36},
+        {"name":"c", "staticSize":24, "exclusiveSize":24, "length":5, "capacity":5, "size":44}
       ]}]'''
diff --git a/test/integration/unions.toml b/test/integration/unions.toml
index 42341de..66df45e 100644
--- a/test/integration/unions.toml
+++ b/test/integration/unions.toml
@@ -45,17 +45,17 @@ definitions = '''
     param_types = ["const MyUnion&"]
     setup = "return 123;"
     expect_json = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}]'
-    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]}]'
   [cases.vector]
     param_types = ["const MyUnion&"]
     setup = "return std::vector{1,2,3};"
     expect_json = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}]'
-    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]}]'
   [cases.unordered_map]
     param_types = ["const MyUnion&"]
     setup = 'return std::unordered_map<std::string, std::string>{{"a", "b"}, {"c","d"}};'
     expect_json = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}]'
-    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]}]'
 
   [cases.alignment]
     # Wrap the union in a pair as a way of inferring its alignment
@@ -67,9 +67,9 @@ definitions = '''
         {"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}
       ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[
-        {"staticSize":1, "dynamicSize":0, "exclusiveSize":1},
-        {"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}
+      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "size":64, "members":[
+        {"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1},
+        {"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]}
       ]}]'''
 
   [cases.tagged_int]
@@ -81,9 +81,9 @@ definitions = '''
         {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "NOT":"members"}
       ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[
-        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]},
-        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "members":[]}
+      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "size":64, "members":[
+        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]},
+        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1, "members":[]}
       ]}]'''
   [cases.tagged_vector]
     param_types = ["const TaggedUnion&"]
@@ -94,9 +94,9 @@ definitions = '''
         {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "NOT":"members"}
       ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[
-        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]},
-        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "members":[]}
+      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "size":64, "members":[
+        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]},
+        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1, "members":[]}
       ]}]'''
   [cases.tagged_unordered_map]
     param_types = ["const TaggedUnion&"]
@@ -107,7 +107,7 @@ definitions = '''
         {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "NOT":"members"}
       ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[
-        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]},
-        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "members":[]}
+      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "size":64, "members":[
+        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]},
+        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1, "members":[]}
       ]}]'''

From f6024e5cf9150f2bb85d1854d2eeb9f57aabaf3d Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 2 Jan 2024 15:51:55 +0000
Subject: [PATCH 101/188] circleci: clean up codegen v1 runs

Remove the CodeGen v1 sections of the CI config because both OID and OIL use
CodeGen v2.

Test plan:
- CI
---
 .circleci/config.yml       | 22 +---------------------
 test/test_add_children.cpp |  2 +-
 2 files changed, 2 insertions(+), 22 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 0a75d99..b91cc2c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -10,47 +10,27 @@ workflows:
           cc: /usr/bin/gcc
           cxx: /usr/bin/g++
           warnings_as_errors: "OFF"
-      - test:
-          name: test-codegenv1-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-Ftype-graph"
-          tests_regex: "OilIntegration\\..*"
       - test:
           name: test-gcc
           requires:
             - build-gcc
-          tests_regex: "OidIntegration\\..*"
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
       - coverage:
           name: coverage
           requires:
             - test-gcc
-      - coverage:
-          name: coverage-codegenv1
-          requires:
-            - test-codegenv1-gcc
 
       - build:
           name: build-clang
           cc: /usr/bin/clang-12
           cxx: /usr/bin/clang++-12
           warnings_as_errors: "ON"
-      - test:
-          name: test-codegenv1-clang
-          requires:
-            - build-clang
-          oid_test_args: "-Ftype-graph"
-          tests_regex: "OilIntegration\\..*"
-          # Tests disabled due to bad DWARF generated by the old clang compiler in CI
-          exclude_regex: "OilIntegration.fbstring_.*|OilIntegration.capture_keys_string|OilIntegration.capture_keys_multi_level"
       - test:
           name: test-clang
           requires:
             - build-clang
-          tests_regex: "OidIntegration\\..*"
           # Tests disabled due to bad DWARF generated by the old clang compiler in CI
-          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_a"
+          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_a|OilIntegration.fbstring_.*|OilIntegration.capture_keys_string|OilIntegration.capture_keys_multi_level"
 
 executors:
   nix-docker:
diff --git a/test/test_add_children.cpp b/test/test_add_children.cpp
index 0de8c5b..bf9736c 100644
--- a/test/test_add_children.cpp
+++ b/test/test_add_children.cpp
@@ -139,7 +139,7 @@ TEST_F(AddChildrenTest, InheritancePolymorphic) {
              Parent (offset: 0)
                [0]
              Member: vec_b (offset: 16)
-[4]            Class: vector<int, std::allocator<int> > [std::vector<<int, std::allocator<int> >] (size: 24)
+[4]            Class: vector<int, std::allocator<int> > [std::vector<int, std::allocator<int> >] (size: 24)
                  Param
                    Primitive: int32_t
                  Param

From 4650e63d7ac87c3779bd8cf21e264463d3161364 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 3 Jan 2024 15:26:45 +0000
Subject: [PATCH 102/188] circleci: clean up codegen v1 runs

Remove the CodeGen v1 sections of the CI config because both OID and OIL use
CodeGen v2.

We were missing running any test that wasn't `Oi{d,l}Integration.*` before.
This now runs the unit tests again and requires a minor fix to one unit test.

Test plan:
- CI
---
 .circleci/config.yml       | 22 +---------------------
 test/test_add_children.cpp |  2 +-
 2 files changed, 2 insertions(+), 22 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 0a75d99..b91cc2c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -10,47 +10,27 @@ workflows:
           cc: /usr/bin/gcc
           cxx: /usr/bin/g++
           warnings_as_errors: "OFF"
-      - test:
-          name: test-codegenv1-gcc
-          requires:
-            - build-gcc
-          oid_test_args: "-Ftype-graph"
-          tests_regex: "OilIntegration\\..*"
       - test:
           name: test-gcc
           requires:
             - build-gcc
-          tests_regex: "OidIntegration\\..*"
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
       - coverage:
           name: coverage
           requires:
             - test-gcc
-      - coverage:
-          name: coverage-codegenv1
-          requires:
-            - test-codegenv1-gcc
 
       - build:
           name: build-clang
           cc: /usr/bin/clang-12
           cxx: /usr/bin/clang++-12
           warnings_as_errors: "ON"
-      - test:
-          name: test-codegenv1-clang
-          requires:
-            - build-clang
-          oid_test_args: "-Ftype-graph"
-          tests_regex: "OilIntegration\\..*"
-          # Tests disabled due to bad DWARF generated by the old clang compiler in CI
-          exclude_regex: "OilIntegration.fbstring_.*|OilIntegration.capture_keys_string|OilIntegration.capture_keys_multi_level"
       - test:
           name: test-clang
           requires:
             - build-clang
-          tests_regex: "OidIntegration\\..*"
           # Tests disabled due to bad DWARF generated by the old clang compiler in CI
-          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_a"
+          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_a|OilIntegration.fbstring_.*|OilIntegration.capture_keys_string|OilIntegration.capture_keys_multi_level"
 
 executors:
   nix-docker:
diff --git a/test/test_add_children.cpp b/test/test_add_children.cpp
index 0de8c5b..bf9736c 100644
--- a/test/test_add_children.cpp
+++ b/test/test_add_children.cpp
@@ -139,7 +139,7 @@ TEST_F(AddChildrenTest, InheritancePolymorphic) {
              Parent (offset: 0)
                [0]
              Member: vec_b (offset: 16)
-[4]            Class: vector<int, std::allocator<int> > [std::vector<<int, std::allocator<int> >] (size: 24)
+[4]            Class: vector<int, std::allocator<int> > [std::vector<int, std::allocator<int> >] (size: 24)
                  Param
                    Primitive: int32_t
                  Param

From 72536b39241eac46b1e6e7ae6a538cc2329a96fa Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 3 Jan 2024 17:02:44 +0000
Subject: [PATCH 103/188] oilgen: add to integration test framework

TODO: Replace the references to local paths.

oilgen (the basis of Ahead Of Time compilation for OIL) has never been passed
through our large test suite and has instead had more focused testing and large
examples. When consuming DWARF information in a similar fashion to JIT OIL this
was okay, but with the new Clang AST based mechanism it means we have very
little coverage.

This change adds an oilgen test for every test case that has an oil test.
Relying on the build system to create the test target as before would make it
difficult to have failing tests, so we move the build into the integration test
runner. This involves:
1. Writing the input source to a file.
2. Consuming it with oilgen to get the implementation object file.
3. Compiling the input source and linking it with this file.
4. Running the newly created target.

This approach can give the full error message at any stage that fails and will
fail the test appropriately. The downside is the build system integration is
more difficult, as we need the correct compiler flags for the target and to use
the correct compiler. It would be very tricky to replicate this in a build
system that's not CMake, so we will likely only run these tests in open source.

Test plan:
- CI
---
 .circleci/config.yml               |   4 +-
 oi/type_graph/ClangTypeParser.cpp  |   4 +-
 oi/type_graph/Types.h              |   2 +-
 test/integration/CMakeLists.txt    |  11 +-
 test/integration/gen_tests.py      | 153 +++++++++++++++++++++------
 test/integration/runner_common.cpp | 163 +++++++++++++++++++++++++++++
 test/integration/runner_common.h   |  14 +++
 7 files changed, 313 insertions(+), 38 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 0a75d99..2fbef4c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -20,7 +20,7 @@ workflows:
           name: test-gcc
           requires:
             - build-gcc
-          tests_regex: "OidIntegration\\..*"
+          tests_regex: "OidIntegration\\..*|OilgenIntegration\\..*"
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
       - coverage:
           name: coverage
@@ -48,7 +48,7 @@ workflows:
           name: test-clang
           requires:
             - build-clang
-          tests_regex: "OidIntegration\\..*"
+          tests_regex: "OidIntegration\\..*|OilgenIntegration\\..*"
           # Tests disabled due to bad DWARF generated by the old clang compiler in CI
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_a"
 
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index 24d5835..dfa3a07 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -350,6 +350,7 @@ Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
     case clang::BuiltinType::WChar_U:
       return makeType<Primitive>(ty, Primitive::Kind::UInt32);
 
+    case clang::BuiltinType::Char8:
     case clang::BuiltinType::Char_S:
     case clang::BuiltinType::SChar:
       return makeType<Primitive>(ty, Primitive::Kind::Int8);
@@ -380,8 +381,9 @@ Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
     case clang::BuiltinType::Float:
       return makeType<Primitive>(ty, Primitive::Kind::Float32);
     case clang::BuiltinType::Double:
-    case clang::BuiltinType::LongDouble:
       return makeType<Primitive>(ty, Primitive::Kind::Float64);
+    case clang::BuiltinType::LongDouble:
+      return makeType<Primitive>(ty, Primitive::Kind::Float128);
 
     case clang::BuiltinType::UInt128:
     case clang::BuiltinType::Int128:
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index 1d5dd79..81cab11 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -592,7 +592,7 @@ class Primitive : public Type {
     Float32,
     Float64,
     Float80,   // TODO worth including?
-    Float128,  // TODO can we generate this?
+    Float128,
     Bool,
 
     StubbedPointer,
diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 4d235b5..b4fe3de 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -58,12 +58,18 @@ target_link_libraries(integration_test_runner PRIVATE
   GTest::gmock_main
   Boost::headers
   ${Boost_LIBRARIES}
+  range-v3
   toml
 )
 target_compile_definitions(integration_test_runner PRIVATE
-  TARGET_EXE_PATH="${CMAKE_CURRENT_BINARY_DIR}/integration_test_target"
+  TARGET_EXE_PATH="$<TARGET_FILE:integration_test_target>"
   OID_EXE_PATH="$<TARGET_FILE:oid>"
-  CONFIG_FILE_PATH="${CMAKE_BINARY_DIR}/testing.oid.toml")
+  OILGEN_EXE_PATH="$<TARGET_FILE:oilgen>"
+  CONFIG_FILE_PATH="${CMAKE_BINARY_DIR}/testing.oid.toml"
+
+  CXX="${CMAKE_CXX_COMPILER}"
+  TARGET_INCLUDE_DIRECTORIES="$<JOIN:$<TARGET_PROPERTY:integration_test_target,INCLUDE_DIRECTORIES>,:>"
+)
 
 if (${THRIFT_FOUND})
   foreach(THRIFT_TEST IN LISTS THRIFT_TESTS)
@@ -85,6 +91,7 @@ if (${THRIFT_FOUND})
 
     add_custom_target(integration_test_thrift_sources_${THRIFT_TEST} DEPENDS ${THRIFT_TYPES_H})
     add_dependencies(integration_test_target integration_test_thrift_sources_${THRIFT_TEST})
+    add_dependencies(integration_test_runner integration_test_thrift_sources_${THRIFT_TEST})
     target_sources(integration_test_target PRIVATE ${THRIFT_DATA_CPP})
   endforeach()
 
diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py
index b72e371..d9fe054 100644
--- a/test/integration/gen_tests.py
+++ b/test/integration/gen_tests.py
@@ -51,6 +51,41 @@ def add_headers(f, custom_headers, thrift_headers):
         f.write(f'#include "{header}"\n')
 
 
+def add_test_getters(f, case_name, case):
+    param_types = ", ".join(
+        f"std::remove_cvref_t<{param}>" for param in case["param_types"]
+    )
+    if "arg_types" in case:
+        arg_types = ", ".join(case["arg_types"])
+    else:
+        arg_types = param_types
+
+    f.write(
+        f"\n"
+        f"  std::tuple<{arg_types}> get_{case_name}() {{\n"
+        f'{case["setup"]}\n'
+        f"  }}\n"
+    )
+
+
+def get_param_str(param, i):
+    if "]" in param:
+        # Array param
+
+        if ")" in param:
+            # "int(&)[5]" ->  "int (&a0)[5]"
+            start, end = param.split(")")
+            return f"{start}a{i}){end}"
+
+        # "int[5]" -> "int a0[5]"
+        # "int[5][10]" -> "int a0[5][10]"
+        type_name, array_size = param.split("[", 1)
+        return f"{type_name} a{i}[{array_size}"
+
+    # Non-array param, e.g. "int&" -> "int& a0"
+    return f"{param} a{i}"
+
+
 def add_test_setup(f, config):
     ns = get_namespace(config["suite"])
     # fmt: off
@@ -65,23 +100,6 @@ def add_test_setup(f, config):
     )
     # fmt: on
 
-    def get_param_str(param, i):
-        if "]" in param:
-            # Array param
-
-            if ")" in param:
-                # "int(&)[5]" ->  "int (&a0)[5]"
-                start, end = param.split(")")
-                return f"{start}a{i}){end}"
-
-            # "int[5]" -> "int a0[5]"
-            # "int[5][10]" -> "int a0[5][10]"
-            type_name, array_size = param.split("[", 1)
-            return f"{type_name} a{i}[{array_size}"
-
-        # Non-array param, e.g. "int&" -> "int& a0"
-        return f"{param} a{i}"
-
     def define_traceable_func(name, params, body):
         return (
             f"\n"
@@ -99,21 +117,7 @@ def add_test_setup(f, config):
             # target func for it
             continue
 
-        # generate getter for an object of this type
-        param_types = ", ".join(
-            f"std::remove_cvref_t<{param}>" for param in case["param_types"]
-        )
-        if "arg_types" in case:
-            arg_types = ", ".join(case["arg_types"])
-        else:
-            arg_types = param_types
-
-        f.write(
-            f"\n"
-            f"  std::tuple<{arg_types}> get_{case_name}() {{\n"
-            f'{case["setup"]}\n'
-            f"  }}\n"
-        )
+        add_test_getters(f, case_name, case)
 
         # generate oid and oil targets
         params_str = ", ".join(
@@ -266,6 +270,7 @@ def add_tests(f, config):
     for case_name, case in config["cases"].items():
         add_oid_integration_test(f, config, case_name, case)
         add_oil_integration_test(f, config, case_name, case)
+        add_oilgen_integration_test(f, config, case_name, case)
 
 
 def add_oid_integration_test(f, config, case_name, case):
@@ -400,6 +405,90 @@ def add_oil_integration_test(f, config, case_name, case):
     f.write(f"}}\n")
 
 
+def add_oilgen_integration_test(f, config, case_name, case):
+    case_str = get_case_name(config["suite"], case_name)
+    exit_code = case.get("expect_oil_exit_code", 0)
+
+    if "oil_disable" in case or "target_function" in case:
+        return
+
+    config_prefix = case.get("config_prefix", "")
+    config_suffix = case.get("config_suffix", "")
+
+    f.write(
+        f"\n"
+        f"TEST_F(OilgenIntegration, {case_str}) {{\n"
+        f"{generate_skip(case, 'oil')}"
+    )
+
+    f.write('  constexpr std::string_view targetSrc = R"--(')
+    headers = set(config.get("includes", []))
+    thrift_headers = [f"thrift/annotation/gen-cpp2/{config['suite']}_types.h"] if is_thrift_test(config) else []
+    add_headers(f, sorted(headers), thrift_headers)
+
+    f.write(
+        f"\n"
+        f'{config.get("raw_definitions", "")}\n'
+        f"#pragma clang diagnostic push\n"
+        f'#pragma clang diagnostic ignored "-Wunused-private-field"\n'
+        f'{config.get("definitions", "")}\n'
+        f"#pragma clang diagnostic pop\n"
+    )
+    add_test_getters(f, case_name, case)
+
+    main = "int main() {\n"
+    main += "  auto pr = oi::exporters::Json(std::cout);\n"
+    main += "  pr.setPretty(true);\n"
+    main += f"  auto val = get_{case_name}();\n"
+    for i in range(len(case["param_types"])):
+        main += f"  auto ret{i} = oi::result::SizedResult(oi::introspect"
+        if "arg_types" in case:
+            main += f"<std::decay_t<{case['param_types'][i]}>>"
+        main += f"(std::get<{i}>(val)));\n"
+        main += f"  pr.print(ret{i});\n"
+    main += "}\n"
+
+    f.write(main)
+    f.write(')--";\n')
+
+    f.write(
+        f'  std::string configPrefix = R"--({config_prefix})--";\n'
+        f'  std::string configSuffix = R"--({config_suffix})--";\n'
+        f"  ba::io_context ctx;\n"
+        f"  auto target = runOilgenTarget({{\n"
+        f"    .ctx = ctx,\n"
+        f"    .targetSrc = targetSrc,\n"
+        f"  }}, std::move(configPrefix), std::move(configSuffix));\n\n"
+        f"  ASSERT_EQ(exit_code(target), {exit_code});\n"
+    )
+
+    key = "expect_json"
+    if "expect_json_v2" in case:
+        key = "expect_json_v2"
+    if key in case:
+        try:
+            json.loads(case[key])
+        except json.decoder.JSONDecodeError as error:
+            print(
+                f"\x1b[31m`expect_json` value for test case {config['suite']}.{case_name} was invalid JSON: {error}\x1b[0m",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+        f.write(
+            f"\n"
+            f"  std::stringstream expected_json_ss;\n"
+            f'  expected_json_ss << R"--({case[key]})--";\n'
+            f"  auto result_json_ss = std::stringstream(stdout_);\n"
+            f"  bpt::ptree expected_json, actual_json;\n"
+            f"  bpt::read_json(expected_json_ss, expected_json);\n"
+            f"  bpt::read_json(result_json_ss, actual_json);\n"
+            f"  compare_json(expected_json, actual_json);\n"
+        )
+
+    f.write(f"}}\n")
+
+
 def generate_skip(case, specific):
     possibly_skip = ""
     skip_reason = case.get("skip", False)
diff --git a/test/integration/runner_common.cpp b/test/integration/runner_common.cpp
index 3c8525a..c843c5d 100644
--- a/test/integration/runner_common.cpp
+++ b/test/integration/runner_common.cpp
@@ -11,6 +11,7 @@
 #include <iostream>
 #include <string>
 #include <utility>
+#include <range/v3/view/split.hpp>
 
 #include "oi/OIOpts.h"
 #include "oi/support/Toml.h"
@@ -447,3 +448,165 @@ Proc OilIntegration::runOilTarget(OilOpts opts,
               std::move(std_out),
               std::move(std_err)};
 }
+
+std::string OilgenIntegration::TmpDirStr() {
+  return std::string("/tmp/oilgen-integration-XXXXXX");
+}
+
+Proc OilgenIntegration::runOilgenTarget(OilgenOpts opts,
+                                        std::string configPrefix,
+                                        std::string configSuffix) {
+  // Run an oilgen test in three stages.
+  // 1. Generate the OIL implementation .o from the input source.
+  // 2. Compile the input source and link to the implementation.
+  // 3. Run the now complete target.
+
+  auto srcPath = workingDir / "input_src.cpp";
+  {
+    std::ofstream file{srcPath, std::ios_base::app};
+    file << opts.targetSrc;
+  }
+
+  std::string generatorExe =
+      std::string(OILGEN_EXE_PATH) + " " + srcPath.native() + " --output=oil_generated.o --debug-level=3 ";
+  if (auto prefix = writeCustomConfig("prefix", configPrefix)) {
+    generatorExe += "--config-file ";
+    generatorExe += *prefix;
+    generatorExe += " ";
+  }
+  generatorExe += "--config-file ";
+  generatorExe += configFile;
+  if (auto suffix = writeCustomConfig("suffix", configSuffix)) {
+    generatorExe += " ";
+    generatorExe += "--config-file ";
+    generatorExe += *suffix;
+  }
+
+
+  // TODO: get this from the CMake arguments for integration_test_target.cpp
+  // somehow
+  // clang-format off
+  std::vector<std::string> clangArgs{
+      "-DOIL_AOT_COMPILATION=1",
+      "--std=c++20",
+      "-resource-dir",
+      "/usr/lib64/clang/15.0.7",
+      "-fpic",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../include/c++/12",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../include/c++/12/x86_64-redhat-linux",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../include/c++/12/backward",
+      "-isystem",
+      "/usr/lib64/clang/15.0.7/include",
+      "-isystem",
+      "/usr/local/include",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../x86_64-redhat-linux/include",
+      "-isystem",
+      "/include",
+      "-isystem",
+      "/usr/include",
+      "-isystem",
+      "/data/users/jakehillion/object-introspection-sl/build/_deps/folly-src",
+      "-fdebug-compilation-dir=/data/users/jakehillion/Downloads/libclang-oilgen-testing",
+      "-fgnuc-version=4.2.1",
+      "-fcxx-exceptions",
+      "-fexceptions",
+      "-faddrsig",
+      "-D__GCC_HAVE_DWARF2_CFI_ASM=1",
+  };
+  // clang-format on
+  for(auto&& rng : TARGET_INCLUDE_DIRECTORIES | ranges::views::split(':')) {
+    clangArgs.emplace_back("-I");
+    clangArgs.emplace_back(&*rng.begin(), ranges::distance(rng));
+  }
+
+  for (const auto& arg : clangArgs) {
+    generatorExe += " --extra-arg=";
+    generatorExe += arg;
+  }
+
+  if (verbose) {
+    std::cerr << "Running: " << generatorExe << std::endl;
+  }
+
+  bp::child generatorProc{generatorExe, opts.ctx};
+  generatorProc.wait();
+  if (generatorProc.exit_code() != 0)
+    throw std::runtime_error("generation failed!");
+
+  std::string compilerExe =
+      std::string(CXX) +
+      " input_src.cpp oil_generated.o -o generated_target "
+      "/data/users/jakehillion/object-introspection-sl/oi/IntrospectionResult.cpp "
+      "/data/users/jakehillion/object-introspection-sl/oi/exporters/ParsedData.cpp ";
+  for (const auto& arg : clangArgs) {
+    compilerExe += ' ';
+    compilerExe += arg;
+  }
+
+  if (verbose) {
+    std::cerr << "Running: " << compilerExe << std::endl;
+  }
+
+  bp::child compilerProc{compilerExe, opts.ctx};
+  compilerProc.wait();
+  if (compilerProc.exit_code() != 0)
+    throw std::runtime_error("compilation failed");
+
+  std::string targetExe = "./generated_target";
+  if (verbose) {
+    std::cerr << "Running: " << targetExe << std::endl;
+  }
+
+  // Use tee to write the output to files. If verbose is on, also redirect the
+  // output to stderr.
+  bp::async_pipe std_out_pipe(opts.ctx), std_err_pipe(opts.ctx);
+  bp::child std_out, std_err;
+  if (verbose) {
+    // clang-format off
+    std_out = bp::child(bp::search_path("tee"),
+                        (workingDir / "stdout").string(),
+                        bp::std_in < std_out_pipe,
+                        bp::std_out > stderr,
+                        opts.ctx);
+    std_err = bp::child(bp::search_path("tee"),
+                        (workingDir / "stderr").string(),
+                        bp::std_in < std_err_pipe,
+                        bp::std_out > stderr,
+                        opts.ctx);
+    // clang-format on
+  } else {
+    // clang-format off
+    std_out = bp::child(bp::search_path("tee"),
+                        (workingDir / "stdout").string(),
+                        bp::std_in < std_out_pipe,
+                        bp::std_out > bp::null,
+                        opts.ctx);
+    std_err = bp::child(bp::search_path("tee"),
+                        (workingDir / "stderr").string(),
+                        bp::std_in < std_err_pipe,
+                        bp::std_out > bp::null,
+                        opts.ctx);
+    // clang-format on
+  }
+
+  /* Spawn `oid` with tracing on and IOs redirected */
+  // clang-format off
+  bp::child targetProc(
+      targetExe,
+      bp::std_in  < bp::null,
+      bp::std_out > std_out_pipe,
+      bp::std_err > std_err_pipe,
+      opts.ctx);
+  // clang-format on
+
+  return Proc{
+      opts.ctx,
+      std::move(targetProc),
+      std::move(std_out),
+      std::move(std_err),
+  };
+}
diff --git a/test/integration/runner_common.h b/test/integration/runner_common.h
index d2b5e57..8c6c8eb 100644
--- a/test/integration/runner_common.h
+++ b/test/integration/runner_common.h
@@ -22,6 +22,11 @@ struct OilOpts {
   std::string targetArgs;
 };
 
+struct OilgenOpts {
+  boost::asio::io_context& ctx;
+  std::string_view targetSrc;
+};
+
 struct Proc {
   boost::asio::io_context& ctx;
   boost::process::child proc;
@@ -81,3 +86,12 @@ class OilIntegration : public IntegrationBase {
                     std::string configPrefix,
                     std::string configSuffix);
 };
+
+class OilgenIntegration : public IntegrationBase {
+ protected:
+  std::string TmpDirStr() override;
+
+  Proc runOilgenTarget(OilgenOpts opts,
+                       std::string configPrefix,
+                       std::string configSuffix);
+};

From 87140bab01018ef46bdbeac6e1d13cf07e2cb789 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 3 Jan 2024 17:02:44 +0000
Subject: [PATCH 104/188] clangparser: add support for parents

TODO: Check the assumption that a "base" always has a Type that can be cast to
RecordType.
TODO: Check the assumption that a "base" always has a Decl that can be cast to
CXXRecordDecl.

Add basic support for class parents. Focus purely on bases for now and ignore
vbases.

Test Plan:
- Tested with a simple example. Base containing a long and a class containing a
  float. Both fields appear in the final flattened code.
---
 oi/type_graph/ClangTypeParser.cpp | 32 ++++++++++++++++++++++++++++++-
 oi/type_graph/ClangTypeParser.h   |  2 ++
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index dfa3a07..6b36104 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -17,8 +17,10 @@
 
 #include <clang/AST/ASTContext.h>
 #include <clang/AST/Decl.h>
+#include <clang/AST/DeclCXX.h>
 #include <clang/AST/DeclTemplate.h>
 #include <clang/AST/QualTypeNames.h>
+#include <clang/AST/RecordLayout.h>
 #include <clang/AST/Type.h>
 #include <clang/Basic/DiagnosticSema.h>
 #include <clang/Sema/Sema.h>
@@ -196,7 +198,7 @@ Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
       ty, kind, std::move(name), std::move(fqName), size, virtuality);
 
   enumerateClassTemplateParams(ty, c.templateParams);
-  // enumerateClassParents(type, c.parents);
+  enumerateClassParents(ty, c.parents);
   enumerateClassMembers(ty, c.members);
   // enumerateClassFunctions(type, c.functions);
 
@@ -275,6 +277,34 @@ std::optional<TemplateParam> ClangTypeParser::enumerateTemplateTemplateParam(
   }
 }
 
+void ClangTypeParser::enumerateClassParents(const clang::RecordType& ty,
+                                            std::vector<Parent>& parents) {
+  assert(parents.empty());
+
+  auto* decl = ty.getDecl();
+  auto* cxxDecl = llvm::dyn_cast<clang::CXXRecordDecl>(decl);
+  if (cxxDecl == nullptr)
+    return;
+
+  const auto& layout = decl->getASTContext().getASTRecordLayout(decl);
+  for (const auto& base : cxxDecl->bases()) {
+    auto baseType = base.getType().getDesugaredType(decl->getASTContext());
+    const auto* baseRecordType = llvm::dyn_cast<clang::RecordType>(&*baseType);
+    if (baseRecordType == nullptr)
+      continue;
+
+    auto* baseDecl = baseRecordType->getDecl();
+    auto* baseCxxDecl = llvm::dyn_cast<clang::CXXRecordDecl>(baseDecl);
+    if (baseCxxDecl == nullptr)
+      continue;
+
+    auto offset = layout.getBaseClassOffset(baseCxxDecl).getQuantity();
+    auto& ptype = enumerateType(*baseType);
+    Parent p{ptype, static_cast<uint64_t>(offset * 8)};
+    parents.push_back(p);
+  }
+}
+
 void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
                                             std::vector<Member>& members) {
   assert(members.empty());
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
index 9b27d64..98f0f67 100644
--- a/oi/type_graph/ClangTypeParser.h
+++ b/oi/type_graph/ClangTypeParser.h
@@ -50,6 +50,7 @@ class Array;
 class Class;
 class Enum;
 class Member;
+struct Parent;
 class Primitive;
 class Reference;
 class Type;
@@ -114,6 +115,7 @@ class ClangTypeParser {
   std::optional<TemplateParam> enumerateTemplateTemplateParam(
       const clang::TemplateName&);
 
+  void enumerateClassParents(const clang::RecordType&, std::vector<Parent>&);
   void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
 
   ContainerInfo* getContainerInfo(const std::string& fqName) const;

From 2b022fc6378a832ba3d3176915e3fc6fe18bcd61 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 3 Jan 2024 17:30:31 +0000
Subject: [PATCH 105/188] tbv2: calculate total memory footprint

Add the option to calculate total size (inclusive size) by wrapping the
existing iterator. This change provides a new iterator, `SizedIterator`, which
wraps an existing iterator and adds a new field `size` to the output element.

This is achieved with a two pass algorithm on the existing iterator:
1. Gather metadata for each element. This includes the total size up until that
   element and the range of elements that should be included in the size.
2. Return the result from the underlying iterator with the additional
   field.

This algorithm is `O(N)` time on the number of elements in the iterator and
`O(N)` time, storing 16 bytes per element. This isn't super expensive but is a
lot more than the current algorithm which requires close to constant space.
Because of this I've implemented it as a wrapper on the iterator rather than on
by default, though it is now on in every one of our integration test cases.

Test plan:
- Added to the integration tests for full coverage.
---
 include/oi/exporters/Json.h                   | 162 +++++++++++++++++-
 include/oi/result/SizedResult-inl.h           | 144 ++++++++++++++++
 include/oi/result/SizedResult.h               |  84 +++++++++
 oi/CMakeLists.txt                             |   4 -
 oi/exporters/Json.cpp                         | 146 ----------------
 test/integration/CMakeLists.txt               |   2 +-
 test/integration/anonymous.toml               |  15 +-
 test/integration/arrays.toml                  |  14 +-
 test/integration/enums.toml                   |  16 +-
 test/integration/enums_params.toml            |  10 +-
 test/integration/fbstring.toml                |   7 +
 test/integration/folly_f14_fast_map.toml      |  26 +--
 test/integration/folly_f14_fast_set.toml      |   9 +-
 test/integration/folly_f14_node_map.toml      |  26 +--
 test/integration/folly_f14_node_set.toml      |   9 +-
 test/integration/folly_f14_value_map.toml     |  26 +--
 test/integration/folly_f14_value_set.toml     |   9 +-
 test/integration/folly_f14_vector_map.toml    |  26 +--
 test/integration/folly_f14_vector_set.toml    |   9 +-
 test/integration/folly_small_vector.toml      |  28 +--
 test/integration/folly_sorted_vector_map.toml |  30 ++--
 test/integration/gen_tests.py                 |   5 +-
 test/integration/inheritance_access.toml      |  22 ++-
 test/integration/inheritance_multiple.toml    |  13 +-
 test/integration/packed.toml                  |   8 +-
 test/integration/padding.toml                 |  26 +--
 test/integration/pointers.toml                |  32 ++--
 test/integration/primitives.toml              |  38 ++--
 test/integration/runner_common.cpp            |   2 +
 test/integration/simple.toml                  |  20 ++-
 test/integration/sorted_vector_set.toml       |   2 +
 test/integration/std_list.toml                |  28 +--
 test/integration/std_list_del_allocator.toml  |   5 +-
 .../std_map_custom_comparator.toml            |   8 +-
 .../std_multimap_custom_comparator.toml       |   6 +-
 .../std_multiset_custom_comparator.toml       |   9 +-
 test/integration/std_optional.toml            |   5 +
 test/integration/std_pair.toml                |  24 +--
 .../std_set_custom_comparator.toml            |   9 +-
 test/integration/std_smart_ptr.toml           |  33 ++--
 .../std_unordered_map_custom_operator.toml    |  10 +-
 ...td_unordered_multimap_custom_operator.toml |  10 +-
 ...td_unordered_multiset_custom_operator.toml |  10 +-
 .../std_unordered_set_custom_operator.toml    |  10 +-
 test/integration/std_vector.toml              |  36 ++--
 .../integration/std_vector_del_allocator.toml |  17 +-
 test/integration/templates.toml               |   4 +
 test/integration/thrift_unions.toml           |  15 +-
 test/integration/typedefed_parent.toml        |  10 +-
 test/integration/unions.toml                  |  30 ++--
 50 files changed, 795 insertions(+), 454 deletions(-)
 create mode 100644 include/oi/result/SizedResult-inl.h
 create mode 100644 include/oi/result/SizedResult.h
 delete mode 100644 oi/exporters/Json.cpp

diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h
index 31aa808..dd88ade 100644
--- a/include/oi/exporters/Json.h
+++ b/include/oi/exporters/Json.h
@@ -17,8 +17,10 @@
 #define INCLUDED_OI_EXPORTERS_JSON_H 1
 
 #include <oi/IntrospectionResult.h>
+#include <oi/result/SizedResult.h>
 
 #include <ostream>
+#include <string_view>
 
 namespace oi::exporters {
 
@@ -26,19 +28,173 @@ class Json {
  public:
   Json(std::ostream& out);
 
-  void print(const IntrospectionResult&);
-  void print(IntrospectionResult::const_iterator& it,
-             IntrospectionResult::const_iterator& end);
+  template <typename Res>
+  void print(const Res& r) {
+    auto begin = r.begin();
+    auto end = r.end();
+    return print(begin, end);
+  }
+  template <typename It>
+  void print(It& it, const It& end);
 
   void setPretty(bool pretty) {
     pretty_ = pretty;
   }
 
  private:
+  std::string_view tab() const;
+  std::string_view space() const;
+  std::string_view endl() const;
+  static std::string makeIndent(size_t depth);
+
+  void printStringField(std::string_view name,
+                        std::string_view value,
+                        std::string_view indent);
+  void printBoolField(std::string_view name,
+                      bool value,
+                      std::string_view indent);
+  void printUnsignedField(std::string_view name,
+                          uint64_t value,
+                          std::string_view indent);
+  void printPointerField(std::string_view name,
+                         uintptr_t value,
+                         std::string_view indent);
+  template <typename Rng>
+  void printListField(std::string_view name,
+                      const Rng& range,
+                      std::string_view indent);
+
+  void printFields(const result::Element&, std::string_view indent);
+  template <typename El>
+  void printFields(const result::SizedElement<El>&, std::string_view indent);
+
   bool pretty_ = false;
   std::ostream& out_;
 };
 
+inline Json::Json(std::ostream& out) : out_(out) {
+}
+
+inline std::string_view Json::tab() const {
+  return pretty_ ? "  " : "";
+}
+inline std::string_view Json::space() const {
+  return pretty_ ? " " : "";
+}
+inline std::string_view Json::endl() const {
+  return pretty_ ? "\n" : "";
+}
+inline std::string Json::makeIndent(size_t depth) {
+  depth = std::max(depth, 1UL);
+  return std::string((depth - 1) * 4, ' ');
+}
+
+inline void Json::printStringField(std::string_view name,
+                                   std::string_view value,
+                                   std::string_view indent) {
+  out_ << tab() << '"' << name << '"' << ':' << space() << "\"" << value
+       << "\"," << endl() << indent;
+}
+inline void Json::printBoolField(std::string_view name,
+                                 bool value,
+                                 std::string_view indent) {
+  out_ << tab() << '"' << name << "\":" << space() << value << ',' << endl()
+       << indent;
+}
+inline void Json::printUnsignedField(std::string_view name,
+                                     uint64_t value,
+                                     std::string_view indent) {
+  out_ << tab() << '"' << name << "\":" << space() << value << ',' << endl()
+       << indent;
+}
+inline void Json::printPointerField(std::string_view name,
+                                    uintptr_t value,
+                                    std::string_view indent) {
+  out_ << tab() << '"' << name << "\":" << space() << "\"0x" << std::hex
+       << value << std::dec << "\"," << endl() << indent;
+}
+template <typename Rng>
+void Json::printListField(std::string_view name,
+                          const Rng& range,
+                          std::string_view indent) {
+  out_ << tab() << '"' << name << '"' << ':' << space() << '[';
+  bool first = true;
+  for (const auto& el : range) {
+    if (!std::exchange(first, false))
+      out_ << ',' << space();
+    out_ << '"' << el << '"';
+  }
+  out_ << "]," << endl() << indent;
+}
+
+template <typename El>
+void Json::printFields(const result::SizedElement<El>& el,
+                       std::string_view indent) {
+  printUnsignedField("size", el.size, indent);
+
+  printFields(el.inner(), indent);
+}
+
+inline void Json::printFields(const result::Element& el,
+                              std::string_view indent) {
+  printStringField("name", el.name, indent);
+  printListField("typePath", el.type_path, indent);
+  printListField("typeNames", el.type_names, indent);
+  printUnsignedField("staticSize", el.static_size, indent);
+  printUnsignedField("exclusiveSize", el.exclusive_size, indent);
+  if (el.pointer.has_value())
+    printUnsignedField("pointer", *el.pointer, indent);
+
+  if (const auto* s = std::get_if<result::Element::Scalar>(&el.data)) {
+    printUnsignedField("data", s->n, indent);
+  } else if (const auto* p = std::get_if<result::Element::Pointer>(&el.data)) {
+    printPointerField("data", p->p, indent);
+  } else if (const auto* str = std::get_if<std::string>(&el.data)) {
+    printStringField("data", *str, indent);
+  }
+
+  if (el.container_stats.has_value()) {
+    printUnsignedField("length", el.container_stats->length, indent);
+    printUnsignedField("capacity", el.container_stats->capacity, indent);
+  }
+  if (el.is_set_stats.has_value())
+    printUnsignedField("is_set", el.is_set_stats->is_set, indent);
+}
+
+template <typename It>
+void Json::print(It& it, const It& end) {
+  const auto depth = it->type_path.size();
+
+  const auto thisIndent = pretty_ ? makeIndent(depth) : "";
+  const auto lastIndent = pretty_ ? makeIndent(depth - 1) : "";
+
+  out_ << '[' << endl() << thisIndent;
+
+  bool first = true;
+  while (it != end && it->type_path.size() >= depth) {
+    if (!std::exchange(first, false))
+      out_ << ',' << endl() << thisIndent;
+
+    out_ << '{' << endl() << thisIndent;
+
+    printFields(*it, thisIndent);
+
+    out_ << tab() << "\"members\":" << space();
+    if (++it != end && it->type_path.size() > depth) {
+      print(it, end);
+    } else {
+      out_ << "[]" << endl();
+    }
+
+    out_ << thisIndent << "}";
+  }
+  if (depth == 1) {
+    out_ << endl() << ']' << endl();
+  } else {
+    out_ << endl() << lastIndent << tab() << ']' << endl();
+  }
+}
+
 }  // namespace oi::exporters
 
 #endif
diff --git a/include/oi/result/SizedResult-inl.h b/include/oi/result/SizedResult-inl.h
new file mode 100644
index 0000000..f86ff0c
--- /dev/null
+++ b/include/oi/result/SizedResult-inl.h
@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#if defined(INCLUDED_OI_RESULT_SIZED_ELEMENT_INL_H) || \
+    !defined(INCLUDED_OI_RESULT_SIZED_ELEMENT_H)
+static_assert(false,
+              "SizedResult-inl.h provides inline declarations for "
+              "SizedResult.h and should only "
+              "be included by SizedResult.h");
+#endif
+#define INCLUDED_OI_RESULT_SIZED_ELEMENT_INL_H 1
+
+#include <utility>
+#include <vector>
+
+#include "SizedResult.h"
+
+namespace oi::result {
+
+template <typename Res>
+SizedResult<Res>::SizedResult(Res res) : res_{std::move(res)} {
+}
+
+template <typename Res>
+typename SizedResult<Res>::const_iterator SizedResult<Res>::begin() const {
+  return ++const_iterator{res_.begin(), res_.end()};
+}
+template <typename Res>
+typename SizedResult<Res>::const_iterator SizedResult<Res>::end() const {
+  return res_.end();
+}
+
+template <typename Res>
+SizedResult<Res>::const_iterator::const_iterator(It it, const It& end)
+    : data_{it} {
+  struct StackEntry {
+    size_t index;
+    size_t depth;
+  };
+  std::vector<StackEntry> stack;
+
+  size_t size = 0;
+  size_t count = -1;
+  for (; it != end; ++it) {
+    ++count;
+
+    auto depth = it->type_path.size();
+    while (!stack.empty() && stack.back().depth >= depth) {
+      auto i = stack.back().index;
+      stack.pop_back();
+      helpers_[i].last_child = count - 1;
+    }
+
+    size += it->exclusive_size;
+    helpers_.emplace_back(SizeHelper{.size = size});
+    stack.emplace_back(StackEntry{.index = count, .depth = depth});
+  }
+  while (!stack.empty()) {
+    auto i = stack.back().index;
+    stack.pop_back();
+    helpers_[i].last_child = count;
+  }
+}
+
+template <typename Res>
+SizedResult<Res>::const_iterator::const_iterator(It end) : data_{end} {
+}
+
+template <typename Res>
+typename SizedResult<Res>::const_iterator
+SizedResult<Res>::const_iterator::operator++(int) {
+  auto old = *this;
+  operator++();
+  return old;
+}
+
+template <typename Res>
+typename SizedResult<Res>::const_iterator&
+SizedResult<Res>::const_iterator::operator++() {
+  // The below iterator is already pointing at the first element while this
+  // iterator is not. Skip incrementing it the first time around.
+  if (count_ != 0)
+    ++data_;
+
+  if (count_ == helpers_.size()) {
+    next_.reset();
+    return *this;
+  }
+
+  size_t size = helpers_[helpers_[count_].last_child].size;
+  if (count_ != 0)
+    size -= helpers_[count_ - 1].size;
+
+  next_.emplace(*data_, size);
+  ++count_;
+  return *this;
+}
+
+template <typename Res>
+const typename SizedResult<Res>::Element&
+SizedResult<Res>::const_iterator::operator*() const {
+  return *next_;
+}
+template <typename Res>
+const typename SizedResult<Res>::Element*
+SizedResult<Res>::const_iterator::operator->() const {
+  return &*next_;
+}
+
+template <typename Res>
+bool SizedResult<Res>::const_iterator::operator==(
+    const SizedResult<Res>::const_iterator& that) const {
+  return this->data_ == that.data_;
+}
+
+template <typename Res>
+bool SizedResult<Res>::const_iterator::operator!=(
+    const SizedResult<Res>::const_iterator& that) const {
+  return !(*this == that);
+}
+
+template <typename El>
+SizedElement<El>::SizedElement(const El& el, size_t size_)
+    : El{el}, size{size_} {
+}
+
+template <typename El>
+const El& SizedElement<El>::inner() const {
+  return static_cast<const El&>(*this);
+}
+
+}  // namespace oi::result
diff --git a/include/oi/result/SizedResult.h b/include/oi/result/SizedResult.h
new file mode 100644
index 0000000..983b804
--- /dev/null
+++ b/include/oi/result/SizedResult.h
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef INCLUDED_OI_RESULT_SIZED_ELEMENT_H
+#define INCLUDED_OI_RESULT_SIZED_ELEMENT_H 1
+
+#include <cstddef>
+#include <optional>
+#include <type_traits>
+#include <vector>
+
+namespace oi::result {
+
+template <typename El>
+struct SizedElement : public El {
+  SizedElement(const El& el, size_t size);
+  const El& inner() const;
+
+  size_t size;
+};
+
+template <typename Res>
+class SizedResult {
+ private:
+  using It = std::decay_t<decltype(std::declval<Res&>().begin())>;
+  using ParentEl = std::decay_t<decltype(std::declval<It&>().operator*())>;
+
+ public:
+  using Element = SizedElement<ParentEl>;
+
+ private:
+  struct SizeHelper {
+    size_t size = -1;
+    size_t last_child = -1;
+  };
+
+ public:
+  class const_iterator {
+    friend SizedResult;
+
+   public:
+    bool operator==(const const_iterator& that) const;
+    bool operator!=(const const_iterator& that) const;
+    const Element& operator*() const;
+    const Element* operator->() const;
+    const_iterator& operator++();
+    const_iterator operator++(int);
+
+   private:
+    const_iterator(It start, const It& end);
+    const_iterator(It end);
+
+    It data_;
+
+    std::vector<SizeHelper> helpers_;
+    size_t count_ = 0;
+    std::optional<Element> next_;
+  };
+
+  SizedResult(Res res);
+
+  const_iterator begin() const;
+  const_iterator end() const;
+
+ private:
+  Res res_;
+};
+
+}  // namespace oi::result
+
+#include "SizedResult-inl.h"
+#endif
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index 3754f41..64dbcae 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -55,10 +55,6 @@ target_link_libraries(codegen
   glog::glog
 )
 
-add_library(exporters_json exporters/Json.cpp)
-target_include_directories(exporters_json PUBLIC ${CMAKE_SOURCE_DIR}/include)
-target_link_libraries(exporters_json oil)
-
 add_library(exporters_csv exporters/CSV.cpp)
 target_include_directories(exporters_csv PUBLIC ${CMAKE_SOURCE_DIR}/include)
 target_link_libraries(exporters_csv oil)
diff --git a/oi/exporters/Json.cpp b/oi/exporters/Json.cpp
deleted file mode 100644
index 8a4d33c..0000000
--- a/oi/exporters/Json.cpp
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include <oi/exporters/Json.h>
-
-#include <algorithm>
-#include <stdexcept>
-#include <variant>
-
-template <class>
-inline constexpr bool always_false_v = false;
-
-namespace oi::exporters {
-namespace {
-
-template <typename It>
-void printStringList(std::ostream& out, It it, It end, bool pretty) {
-  out << '[';
-  for (; it != end; ++it) {
-    out << '"' << (*it) << '"';
-    if (it != end - 1)
-      out << (pretty ? ", " : ",");
-  }
-  out << ']';
-}
-
-std::string makeIndent(size_t depth) {
-  depth = std::max(depth, 1UL);
-  return std::string((depth - 1) * 4, ' ');
-}
-
-}  // namespace
-
-Json::Json(std::ostream& out) : out_(out) {
-}
-
-void Json::print(const IntrospectionResult& r) {
-  auto begin = r.cbegin();
-  auto end = r.cend();
-  return print(begin, end);
-}
-
-void Json::print(IntrospectionResult::const_iterator& it,
-                 IntrospectionResult::const_iterator& end) {
-  const auto firstTypePathSize = it->type_path.size();
-
-  const auto indent = pretty_ ? makeIndent(firstTypePathSize) : "";
-  const auto lastIndent =
-      pretty_ ? makeIndent(std::max(firstTypePathSize, 1UL) - 1) : "";
-  const auto* tab = pretty_ ? "  " : "";
-  const auto* space = pretty_ ? " " : "";
-  const auto* endl = pretty_ ? "\n" : "";
-
-  out_ << '[' << endl << indent;
-
-  bool first = true;
-  while (it != end) {
-    if (it->type_path.size() < firstTypePathSize) {
-      // no longer a sibling, must be a sibling of the type we're printing
-      break;
-    }
-
-    if (!first)
-      out_ << ',' << endl << indent;
-    first = false;
-
-    out_ << '{' << endl << indent;
-
-    out_ << tab << "\"name\"" << space << ':' << space << "\"" << it->name
-         << "\"," << endl
-         << indent;
-
-    out_ << tab << "\"typePath\"" << space << ':' << space << "";
-    printStringList(out_, it->type_path.begin(), it->type_path.end(), pretty_);
-    out_ << (pretty_ ? ",\n" : ",") << indent;
-
-    out_ << tab << "\"typeNames\"" << space << ':' << space;
-    printStringList(
-        out_, it->type_names.begin(), it->type_names.end(), pretty_);
-    out_ << ',' << endl << indent;
-
-    out_ << tab << "\"staticSize\":" << space << it->static_size << ',' << endl
-         << indent;
-    out_ << tab << "\"exclusiveSize\":" << space << it->exclusive_size << ','
-         << endl
-         << indent;
-
-    if (it->pointer.has_value()) {
-      out_ << tab << "\"pointer\":" << space << *(it->pointer) << ',' << endl
-           << indent;
-    }
-
-    if (auto* s = std::get_if<result::Element::Scalar>(&it->data)) {
-      out_ << tab << "\"data\":" << space << s->n << ',' << endl << indent;
-    } else if (auto* p = std::get_if<result::Element::Pointer>(&it->data)) {
-      out_ << tab << "\"data\":" << space << "\"0x" << std::hex << p->p
-           << std::dec << "\"," << endl
-           << indent;
-    } else if (auto* str = std::get_if<std::string>(&it->data)) {
-      out_ << tab << "\"data\":" << space << "\"" << *str << "\"," << endl
-           << indent;
-    }
-
-    if (it->container_stats.has_value()) {
-      out_ << tab << "\"length\":" << space << it->container_stats->length
-           << ',' << endl
-           << indent;
-      out_ << tab << "\"capacity\":" << space << it->container_stats->capacity
-           << ',' << endl
-           << indent;
-    }
-    if (it->is_set_stats.has_value()) {
-      out_ << tab << "\"is_set\":" << space << it->is_set_stats->is_set << ','
-           << endl
-           << indent;
-    }
-
-    out_ << tab << "\"members\":" << space;
-    if (++it != end && it->type_path.size() > firstTypePathSize) {
-      print(it, end);
-    } else {
-      out_ << "[]" << endl;
-    }
-
-    out_ << indent << "}";
-  }
-  if (firstTypePathSize == 1) {
-    out_ << endl << ']' << endl;
-  } else {
-    out_ << endl << lastIndent << tab << ']' << endl;
-  }
-}
-
-}  // namespace oi::exporters
diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 12cb239..4d235b5 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -50,7 +50,7 @@ add_executable(integration_test_target
   ${INTEGRATION_TEST_TARGET_SRC}
   folly_shims.cpp)
 target_compile_options(integration_test_target PRIVATE -O1)
-target_link_libraries(integration_test_target PRIVATE oil_jit exporters_json Boost::headers ${Boost_LIBRARIES})
+target_link_libraries(integration_test_target PRIVATE oil_jit Boost::headers ${Boost_LIBRARIES})
 
 add_executable(integration_test_runner ${INTEGRATION_TEST_RUNNER_SRC} runner_common.cpp)
 target_include_directories(integration_test_runner PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
diff --git a/test/integration/anonymous.toml b/test/integration/anonymous.toml
index 6a91293..15b9f10 100644
--- a/test/integration/anonymous.toml
+++ b/test/integration/anonymous.toml
@@ -88,10 +88,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":12,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size": 12,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":4, "dynamicSize":0},
-        {"name":"c", "staticSize":4, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"c", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}
       ]}]'''
 
   [cases.anon_struct]
@@ -207,10 +209,11 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize": 24,
       "exclusiveSize": 10,
+      "size": 24,
       "members": [
-        {"name":"__oi_anon_0", "staticSize":2, "exclusiveSize":2},
-        {"name":"__oi_anon_2", "staticSize":8, "exclusiveSize":8},
-        {"name":"e", "staticSize":4, "exclusiveSize":4, "typeNames":["int32_t"]}
+        {"name":"__oi_anon_0", "staticSize":2, "exclusiveSize":2, "size":2},
+        {"name":"__oi_anon_2", "staticSize":8, "exclusiveSize":8, "size":8},
+        {"name":"e", "staticSize":4, "exclusiveSize":4, "size":4, "typeNames":["int32_t"]}
       ]
     }]'''
 
diff --git a/test/integration/arrays.toml b/test/integration/arrays.toml
index 4a06b13..5074b62 100644
--- a/test/integration/arrays.toml
+++ b/test/integration/arrays.toml
@@ -33,9 +33,11 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":40,
       "exclusiveSize":0,
+      "size":40,
       "members":[{
         "staticSize":40,
         "exclusiveSize":0,
+        "size":40,
         "length":10,
         "capacity":10
       }]}]'''
@@ -56,9 +58,11 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":1,
       "exclusiveSize":1,
+      "size":1,
       "members":[{
         "staticSize":0,
-        "exclusiveSize":0
+        "exclusiveSize":0,
+        "size":1
       }]}]'''
   [cases.multidim_legacy] # Test for legacy behaviour. Remove with OICodeGen
     oil_disable = 'oil only runs on codegen v2'
@@ -86,10 +90,10 @@ definitions = '''
           {"staticSize":12, "dynamicSize":0, "exclusiveSize":12, "length":3, "capacity":3, "elementStaticSize":4}]
       }]}]'''
     expect_json_v2 = '''[
-      {"staticSize":24, "exclusiveSize":0, "members":[
-        {"staticSize":24, "exclusiveSize":0, "length":2, "capacity":2, "members":[
-          {"staticSize":12, "exclusiveSize":0, "length":3, "capacity":3},
-          {"staticSize":12, "exclusiveSize":0, "length":3, "capacity":3}]
+      {"staticSize":24, "exclusiveSize":0, "size":24, "members":[
+        {"staticSize":24, "exclusiveSize":0, "size":24, "length":2, "capacity":2, "members":[
+          {"staticSize":12, "exclusiveSize":0, "size":12, "length":3, "capacity":3},
+          {"staticSize":12, "exclusiveSize":0, "size":12, "length":3, "capacity":3}]
       }]}]'''
   [cases.direct_int10]
     skip = "Direct array arguments don't work"
diff --git a/test/integration/enums.toml b/test/integration/enums.toml
index f112b66..fa91cbc 100644
--- a/test/integration/enums.toml
+++ b/test/integration/enums.toml
@@ -36,31 +36,31 @@ definitions = '''
     param_types = ["ScopedEnum"]
     setup = "return {};"
     expect_json = '[{"staticSize":4, "dynamicSize":0}]'
-    expect_json_v2 = '[{"typeNames": ["ScopedEnum"], "staticSize":4, "exclusiveSize":4}]'
+    expect_json_v2 = '[{"typeNames": ["ScopedEnum"], "staticSize":4, "exclusiveSize":4, "size":4}]'
   [cases.scoped_uint8]
     param_types = ["ScopedEnumUint8"]
     setup = "return {};"
     expect_json = '[{"staticSize":1, "dynamicSize":0}]'
-    expect_json_v2 = '[{"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1}]'
+    expect_json_v2 = '[{"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1}]'
   [cases.unscoped]
     param_types = ["UNSCOPED_ENUM"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.anonymous]
     skip = "TreeBuilder crashes" # https://github.com/facebookexperimental/object-introspection/issues/232
     param_types = ["Holder&"]
     setup = "return {};"
     expect_json = '''[
-      {"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "members":[
-        {"name":"e", "staticSize":4, "dynamicSize":0, "exclusiveSize":4}
+      {"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4, "members":[
+        {"name":"e", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}
     ]}]'''
   [cases.paired_with_element]
     param_types = ["Pair<ScopedEnumUint8, uint8_t>"]
     setup = "return {};"
     expect_json = '[{"staticSize":2, "dynamicSize":0}]'
     expect_json_v2 = '''[
-      {"staticSize": 2, "exclusiveSize": 0, "members": [
-        {"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1},
-        {"typeNames": ["uint8_t"], "staticSize":1, "exclusiveSize":1}
+      {"staticSize": 2, "exclusiveSize": 0, "size":2, "members": [
+        {"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1},
+        {"typeNames": ["uint8_t"], "staticSize":1, "exclusiveSize":1, "size":1}
       ]}
     ]'''
diff --git a/test/integration/enums_params.toml b/test/integration/enums_params.toml
index cf61d6f..dd66f31 100644
--- a/test/integration/enums_params.toml
+++ b/test/integration/enums_params.toml
@@ -39,20 +39,20 @@ definitions = '''
     param_types = ["const std::array<int, static_cast<size_t>(MyNS::ScopedEnum::Two)>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":8, "length":2, "capacity":2, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":8, "dynamicSize":0, "length":2, "capacity":2}]'
+    expect_json_v2 = '[{"staticSize":8, "exclusiveSize":0, "size":8, "length":2, "capacity":2}]'
 
   [cases.scoped_enum_val]
     param_types = ["const MyClass<MyNS::ScopedEnum::One>&"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4}]'
   [cases.scoped_enum_val_gaps]
     param_types = ["const ClassGaps<MyNS::EnumWithGaps::Twenty>&"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4}]'
   [cases.scoped_enum_val_negative]
     param_types = ["const ClassGaps<MyNS::EnumWithGaps::MinusTwo>&"]
     setup = "return {};"
-    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":0, "size":4}]'
 
   [cases.unscoped_enum_type]
     param_types = ["const std::vector<MyNS::UNSCOPED_ENUM>&"]
@@ -61,4 +61,4 @@ definitions = '''
     param_types = ["const std::array<int, MyNS::ONE>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":4, "length":1, "capacity":1, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":4, "length":1, "capacity":1}]'
+    expect_json_v2 = '[{"staticSize":4, "exclusiveSize":0, "size":4, "length":1, "capacity":1}]'
diff --git a/test/integration/fbstring.toml b/test/integration/fbstring.toml
index 6e68d99..cec73ca 100644
--- a/test/integration/fbstring.toml
+++ b/test/integration/fbstring.toml
@@ -28,6 +28,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 24,
+      "size": 24,
       "length": 0,
       "capacity": 23
     }]'''
@@ -60,6 +61,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 24,
+      "size": 24,
       "length": 6,
       "capacity": 23
     }]'''
@@ -92,6 +94,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 50,
+      "size": 50,
       "length": 26,
       "capacity": 26
     }]'''
@@ -124,6 +127,7 @@ includes = ["folly/FBString.h", "utility"]
       "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
       "staticSize": 24,
       "exclusiveSize": 1056,
+      "size": 1056,
       "length": 1024,
       "capacity": 1024
     }]'''
@@ -137,17 +141,20 @@ includes = ["folly/FBString.h", "utility"]
     expect_json_v2 = '''[{
       "staticSize": 48,
       "exclusiveSize": 0,
+      "size": 1080,
       "members": [
         {
           "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
           "staticSize": 24,
           "exclusiveSize": 1056,
+          "size": 1056,
           "length": 1024,
           "capacity": 1024
         }, {
           "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
           "staticSize": 24,
           "exclusiveSize": 24,
+          "size": 24,
           "length": 1024,
           "capacity": 1024
         }
diff --git a/test/integration/folly_f14_fast_map.toml b/test/integration/folly_f14_fast_map.toml
index 5f9e509..273c222 100644
--- a/test/integration/folly_f14_fast_map.toml
+++ b/test/integration/folly_f14_fast_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 464,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "size": 72, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "size": 104, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "size": 104,"length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 40,
+          "size": 184,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_fast_set.toml b/test/integration/folly_f14_fast_set.toml
index 486b9ba..24d8abe 100644
--- a/test/integration/folly_f14_fast_set.toml
+++ b/test/integration/folly_f14_fast_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size":304,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "size":56, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "size":88, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "size":72, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "size":88, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_f14_node_map.toml b/test/integration/folly_f14_node_map.toml
index 78d414d..07b5b89 100644
--- a/test/integration/folly_f14_node_map.toml
+++ b/test/integration/folly_f14_node_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size":668,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "size": 96, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "size": 148, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "size": 160, "length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 120,
+          "size": 264,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_node_set.toml b/test/integration/folly_f14_node_set.toml
index 497976c..a68cc7b 100644
--- a/test/integration/folly_f14_node_set.toml
+++ b/test/integration/folly_f14_node_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 500,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 120, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 72, "size": 84, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 88, "size": 128, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 104, "size": 132, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 120, "size": 156, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_f14_value_map.toml b/test/integration/folly_f14_value_map.toml
index fac9829..a1a09e8 100644
--- a/test/integration/folly_f14_value_map.toml
+++ b/test/integration/folly_f14_value_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 464,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 48, "size": 72, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 44, "size": 104, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 48, "size": 104, "length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 40,
+          "size": 184,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_value_set.toml b/test/integration/folly_f14_value_set.toml
index 3f19b31..2cf03df 100644
--- a/test/integration/folly_f14_value_set.toml
+++ b/test/integration/folly_f14_value_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 304,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 44, "size": 56, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 48, "size": 88, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 44, "size": 72, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 52, "size": 88, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_f14_vector_map.toml b/test/integration/folly_f14_vector_map.toml
index 9912141..4126243 100644
--- a/test/integration/folly_f14_vector_map.toml
+++ b/test/integration/folly_f14_vector_map.toml
@@ -70,26 +70,28 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 576,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 64, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 60, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 80, "length": 7, "capacity": 7},
+        {"name":"m1", "staticSize":24, "exclusiveSize": 64, "size": 88, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 60, "size": 120, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 80, "size": 136, "length": 7, "capacity": 7},
         {
           "name":"m4",
           "staticSize":24,
           "exclusiveSize": 88,
+          "size": 232,
           "length": 9,
           "capacity": 9,
           "members":[
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4},
-            {"staticSize":16, "exclusiveSize": 4}
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16},
+            {"staticSize":16, "exclusiveSize": 4, "size":16}
           ]
         }]
       }]'''
diff --git a/test/integration/folly_f14_vector_set.toml b/test/integration/folly_f14_vector_set.toml
index 3ffad84..c67f649 100644
--- a/test/integration/folly_f14_vector_set.toml
+++ b/test/integration/folly_f14_vector_set.toml
@@ -75,9 +75,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize": 0,
+      "size": 400,
       "members":[
-        {"name":"m1", "staticSize":24, "exclusiveSize": 60, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":24, "exclusiveSize": 64, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":24, "exclusiveSize": 76, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":24, "exclusiveSize": 84, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":24, "exclusiveSize": 60, "size": 72, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":24, "exclusiveSize": 64, "size": 104, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":24, "exclusiveSize": 76, "size": 104, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":24, "exclusiveSize": 84, "size": 120, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/folly_small_vector.toml b/test/integration/folly_small_vector.toml
index 5ae53f3..b97b5ff 100644
--- a/test/integration/folly_small_vector.toml
+++ b/test/integration/folly_small_vector.toml
@@ -4,23 +4,23 @@ includes = ["folly/small_vector.h", "vector"]
     param_types = ["const folly::small_vector<int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "length":0, "capacity":2, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "length":0, "capacity":2}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "size":16, "length":0, "capacity":2}]'
   [cases.int_default_inlined]
     param_types = ["const folly::small_vector<int>&"]
     setup = "return {{1,2}};"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "length":2, "capacity":2, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":8, "length":2, "capacity":2}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":8, "size":16, "length":2, "capacity":2}]'
   [cases.int_default_overflow]
     param_types = ["const folly::small_vector<int>&"]
     setup = "return {{1,2,3,4}};"
     expect_json = '[{"staticSize":16, "dynamicSize":24, "exclusiveSize":40, "length":4, "capacity":6, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":24, "length":4, "capacity":6}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":24, "size":40, "length":4, "capacity":6}]'
 
   [cases.vector_3_empty]
     param_types = ["const folly::small_vector<std::vector<int>, 3>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":80, "dynamicSize":0, "exclusiveSize":80, "length":0, "capacity":3, "elementStaticSize":24}]'
-    expect_json_v2 = '[{"staticSize":80, "exclusiveSize":80, "length":0, "capacity":3}]'
+    expect_json_v2 = '[{"staticSize":80, "exclusiveSize":80, "size":80, "length":0, "capacity":3}]'
   [cases.vector_3_inlined]
     param_types = ["const folly::small_vector<std::vector<int>, 3>&"]
     setup = "return {{ {1,2,3}, {4}, {5,6} }};"
@@ -31,10 +31,10 @@ includes = ["folly/small_vector.h", "vector"]
         {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
     ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":80, "length":3, "exclusiveSize":8, "capacity":3, "members":[
-        {"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3},
-        {"staticSize":24, "exclusiveSize":24, "length":1, "capacity":1},
-        {"staticSize":24, "exclusiveSize":24, "length":2, "capacity":2}
+      {"staticSize":80, "length":3, "exclusiveSize":8, "size":104, "capacity":3, "members":[
+        {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity":3},
+        {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity":1},
+        {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity":2}
     ]}]'''
   [cases.vector_3_overflow]
     param_types = ["const folly::small_vector<std::vector<int>, 3>&"]
@@ -47,15 +47,15 @@ includes = ["folly/small_vector.h", "vector"]
         {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4}
     ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":80, "exclusiveSize":104, "length":4, "capacity":5, "members":[
-        {"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3},
-        {"staticSize":24, "exclusiveSize":24, "length":1, "capacity":1},
-        {"staticSize":24, "exclusiveSize":24, "length":2, "capacity":2},
-        {"staticSize":24, "exclusiveSize":24, "length":1, "capacity":1}
+      {"staticSize":80, "exclusiveSize":104, "size":228, "length":4, "capacity":5, "members":[
+        {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity":3},
+        {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity":1},
+        {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity":2},
+        {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity":1}
     ]}]'''
 
   [cases.int_always_heap]
     param_types = ["const folly::small_vector<int, 0>&"]
     setup = "return {{1}};"
     expect_json = '[{"staticSize":16, "dynamicSize":4, "exclusiveSize":20, "length":1, "capacity":1, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "length":1, "capacity":1}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16, "size":20, "length":1, "capacity":1}]'
diff --git a/test/integration/folly_sorted_vector_map.toml b/test/integration/folly_sorted_vector_map.toml
index 6d3a2a5..44f91e0 100644
--- a/test/integration/folly_sorted_vector_map.toml
+++ b/test/integration/folly_sorted_vector_map.toml
@@ -4,19 +4,19 @@ includes = ["folly/sorted_vector_types.h", "vector"]
     param_types = ["const folly::sorted_vector_map<int, int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "exclusiveSize":24, "length":0, "capacity":0, "elementStaticSize":8}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size":24, "length":0, "capacity":0, "members":[]}]'
   [cases.int_int_some]
     param_types = ["const folly::sorted_vector_map<int, int>&"]
     setup = "return {{ {1,2}, {3,4} }};"
     expect_json = '[{"staticSize":24, "dynamicSize":16, "exclusiveSize":40, "length":2, "capacity":2, "elementStaticSize":8}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":2, "capacity":2, "members":[
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":40, "length":2, "capacity":2, "members":[
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]},
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}
     ]}]'''
   [cases.int_int_reserve]
@@ -27,14 +27,14 @@ includes = ["folly/sorted_vector_types.h", "vector"]
       return m;
     '''
     expect_json = '[{"staticSize":24, "dynamicSize":80, "exclusiveSize":104, "length":2, "capacity":10, "elementStaticSize":8}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":88, "length":2, "capacity":10, "members":[
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":88, "size":104, "length":2, "capacity":10, "members":[
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]},
-      {"staticSize":8, "exclusiveSize":0, "members": [
-        {"name":"key", "staticSize":4, "exclusiveSize":4},
-        {"name":"value", "staticSize":4, "exclusiveSize":4}
+      {"staticSize":8, "exclusiveSize":0, "size":8, "members": [
+        {"name":"key", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"value", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}
     ]}]'''
 
diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py
index 2dbc3ff..b72e371 100644
--- a/test/integration/gen_tests.py
+++ b/test/integration/gen_tests.py
@@ -40,6 +40,7 @@ def add_headers(f, custom_headers, thrift_headers):
 
 #include <oi/oi.h>
 #include <oi/exporters/Json.h>
+#include <oi/result/SizedResult.h>
 
 """
     )
@@ -145,8 +146,8 @@ def add_test_setup(f, config):
         oil_func_body += "    auto pr = oi::exporters::Json(std::cout);\n"
         oil_func_body += "    pr.setPretty(true);\n"
         for i in range(len(case["param_types"])):
-            oil_func_body += f"    auto ret{i} = oi::setupAndIntrospect(a{i}, opts);\n"
-            oil_func_body += f"    pr.print(*ret{i});\n"
+            oil_func_body += f"    auto ret{i} = oi::result::SizedResult(*oi::setupAndIntrospect(a{i}, opts));\n"
+            oil_func_body += f"    pr.print(ret{i});\n"
 
         f.write(
             define_traceable_func(
diff --git a/test/integration/inheritance_access.toml b/test/integration/inheritance_access.toml
index faeb35c..9751052 100644
--- a/test/integration/inheritance_access.toml
+++ b/test/integration/inheritance_access.toml
@@ -21,27 +21,33 @@ definitions = '''
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"},
-        {"name":"public_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"},
+        {"name":"public_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
   [cases.protected]
     param_types = ["const Protected&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"},
-        {"name":"protected_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"},
+        {"name":"protected_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
   [cases.private]
     param_types = ["const Private&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"},
-        {"name":"private_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"},
+        {"name":"private_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
   [cases.public_as_base]
     param_types = ["const Base&"]
@@ -49,6 +55,8 @@ definitions = '''
     setup = "return {};"
     expect_json = '''[{
       "staticSize":4,
+      "exclusiveSize":0,
+      "size":4,
       "members":[
-        {"name":"base_int", "staticSize":4, "typeName": "int32_t"}
+        {"name":"base_int", "staticSize":4, "exclusiveSize":4, "size":4, "typeName": "int32_t"}
       ]}]'''
diff --git a/test/integration/inheritance_multiple.toml b/test/integration/inheritance_multiple.toml
index 61d725f..53f956f 100644
--- a/test/integration/inheritance_multiple.toml
+++ b/test/integration/inheritance_multiple.toml
@@ -37,11 +37,12 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":24,
       "exclusiveSize":0,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "exclusiveSize":4},
-        {"name":"b", "staticSize":4, "exclusiveSize":4},
-        {"name":"c", "staticSize":4, "exclusiveSize":4},
-        {"name":"d", "staticSize":4, "exclusiveSize":4},
-        {"name":"e", "staticSize":4, "exclusiveSize":4},
-        {"name":"f", "staticSize":4, "exclusiveSize":4}
+        {"name":"a", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"c", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"d", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"e", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"f", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}]'''
diff --git a/test/integration/packed.toml b/test/integration/packed.toml
index 2bbef02..0e45479 100644
--- a/test/integration/packed.toml
+++ b/test/integration/packed.toml
@@ -12,8 +12,10 @@ definitions = '''
     expect_json = '''[{
       "staticSize":17,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size":17,
       "members":[
-        {"name":"p", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":1, "dynamicSize":0},
-        {"name":"x", "staticSize":8, "dynamicSize":0}
+        {"name":"p", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1},
+        {"name":"x", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
diff --git a/test/integration/padding.toml b/test/integration/padding.toml
index 6cc17d1..5001be6 100644
--- a/test/integration/padding.toml
+++ b/test/integration/padding.toml
@@ -65,10 +65,11 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize": 7,
       "members":[
-        { "name":"a", "staticSize":8, "dynamicSize":0 },
-        { "name":"b", "staticSize":1, "dynamicSize":0 },
-        { "name":"c", "staticSize":8, "dynamicSize":0 }
+        { "name":"a", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
+        { "name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1 },
+        { "name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 }
       ]}]'''
 
   [cases.nested_padding]
@@ -77,18 +78,22 @@ definitions = '''
     expect_json = '''[{
       "staticSize":48,
       "dynamicSize":0,
+      "exclusiveSize":7,
+      "size":48,
       "members":[
-        { "name":"a", "staticSize":8, "dynamicSize":0 },
-        { "name":"b", "staticSize":1, "dynamicSize":0 },
-        { "name":"c", "staticSize":8, "dynamicSize":0 },
+        { "name":"a", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
+        { "name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1 },
+        { "name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
         {
           "name":"d",
           "staticSize":24,
           "dynamicSize":0,
+          "exclusiveSize":7,
+          "size":24,
           "members": [
-            { "name":"a", "staticSize":8, "dynamicSize":0 },
-            { "name":"b", "staticSize":1, "dynamicSize":0 },
-            { "name":"c", "staticSize":8, "dynamicSize":0 }
+            { "name":"a", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 },
+            { "name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1 },
+            { "name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8 }
         ]}
       ]}]'''
 
@@ -103,5 +108,6 @@ definitions = '''
     }]'''
     expect_json_v2 = '''[{
       "staticSize": 104,
-      "exclusiveSize": 32
+      "exclusiveSize": 32,
+      "size": 104
     }]'''
diff --git a/test/integration/pointers.toml b/test/integration/pointers.toml
index 389ba29..71bf2ad 100644
--- a/test/integration/pointers.toml
+++ b/test/integration/pointers.toml
@@ -153,10 +153,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize":4,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_primitive_ptrs_null]
     param_types = ["const PrimitivePtrs&"]
@@ -165,10 +167,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize":4,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
 
 
@@ -189,8 +193,10 @@ definitions = '''
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"vec", "staticSize":8, "dynamicSize":0}
+        {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_vector_ptr_null]
     param_types = ["const VectorPtr&"]
@@ -199,8 +205,10 @@ definitions = '''
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
+      "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"name":"vec", "staticSize":8, "dynamicSize":0}
+        {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
 
 
@@ -242,10 +250,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
+      "exclusiveSize":4,
+      "size":24,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":8, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4},
+        {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.feature_config]
     oil_disable = "oil can't chase raw pointers safely"
diff --git a/test/integration/primitives.toml b/test/integration/primitives.toml
index 429712f..94784fb 100644
--- a/test/integration/primitives.toml
+++ b/test/integration/primitives.toml
@@ -2,76 +2,76 @@
   [cases.short]
     param_types = ["short"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":2, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":2, "dynamicSize":0, "exclusiveSize":2, "size":2}]'
   [cases.ushort]
     param_types = ["unsigned short"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":2, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":2, "dynamicSize":0, "exclusiveSize":2, "size":2}]'
   [cases.int]
     param_types = ["int"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.uint]
     param_types = ["unsigned int"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.long]
     param_types = ["long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.ulong]
     param_types = ["unsigned long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.longlong]
     param_types = ["long long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.ulonglong]
     param_types = ["unsigned long long"]
     setup = "return 123;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.bool]
     param_types = ["bool"]
     setup = "return true;"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.char]
     param_types = ["char"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.uchar]
     param_types = ["unsigned char"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.schar]
     param_types = ["signed char"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.wchar_t]
     param_types = ["wchar_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.char8_t]
     param_types = ["char8_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":1, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1}]'
   [cases.char16_t]
     param_types = ["char16_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":2, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":2, "dynamicSize":0, "exclusiveSize":2, "size":2}]'
   [cases.char32_t]
     param_types = ["char32_t"]
     setup = "return 'a';"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.float]
     param_types = ["float"]
     setup = "return 3.14;"
-    expect_json = '[{"staticSize":4, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":4, "dynamicSize":0, "exclusiveSize":4, "size":4}]'
   [cases.double]
     param_types = ["double"]
     setup = "return 3.14;"
-    expect_json = '[{"staticSize":8, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}]'
   [cases.long_double]
     param_types = ["long double"]
     setup = "return 3.14;"
-    expect_json = '[{"staticSize":16, "dynamicSize":0}]'
+    expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "size":16}]'
diff --git a/test/integration/runner_common.cpp b/test/integration/runner_common.cpp
index c86bd59..3c8525a 100644
--- a/test/integration/runner_common.cpp
+++ b/test/integration/runner_common.cpp
@@ -365,6 +365,8 @@ void IntegrationBase::compare_json(const bpt::ptree& expected_json,
         }
       } else if (key == "dynamicSize" && val.get_value<size_t>() == 0) {
         continue;
+      } else if (key == "size") {
+        continue;
       }
 
       ADD_FAILURE() << "Expected key not found in output: " << curr_key;
diff --git a/test/integration/simple.toml b/test/integration/simple.toml
index 71a215a..0945f70 100644
--- a/test/integration/simple.toml
+++ b/test/integration/simple.toml
@@ -23,10 +23,12 @@ definitions = '''
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
+      "exclusiveSize": 3,
+      "size": 16,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":1, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
+        {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
   [cases.class]
     param_types = ["const SimpleClass&"]
@@ -34,15 +36,19 @@ definitions = '''
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
+      "exclusiveSize": 3,
+      "size": 16,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0},
-        {"name":"b", "staticSize":1, "dynamicSize":0},
-        {"name":"c", "staticSize":8, "dynamicSize":0}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
+        {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
   [cases.union]
     param_types = ["const SimpleUnion&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
-      "dynamicSize":0
+      "dynamicSize":0,
+      "exclusiveSize":8,
+      "size":8
     }]'''
diff --git a/test/integration/sorted_vector_set.toml b/test/integration/sorted_vector_set.toml
index 8e37abd..0a4e416 100644
--- a/test/integration/sorted_vector_set.toml
+++ b/test/integration/sorted_vector_set.toml
@@ -23,6 +23,7 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize": 24,
       "exclusiveSize": 24,
+      "size": 24,
       "length": 0,
       "capacity": 0
     }]'''
@@ -56,6 +57,7 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize": 24,
       "exclusiveSize": 28,
+      "size": 40,
       "length": 3,
       "capacity": 4
     }]'''
diff --git a/test/integration/std_list.toml b/test/integration/std_list.toml
index dfe2d8f..92c039a 100644
--- a/test/integration/std_list.toml
+++ b/test/integration/std_list.toml
@@ -13,30 +13,30 @@ definitions = '''
     param_types = ["const std::list<int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size":24, "length":0, "capacity":0, "members":[]}]'
   [cases.int_some]
     param_types = ["const std::list<int>&"]
     setup = "return {{1,2,3}};"
     expect_json = '[{"staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity":3, "members":[
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4}
     ]}]'''
   [cases.struct_some]
     param_types = ["const std::list<SimpleStruct>&"]
     setup = "return {{{}, {}, {}}};"
     expect_json = '[{"staticSize":24, "dynamicSize":48, "length":3, "capacity":3, "elementStaticSize":16}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
-      {"staticSize":16, "exclusiveSize":3},
-      {"staticSize":16, "exclusiveSize":3},
-      {"staticSize":16, "exclusiveSize":3}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":72, "length":3, "capacity":3, "members":[
+      {"staticSize":16, "exclusiveSize":3, "size":16},
+      {"staticSize":16, "exclusiveSize":3, "size":16},
+      {"staticSize":16, "exclusiveSize":3, "size":16}
     ]}]'''
   [cases.list_int_empty]
     param_types = ["const std::list<std::list<int>>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":24}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size":24, "length":0, "capacity":0, "members":[]}]'
   [cases.list_int_some]
     param_types = ["const std::list<std::list<int>>&"]
     setup = "return {{{1,2,3},{4},{5,6}}};"
@@ -52,8 +52,8 @@ definitions = '''
         {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4},
         {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
       ]}]'''
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[
-      {"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[]},
-      {"staticSize":24, "exclusiveSize":24, "length":1, "capacity": 1, "members":[]},
-      {"staticSize":24, "exclusiveSize":24, "length":2, "capacity": 2, "members":[]}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":120, "length":3, "capacity": 3, "members":[
+      {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity": 3},
+      {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity": 1},
+      {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity": 2}
     ]}]'''
diff --git a/test/integration/std_list_del_allocator.toml b/test/integration/std_list_del_allocator.toml
index b986996..c85d3ab 100644
--- a/test/integration/std_list_del_allocator.toml
+++ b/test/integration/std_list_del_allocator.toml
@@ -60,7 +60,8 @@ includes = ["list"]
     expect_json_v2 = '''[{
       "staticSize": 48,
       "exclusiveSize": 0,
+      "size": 60,
       "members": [
-        {"name": "v1", "staticSize": 24, "exclusiveSize": 24, "length": 1, "capacity": 1},
-        {"name": "v2", "staticSize": 24, "exclusiveSize": 24, "length": 2, "capacity": 2}
+        {"name": "v1", "staticSize": 24, "exclusiveSize": 24, "size": 28, "length": 1, "capacity": 1},
+        {"name": "v2", "staticSize": 24, "exclusiveSize": 24, "size": 32, "length": 2, "capacity": 2}
       ]}]'''
diff --git a/test/integration/std_map_custom_comparator.toml b/test/integration/std_map_custom_comparator.toml
index f8edd24..5f30b36 100644
--- a/test/integration/std_map_custom_comparator.toml
+++ b/test/integration/std_map_custom_comparator.toml
@@ -70,10 +70,12 @@ includes = ["map", "functional"]
     expect_json_v2 = '''[{
       "staticSize":8184,
       "exclusiveSize":0,
+      "size":9168,
       "members":[
         {"name":"m1",
          "staticSize":48,
          "exclusiveSize":48,
+         "size":192,
          "length":3,
          "capacity":3,
          "members": [
@@ -81,7 +83,7 @@ includes = ["map", "functional"]
           {},
           {}
          ]},
-        {"name":"m2", "staticSize":48, "exclusiveSize":48, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":48, "exclusiveSize":48, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":8040, "exclusiveSize":8040, "length":9, "capacity":9}
+        {"name":"m2", "staticSize":48, "exclusiveSize":48, "size":248, "length":5, "capacity":5},
+        {"name":"m3", "staticSize":48, "exclusiveSize":48, "size":328, "length":7, "capacity":7},
+        {"name":"m4", "staticSize":8040, "exclusiveSize":8040, "size":8400, "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_multimap_custom_comparator.toml b/test/integration/std_multimap_custom_comparator.toml
index 7f8171f..883c4cd 100644
--- a/test/integration/std_multimap_custom_comparator.toml
+++ b/test/integration/std_multimap_custom_comparator.toml
@@ -38,16 +38,18 @@ includes = ["map"]
     expect_json_v2 = '''[{
       "staticSize":96,
       "exclusiveSize":0,
+      "size":440,
       "members":[
         {"name":"m1",
          "staticSize":48,
          "exclusiveSize":48,
+         "size":192,
          "length":3,
          "capacity":3,
          "members": [
-          {"name":"[]", "staticSize":48, "exclusiveSize":36},
+          {"name":"[]", "staticSize":48, "exclusiveSize":36, "size":48},
           {},
           {}
          ]},
-        {"name":"m2", "staticSize":48, "exclusiveSize":48, "length":5, "capacity":5}
+        {"name":"m2", "staticSize":48, "exclusiveSize":48, "size":248, "length":5, "capacity":5}
       ]}]'''
diff --git a/test/integration/std_multiset_custom_comparator.toml b/test/integration/std_multiset_custom_comparator.toml
index 04d5da8..e8ef1aa 100644
--- a/test/integration/std_multiset_custom_comparator.toml
+++ b/test/integration/std_multiset_custom_comparator.toml
@@ -65,9 +65,10 @@ includes = ["set", "functional"]
     expect_json_v2 = '''[{
       "staticSize":8184,
       "exclusiveSize": 0,
+      "size":9144,
       "members":[
-        {"name":"m1", "staticSize":48, "exclusiveSize": 156, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":48, "exclusiveSize": 228, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":48, "exclusiveSize": 300, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":8040, "exclusiveSize": 8364, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":48, "exclusiveSize": 156, "size": 168, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":48, "exclusiveSize": 228, "size": 248, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":48, "exclusiveSize": 300, "size": 328, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":8040, "exclusiveSize": 8364, "size": 8400, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/std_optional.toml b/test/integration/std_optional.toml
index 7475e28..9da0271 100644
--- a/test/integration/std_optional.toml
+++ b/test/integration/std_optional.toml
@@ -20,6 +20,7 @@ includes = ["optional", "cstdint", "vector"]
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 16,
         "length": 0,
         "capacity": 1,
         "members": []
@@ -46,6 +47,7 @@ includes = ["optional", "cstdint", "vector"]
       {
         "staticSize": 16,
         "exclusiveSize": 8,
+        "size": 16,
         "length": 1,
         "capacity": 1,
         "members": [
@@ -77,6 +79,7 @@ includes = ["optional", "cstdint", "vector"]
       {
         "staticSize": 32,
         "exclusiveSize": 32,
+        "size": 32,
         "length": 0,
         "capacity": 1,
         "members": []
@@ -111,12 +114,14 @@ includes = ["optional", "cstdint", "vector"]
       {
         "staticSize": 32,
         "exclusiveSize": 8,
+        "size": 72,
         "length": 1,
         "capacity": 1,
         "members": [
           {
             "staticSize": 24,
             "exclusiveSize": 24,
+            "size": 64,
             "length": 5,
             "capacity": 5
           }
diff --git a/test/integration/std_pair.toml b/test/integration/std_pair.toml
index c60e784..1780951 100644
--- a/test/integration/std_pair.toml
+++ b/test/integration/std_pair.toml
@@ -14,9 +14,9 @@ includes = ["vector", "utility", "cstdint"]
     ]
     '''
     expect_json_v2 = '''[
-      {"staticSize": 16, "exclusiveSize": 0, "members": [
-        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8},
-        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8}
+      {"staticSize": 16, "exclusiveSize": 0, "size": 16, "members": [
+        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8, "size": 8},
+        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8, "size": 8}
       ]}
     ]'''
   [cases.uint64_uint32]
@@ -34,9 +34,9 @@ includes = ["vector", "utility", "cstdint"]
     ]
     '''
     expect_json_v2 = '''[
-      {"staticSize": 16, "exclusiveSize": 4, "members": [
-        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8},
-        {"typeNames": ["uint32_t"], "staticSize": 4, "exclusiveSize": 4}
+      {"staticSize": 16, "exclusiveSize": 4, "size": 16, "members": [
+        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8, "size": 8},
+        {"typeNames": ["uint32_t"], "staticSize": 4, "exclusiveSize": 4, "size": 4}
       ]}
     ]'''
 
@@ -48,9 +48,9 @@ includes = ["vector", "utility", "cstdint"]
     param_types = ["std::pair<uint64_t, uint64_t*>&"]
     setup = "return {{0, nullptr}};"
     expect_json_v2 = '''[
-      {"staticSize": 16, "exclusiveSize": 0, "members": [
-        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8},
-        {"typeNames": ["uintptr_t (stubbed)"], "staticSize": 8, "exclusiveSize": 8}
+      {"staticSize": 16, "exclusiveSize": 0, "size": 16, "members": [
+        {"typeNames": ["uint64_t"], "staticSize": 8, "exclusiveSize": 8, "size": 8},
+        {"typeNames": ["uintptr_t (stubbed)"], "staticSize": 8, "exclusiveSize": 8, "size": 8}
       ]}
     ]'''
 
@@ -82,8 +82,8 @@ includes = ["vector", "utility", "cstdint"]
     ]
     '''
     expect_json_v2 = '''[
-      {"staticSize": 48, "exclusiveSize": 0, "members": [
-        {"typeNames": ["std::vector<uint64_t, std::allocator<uint64_t>>"], "staticSize": 24, "exclusiveSize": 24},
-        {"typeNames": ["std::vector<uint64_t, std::allocator<uint64_t>>"], "staticSize": 24, "exclusiveSize": 24}
+      {"staticSize": 48, "exclusiveSize": 0, "size": 104, "members": [
+        {"typeNames": ["std::vector<uint64_t, std::allocator<uint64_t>>"], "staticSize": 24, "exclusiveSize": 24, "size": 48},
+        {"typeNames": ["std::vector<uint64_t, std::allocator<uint64_t>>"], "staticSize": 24, "exclusiveSize": 24, "size": 56}
       ]}
     ]'''
diff --git a/test/integration/std_set_custom_comparator.toml b/test/integration/std_set_custom_comparator.toml
index 66031f7..49d07f8 100644
--- a/test/integration/std_set_custom_comparator.toml
+++ b/test/integration/std_set_custom_comparator.toml
@@ -65,9 +65,10 @@ includes = ["set", "functional"]
     expect_json_v2 = '''[{
       "staticSize":8184,
       "exclusiveSize": 0,
+      "size": 9144,
       "members":[
-        {"name":"m1", "staticSize":48, "exclusiveSize": 156, "length": 3, "capacity": 3},
-        {"name":"m2", "staticSize":48, "exclusiveSize": 228, "length": 5, "capacity": 5},
-        {"name":"m3", "staticSize":48, "exclusiveSize": 300, "length": 7, "capacity": 7},
-        {"name":"m4", "staticSize":8040, "exclusiveSize": 8364, "length": 9, "capacity": 9}
+        {"name":"m1", "staticSize":48, "exclusiveSize": 156, "size": 168, "length": 3, "capacity": 3},
+        {"name":"m2", "staticSize":48, "exclusiveSize": 228, "size": 248, "length": 5, "capacity": 5},
+        {"name":"m3", "staticSize":48, "exclusiveSize": 300, "size": 328, "length": 7, "capacity": 7},
+        {"name":"m4", "staticSize":8040, "exclusiveSize": 8364, "size": 8400, "length": 9, "capacity": 9}
       ]}]'''
diff --git a/test/integration/std_smart_ptr.toml b/test/integration/std_smart_ptr.toml
index 8fc66bf..dfb86ae 100644
--- a/test/integration/std_smart_ptr.toml
+++ b/test/integration/std_smart_ptr.toml
@@ -26,6 +26,7 @@ definitions = '''
       {
         "staticSize": 8,
         "exclusiveSize": 8,
+        "size": 8,
         "length": 0,
         "capacity": 1
       }
@@ -50,6 +51,7 @@ definitions = '''
       {
         "staticSize": 8,
         "exclusiveSize": 8,
+        "size": 16,
         "length": 1,
         "capacity": 1
       }
@@ -74,6 +76,7 @@ definitions = '''
       {
         "staticSize": 8,
         "exclusiveSize": 8,
+        "size": 8,
         "length": 0,
         "capacity": 1
       }
@@ -106,6 +109,7 @@ definitions = '''
       {
         "staticSize": 8,
         "exclusiveSize": 8,
+        "size": 72,
         "length": 1,
         "capacity": 1,
         "members": [
@@ -134,7 +138,8 @@ definitions = '''
     [
       {
         "staticSize": 16,
-        "exclusiveSize": 16
+        "exclusiveSize": 16,
+        "size": 16
       }
     ]
     '''
@@ -153,7 +158,8 @@ definitions = '''
     [
       {
         "staticSize": 16,
-        "exclusiveSize": 16
+        "exclusiveSize": 16,
+        "size": 16
       }
     ]
     '''
@@ -176,6 +182,7 @@ definitions = '''
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 16,
         "length": 0,
         "capacity": 1
       }
@@ -200,6 +207,7 @@ definitions = '''
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 24,
         "length": 1,
         "capacity": 1,
         "members": [
@@ -227,6 +235,7 @@ definitions = '''
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 16,
         "length": 0,
         "capacity": 1
       }
@@ -257,12 +266,14 @@ definitions = '''
       {
         "staticSize": 16,
         "exclusiveSize": 16,
+        "size": 80,
         "length": 1,
         "capacity": 1,
         "members": [
           {
             "staticSize": 24,
             "exclusiveSize": 24,
+            "size": 64,
             "length": 5,
             "capacity": 5
           }
@@ -278,7 +289,8 @@ definitions = '''
       {
         "staticSize": 16,
         "dynamicSize": 0,
-        "exclusiveSize": 16
+        "exclusiveSize": 16,
+        "size": 16
       }
     ]
     '''
@@ -290,7 +302,8 @@ definitions = '''
       {
         "staticSize": 16,
         "dynamicSize": 0,
-        "exclusiveSize": 16
+        "exclusiveSize": 16,
+        "size": 16
       }
     ]
     '''
@@ -307,7 +320,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_void_empty]
     param_types = ["std::weak_ptr<void>&"]
     setup = "return std::weak_ptr<void>();"
@@ -321,7 +334,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_present]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
     setup = '''
@@ -339,7 +352,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_expired]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
     setup = '''
@@ -357,7 +370,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_present_chase]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
     cli_options = ["-fchase-raw-pointers"]
@@ -376,7 +389,7 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_expired_chase]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
     cli_options = ["-fchase-raw-pointers"]
@@ -393,4 +406,4 @@ definitions = '''
       }
     ]
     '''
-    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]'''
+    expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
diff --git a/test/integration/std_unordered_map_custom_operator.toml b/test/integration/std_unordered_map_custom_operator.toml
index 3dcf55d..ea78bff 100644
--- a/test/integration/std_unordered_map_custom_operator.toml
+++ b/test/integration/std_unordered_map_custom_operator.toml
@@ -66,9 +66,11 @@ includes = ["unordered_map"]
       ]}]'''
     expect_json_v2 = '''[{
       "staticSize":480,
+      "exclusiveSize": 0,
+      "size": 1952,
       "members":[
-        {"name":"m1", "staticSize":56, "exclusiveSize":220, "length":3, "capacity":3},
-        {"name":"m2", "staticSize":120, "exclusiveSize":324, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":120, "exclusiveSize":364, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":184, "exclusiveSize":468, "length":9, "capacity":9}
+        {"name":"m1", "staticSize":56, "exclusiveSize":220, "size":292, "length":3, "capacity":3},
+        {"name":"m2", "staticSize":120, "exclusiveSize":324, "size":444,  "length":5, "capacity":5},
+        {"name":"m3", "staticSize":120, "exclusiveSize":364, "size":532,  "length":7, "capacity":7},
+        {"name":"m4", "staticSize":184, "exclusiveSize":468, "size":684,  "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_unordered_multimap_custom_operator.toml b/test/integration/std_unordered_multimap_custom_operator.toml
index 37fe213..f36a296 100644
--- a/test/integration/std_unordered_multimap_custom_operator.toml
+++ b/test/integration/std_unordered_multimap_custom_operator.toml
@@ -66,9 +66,11 @@ includes = ["unordered_map"]
       ]}]'''
     expect_json_v2 = '''[{
       "staticSize":480,
+      "exclusiveSize":0,
+      "size":1952,
       "members":[
-        {"name":"m1", "staticSize":56, "exclusiveSize":220, "length":3, "capacity":3},
-        {"name":"m2", "staticSize":120, "exclusiveSize":324, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":120, "exclusiveSize":364, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":184, "exclusiveSize":468, "length":9, "capacity":9}
+        {"name":"m1", "staticSize":56, "exclusiveSize":220, "size":292, "length":3, "capacity":3},
+        {"name":"m2", "staticSize":120, "exclusiveSize":324, "size":444,  "length":5, "capacity":5},
+        {"name":"m3", "staticSize":120, "exclusiveSize":364, "size":532,  "length":7, "capacity":7},
+        {"name":"m4", "staticSize":184, "exclusiveSize":468, "size":684,  "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_unordered_multiset_custom_operator.toml b/test/integration/std_unordered_multiset_custom_operator.toml
index ef08dcb..d9ce1cc 100644
--- a/test/integration/std_unordered_multiset_custom_operator.toml
+++ b/test/integration/std_unordered_multiset_custom_operator.toml
@@ -66,9 +66,11 @@ includes = ["unordered_set"]
       ]}]'''
     expect_json_v2 = '''[{
       "staticSize":480,
+      "exclusiveSize":0,
+      "size":1472,
       "members":[
-        {"name":"m1", "staticSize":56, "exclusiveSize":220, "length":3, "capacity":3},
-        {"name":"m2", "staticSize":120, "exclusiveSize":324, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":120, "exclusiveSize":364, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":184, "exclusiveSize":468, "length":9, "capacity":9}
+        {"name":"m1", "staticSize":56, "exclusiveSize":220, "size":232, "length":3, "capacity":3},
+        {"name":"m2", "staticSize":120, "exclusiveSize":324, "size":344, "length":5, "capacity":5},
+        {"name":"m3", "staticSize":120, "exclusiveSize":364, "size":392, "length":7, "capacity":7},
+        {"name":"m4", "staticSize":184, "exclusiveSize":468, "size":504, "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_unordered_set_custom_operator.toml b/test/integration/std_unordered_set_custom_operator.toml
index 3c01f6c..797c5ea 100644
--- a/test/integration/std_unordered_set_custom_operator.toml
+++ b/test/integration/std_unordered_set_custom_operator.toml
@@ -66,9 +66,11 @@ includes = ["unordered_set"]
       ]}]'''
     expect_json_v2 = '''[{
       "staticSize":480,
+      "exclusiveSize":0,
+      "size":1472,
       "members":[
-        {"name":"m1", "staticSize":56, "exclusiveSize":220, "length":3, "capacity":3},
-        {"name":"m2", "staticSize":120, "exclusiveSize":324, "length":5, "capacity":5},
-        {"name":"m3", "staticSize":120, "exclusiveSize":364, "length":7, "capacity":7},
-        {"name":"m4", "staticSize":184, "exclusiveSize":468, "length":9, "capacity":9}
+        {"name":"m1", "staticSize":56, "exclusiveSize":220, "size":232,"length":3, "capacity":3},
+        {"name":"m2", "staticSize":120, "exclusiveSize":324, "size":344, "length":5, "capacity":5},
+        {"name":"m3", "staticSize":120, "exclusiveSize":364, "size":392, "length":7, "capacity":7},
+        {"name":"m4", "staticSize":184, "exclusiveSize":468, "size":504, "length":9, "capacity":9}
       ]}]'''
diff --git a/test/integration/std_vector.toml b/test/integration/std_vector.toml
index d466891..20e00d8 100644
--- a/test/integration/std_vector.toml
+++ b/test/integration/std_vector.toml
@@ -13,24 +13,24 @@ definitions = '''
     param_types = ["const std::vector<int>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size": 24, "length":0, "capacity":0, "members":[]}]'
   [cases.int_some]
     param_types = ["const std::vector<int>&"]
     setup = "return {{1,2,3}};"
     expect_json = '[{"staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":36,"length":3, "capacity":3, "members":[
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4}
     ]}]'''
   [cases.struct_some]
     param_types = ["const std::vector<SimpleStruct>&"]
     setup = "return {{{}, {}, {}}};"
     expect_json = '[{"staticSize":24, "dynamicSize":48, "length":3, "capacity":3, "elementStaticSize":16}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[
-      {"staticSize":16, "exclusiveSize":3},
-      {"staticSize":16, "exclusiveSize":3},
-      {"staticSize":16, "exclusiveSize":3}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":72, "length":3, "capacity":3, "members":[
+      {"staticSize":16, "exclusiveSize":3, "size":16},
+      {"staticSize":16, "exclusiveSize":3, "size":16},
+      {"staticSize":16, "exclusiveSize":3, "size":16}
     ]}]'''
   [cases.bool_empty]
     skip = true # https://github.com/facebookexperimental/object-introspection/issues/14
@@ -46,7 +46,7 @@ definitions = '''
     param_types = ["const std::vector<std::vector<int>>&"]
     setup = "return {};"
     expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":24}]'
-    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "size":24, "length":0, "capacity":0, "members":[]}]'
   [cases.vector_int_some]
     param_types = ["const std::vector<std::vector<int>>&"]
     setup = "return {{{1,2,3},{4},{5,6}}};"
@@ -62,10 +62,10 @@ definitions = '''
         {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4},
         {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
       ]}]'''
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[
-      {"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[]},
-      {"staticSize":24, "exclusiveSize":24, "length":1, "capacity": 1, "members":[]},
-      {"staticSize":24, "exclusiveSize":24, "length":2, "capacity": 2, "members":[]}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":120, "length":3, "capacity": 3, "members":[
+      {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity": 3, "members":[]},
+      {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity": 1, "members":[]},
+      {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity": 2, "members":[]}
     ]}]'''
   [cases.reserve]
     param_types = ["const std::vector<int>&"]
@@ -75,8 +75,8 @@ definitions = '''
       return ret;
     '''
     expect_json = '[{"staticSize":24, "dynamicSize":40, "exclusiveSize":64, "length":3, "capacity":10, "elementStaticSize":4}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":52, "length":3, "capacity":10, "members":[
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4},
-      {"staticSize":4, "exclusiveSize":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":52, "size":64, "length":3, "capacity":10, "members":[
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4},
+      {"staticSize":4, "exclusiveSize":4, "size":4}
     ]}]'''
diff --git a/test/integration/std_vector_del_allocator.toml b/test/integration/std_vector_del_allocator.toml
index 934bd34..32edcc2 100644
--- a/test/integration/std_vector_del_allocator.toml
+++ b/test/integration/std_vector_del_allocator.toml
@@ -60,18 +60,19 @@ includes = ["vector"]
     expect_json_v2 = '''[{
       "staticSize":48,
       "exclusiveSize":0,
+      "size":60,
       "members":[
-        {"name":"v1", "staticSize":24, "exclusiveSize":24, "length":1, "capacity":1, "members":[
-          {"name":"[]", "staticSize":4, "exclusiveSize":0, "members":[
-            {"name":"a", "staticSize":4, "exclusiveSize":4, "members":[]}
+        {"name":"v1", "staticSize":24, "exclusiveSize":24, "size": 28, "length":1, "capacity":1, "members":[
+          {"name":"[]", "staticSize":4, "exclusiveSize":0, "size": 4, "members":[
+            {"name":"a", "staticSize":4, "exclusiveSize":4, "size": 4, "members":[]}
           ]}
         ]},
-        {"name":"v2", "staticSize":24, "exclusiveSize":24, "length":2, "capacity":2, "members":[
-          {"name":"[]", "staticSize":4, "exclusiveSize":0, "members":[
-            {"name":"b", "staticSize":4, "exclusiveSize":4, "members":[]}
+        {"name":"v2", "staticSize":24, "exclusiveSize":24, "size": 32, "length":2, "capacity":2, "members":[
+          {"name":"[]", "staticSize":4, "exclusiveSize":0, "size": 4, "members":[
+            {"name":"b", "staticSize":4, "exclusiveSize":4, "size": 4, "members":[]}
           ]},
-          {"name":"[]", "staticSize":4, "exclusiveSize":0, "members":[
-            {"name":"b", "staticSize":4, "exclusiveSize":4, "members":[]}
+          {"name":"[]", "staticSize":4, "exclusiveSize":0, "size": 4, "members":[
+            {"name":"b", "staticSize":4, "exclusiveSize":4, "size": 4, "members":[]}
           ]}
         ]}
       ]
diff --git a/test/integration/templates.toml b/test/integration/templates.toml
index b02cfc8..8be4dc9 100644
--- a/test/integration/templates.toml
+++ b/test/integration/templates.toml
@@ -53,10 +53,12 @@ definitions = '''
       "typeName":"ns_templates::TemplatedClass1<std::vector<int, std::allocator<int> > >",
       "staticSize":24,
       "exclusiveSize":0,
+      "size":24,
       "members":[{
         "typeName":"std::vector<int32_t, std::allocator<int32_t>>",
         "staticSize":24,
         "exclusiveSize":24,
+        "size":24,
         "length":0,
         "capacity":0
     }]}]'''
@@ -90,9 +92,11 @@ definitions = '''
       "typeName":"ns_templates::TemplatedClassVal<3>",
       "staticSize":12,
       "exclusiveSize":0,
+      "size":12,
       "members":[{
         "staticSize":12,
         "exclusiveSize":0,
+        "size":12,
         "length":3,
         "capacity":3
     }]}]'''
diff --git a/test/integration/thrift_unions.toml b/test/integration/thrift_unions.toml
index 5f0ba1f..651869b 100644
--- a/test/integration/thrift_unions.toml
+++ b/test/integration/thrift_unions.toml
@@ -37,9 +37,10 @@ namespace cpp2 {
     expect_json_v2 = '''[{
       "staticSize":8,
       "exclusiveSize":0,
+      "size":8,
       "members":[
-        {"typeNames":["storage_type"], "name":"value_", "staticSize":4, "exclusiveSize":4},
-        {"typeNames":["underlying_type_t<cpp2::StaticUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4}
+        {"typeNames":["storage_type"], "name":"value_", "staticSize":4, "exclusiveSize":4, "size":4},
+        {"typeNames":["underlying_type_t<cpp2::StaticUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}]'''
   [cases.dynamic_int]
     param_types = ["const cpp2::DynamicUnion&"]
@@ -58,9 +59,10 @@ namespace cpp2 {
     expect_json_v2 = '''[{
       "staticSize":32,
       "exclusiveSize":4,
+      "size":32,
       "members":[
-        {"typeNames":["storage_type"], "name":"value_", "staticSize":24, "exclusiveSize":24},
-        {"typeNames":["underlying_type_t<cpp2::DynamicUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4}
+        {"typeNames":["storage_type"], "name":"value_", "staticSize":24, "exclusiveSize":24, "size":24},
+        {"typeNames":["underlying_type_t<cpp2::DynamicUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}]'''
   [cases.dynamic_vec]
     param_types = ["const cpp2::DynamicUnion&"]
@@ -79,7 +81,8 @@ namespace cpp2 {
     expect_json_v2 = '''[{
       "staticSize":32,
       "exclusiveSize":4,
+      "size":32,
       "members":[
-        {"typeNames":["storage_type"], "name":"value_", "staticSize":24, "exclusiveSize":24},
-        {"typeNames":["underlying_type_t<cpp2::DynamicUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4}
+        {"typeNames":["storage_type"], "name":"value_", "staticSize":24, "exclusiveSize":24, "size":24},
+        {"typeNames":["underlying_type_t<cpp2::DynamicUnion::Type>", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4, "size":4}
       ]}]'''
diff --git a/test/integration/typedefed_parent.toml b/test/integration/typedefed_parent.toml
index 72c4a57..25631f9 100644
--- a/test/integration/typedefed_parent.toml
+++ b/test/integration/typedefed_parent.toml
@@ -30,9 +30,10 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":48,
       "exclusiveSize":0,
+      "size":80,
       "members":[
-        {"name":"a", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3},
-        {"name":"b", "staticSize":24, "exclusiveSize":24, "length":5, "capacity":5}
+        {"name":"a", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "size":36},
+        {"name":"b", "staticSize":24, "exclusiveSize":24, "length":5, "capacity":5, "size":44}
       ]}]'''
   [cases.multilevel_typedef_parent]
     param_types = ["const Bar_2&"]
@@ -50,7 +51,8 @@ definitions = '''
     expect_json_v2 = '''[{
       "staticSize":48,
       "exclusiveSize":0,
+      "size":80,
       "members":[
-        {"name":"a", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3},
-        {"name":"c", "staticSize":24, "exclusiveSize":24, "length":5, "capacity":5}
+        {"name":"a", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "size":36},
+        {"name":"c", "staticSize":24, "exclusiveSize":24, "length":5, "capacity":5, "size":44}
       ]}]'''
diff --git a/test/integration/unions.toml b/test/integration/unions.toml
index 42341de..66df45e 100644
--- a/test/integration/unions.toml
+++ b/test/integration/unions.toml
@@ -45,17 +45,17 @@ definitions = '''
     param_types = ["const MyUnion&"]
     setup = "return 123;"
     expect_json = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}]'
-    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]}]'
   [cases.vector]
     param_types = ["const MyUnion&"]
     setup = "return std::vector{1,2,3};"
     expect_json = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}]'
-    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]}]'
   [cases.unordered_map]
     param_types = ["const MyUnion&"]
     setup = 'return std::unordered_map<std::string, std::string>{{"a", "b"}, {"c","d"}};'
     expect_json = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}]'
-    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}]'
+    expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]}]'
 
   [cases.alignment]
     # Wrap the union in a pair as a way of inferring its alignment
@@ -67,9 +67,9 @@ definitions = '''
         {"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}
       ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[
-        {"staticSize":1, "dynamicSize":0, "exclusiveSize":1},
-        {"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}
+      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "size":64, "members":[
+        {"staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1},
+        {"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]}
       ]}]'''
 
   [cases.tagged_int]
@@ -81,9 +81,9 @@ definitions = '''
         {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "NOT":"members"}
       ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[
-        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]},
-        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "members":[]}
+      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "size":64, "members":[
+        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]},
+        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1, "members":[]}
       ]}]'''
   [cases.tagged_vector]
     param_types = ["const TaggedUnion&"]
@@ -94,9 +94,9 @@ definitions = '''
         {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "NOT":"members"}
       ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[
-        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]},
-        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "members":[]}
+      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "size":64, "members":[
+        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]},
+        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1, "members":[]}
       ]}]'''
   [cases.tagged_unordered_map]
     param_types = ["const TaggedUnion&"]
@@ -107,7 +107,7 @@ definitions = '''
         {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "NOT":"members"}
       ]}]'''
     expect_json_v2 = '''[
-      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[
-        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]},
-        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "members":[]}
+      {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "size":64, "members":[
+        {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "size":56, "members":[]},
+        {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "size":1, "members":[]}
       ]}]'''

From 2f40d2a47e3d24290e346b3552f02458a40c8fa0 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 3 Jan 2024 17:30:31 +0000
Subject: [PATCH 106/188] oilgen: add to integration test framework

TODO: Replace the references to local paths.

oilgen (the basis of Ahead Of Time compilation for OIL) has never been passed
through our large test suite and has instead had more focused testing and large
examples. When consuming DWARF information in a similar fashion to JIT OIL this
was okay, but with the new Clang AST based mechanism it means we have very
little coverage.

This change adds an oilgen test for every test case that has an oil test.
Relying on the build system to create the test target as before would make it
difficult to have failing tests, so we move the build into the integration test
runner. This involves:
1. Writing the input source to a file.
2. Consuming it with oilgen to get the implementation object file.
3. Compiling the input source and linking it with this file.
4. Running the newly created target.

This approach can give the full error message at any stage that fails and will
fail the test appropriately. The downside is the build system integration is
more difficult, as we need the correct compiler flags for the target and to use
the correct compiler. It would be very tricky to replicate this in a build
system that's not CMake, so we will likely only run these tests in open source.

Test plan:
- CI
---
 oi/type_graph/ClangTypeParser.cpp  |   4 +-
 oi/type_graph/Types.h              |   2 +-
 test/integration/CMakeLists.txt    |  11 +-
 test/integration/gen_tests.py      | 153 +++++++++++++++++++++------
 test/integration/runner_common.cpp | 163 +++++++++++++++++++++++++++++
 test/integration/runner_common.h   |  14 +++
 6 files changed, 311 insertions(+), 36 deletions(-)

diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index 24d5835..dfa3a07 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -350,6 +350,7 @@ Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
     case clang::BuiltinType::WChar_U:
       return makeType<Primitive>(ty, Primitive::Kind::UInt32);
 
+    case clang::BuiltinType::Char8:
     case clang::BuiltinType::Char_S:
     case clang::BuiltinType::SChar:
       return makeType<Primitive>(ty, Primitive::Kind::Int8);
@@ -380,8 +381,9 @@ Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
     case clang::BuiltinType::Float:
       return makeType<Primitive>(ty, Primitive::Kind::Float32);
     case clang::BuiltinType::Double:
-    case clang::BuiltinType::LongDouble:
       return makeType<Primitive>(ty, Primitive::Kind::Float64);
+    case clang::BuiltinType::LongDouble:
+      return makeType<Primitive>(ty, Primitive::Kind::Float128);
 
     case clang::BuiltinType::UInt128:
     case clang::BuiltinType::Int128:
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index 1d5dd79..81cab11 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -592,7 +592,7 @@ class Primitive : public Type {
     Float32,
     Float64,
     Float80,   // TODO worth including?
-    Float128,  // TODO can we generate this?
+    Float128,
     Bool,
 
     StubbedPointer,
diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 4d235b5..b4fe3de 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -58,12 +58,18 @@ target_link_libraries(integration_test_runner PRIVATE
   GTest::gmock_main
   Boost::headers
   ${Boost_LIBRARIES}
+  range-v3
   toml
 )
 target_compile_definitions(integration_test_runner PRIVATE
-  TARGET_EXE_PATH="${CMAKE_CURRENT_BINARY_DIR}/integration_test_target"
+  TARGET_EXE_PATH="$<TARGET_FILE:integration_test_target>"
   OID_EXE_PATH="$<TARGET_FILE:oid>"
-  CONFIG_FILE_PATH="${CMAKE_BINARY_DIR}/testing.oid.toml")
+  OILGEN_EXE_PATH="$<TARGET_FILE:oilgen>"
+  CONFIG_FILE_PATH="${CMAKE_BINARY_DIR}/testing.oid.toml"
+
+  CXX="${CMAKE_CXX_COMPILER}"
+  TARGET_INCLUDE_DIRECTORIES="$<JOIN:$<TARGET_PROPERTY:integration_test_target,INCLUDE_DIRECTORIES>,:>"
+)
 
 if (${THRIFT_FOUND})
   foreach(THRIFT_TEST IN LISTS THRIFT_TESTS)
@@ -85,6 +91,7 @@ if (${THRIFT_FOUND})
 
     add_custom_target(integration_test_thrift_sources_${THRIFT_TEST} DEPENDS ${THRIFT_TYPES_H})
     add_dependencies(integration_test_target integration_test_thrift_sources_${THRIFT_TEST})
+    add_dependencies(integration_test_runner integration_test_thrift_sources_${THRIFT_TEST})
     target_sources(integration_test_target PRIVATE ${THRIFT_DATA_CPP})
   endforeach()
 
diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py
index b72e371..d9fe054 100644
--- a/test/integration/gen_tests.py
+++ b/test/integration/gen_tests.py
@@ -51,6 +51,41 @@ def add_headers(f, custom_headers, thrift_headers):
         f.write(f'#include "{header}"\n')
 
 
+def add_test_getters(f, case_name, case):
+    param_types = ", ".join(
+        f"std::remove_cvref_t<{param}>" for param in case["param_types"]
+    )
+    if "arg_types" in case:
+        arg_types = ", ".join(case["arg_types"])
+    else:
+        arg_types = param_types
+
+    f.write(
+        f"\n"
+        f"  std::tuple<{arg_types}> get_{case_name}() {{\n"
+        f'{case["setup"]}\n'
+        f"  }}\n"
+    )
+
+
+def get_param_str(param, i):
+    if "]" in param:
+        # Array param
+
+        if ")" in param:
+            # "int(&)[5]" ->  "int (&a0)[5]"
+            start, end = param.split(")")
+            return f"{start}a{i}){end}"
+
+        # "int[5]" -> "int a0[5]"
+        # "int[5][10]" -> "int a0[5][10]"
+        type_name, array_size = param.split("[", 1)
+        return f"{type_name} a{i}[{array_size}"
+
+    # Non-array param, e.g. "int&" -> "int& a0"
+    return f"{param} a{i}"
+
+
 def add_test_setup(f, config):
     ns = get_namespace(config["suite"])
     # fmt: off
@@ -65,23 +100,6 @@ def add_test_setup(f, config):
     )
     # fmt: on
 
-    def get_param_str(param, i):
-        if "]" in param:
-            # Array param
-
-            if ")" in param:
-                # "int(&)[5]" ->  "int (&a0)[5]"
-                start, end = param.split(")")
-                return f"{start}a{i}){end}"
-
-            # "int[5]" -> "int a0[5]"
-            # "int[5][10]" -> "int a0[5][10]"
-            type_name, array_size = param.split("[", 1)
-            return f"{type_name} a{i}[{array_size}"
-
-        # Non-array param, e.g. "int&" -> "int& a0"
-        return f"{param} a{i}"
-
     def define_traceable_func(name, params, body):
         return (
             f"\n"
@@ -99,21 +117,7 @@ def add_test_setup(f, config):
             # target func for it
             continue
 
-        # generate getter for an object of this type
-        param_types = ", ".join(
-            f"std::remove_cvref_t<{param}>" for param in case["param_types"]
-        )
-        if "arg_types" in case:
-            arg_types = ", ".join(case["arg_types"])
-        else:
-            arg_types = param_types
-
-        f.write(
-            f"\n"
-            f"  std::tuple<{arg_types}> get_{case_name}() {{\n"
-            f'{case["setup"]}\n'
-            f"  }}\n"
-        )
+        add_test_getters(f, case_name, case)
 
         # generate oid and oil targets
         params_str = ", ".join(
@@ -266,6 +270,7 @@ def add_tests(f, config):
     for case_name, case in config["cases"].items():
         add_oid_integration_test(f, config, case_name, case)
         add_oil_integration_test(f, config, case_name, case)
+        add_oilgen_integration_test(f, config, case_name, case)
 
 
 def add_oid_integration_test(f, config, case_name, case):
@@ -400,6 +405,90 @@ def add_oil_integration_test(f, config, case_name, case):
     f.write(f"}}\n")
 
 
+def add_oilgen_integration_test(f, config, case_name, case):
+    case_str = get_case_name(config["suite"], case_name)
+    exit_code = case.get("expect_oil_exit_code", 0)
+
+    if "oil_disable" in case or "target_function" in case:
+        return
+
+    config_prefix = case.get("config_prefix", "")
+    config_suffix = case.get("config_suffix", "")
+
+    f.write(
+        f"\n"
+        f"TEST_F(OilgenIntegration, {case_str}) {{\n"
+        f"{generate_skip(case, 'oil')}"
+    )
+
+    f.write('  constexpr std::string_view targetSrc = R"--(')
+    headers = set(config.get("includes", []))
+    thrift_headers = [f"thrift/annotation/gen-cpp2/{config['suite']}_types.h"] if is_thrift_test(config) else []
+    add_headers(f, sorted(headers), thrift_headers)
+
+    f.write(
+        f"\n"
+        f'{config.get("raw_definitions", "")}\n'
+        f"#pragma clang diagnostic push\n"
+        f'#pragma clang diagnostic ignored "-Wunused-private-field"\n'
+        f'{config.get("definitions", "")}\n'
+        f"#pragma clang diagnostic pop\n"
+    )
+    add_test_getters(f, case_name, case)
+
+    main = "int main() {\n"
+    main += "  auto pr = oi::exporters::Json(std::cout);\n"
+    main += "  pr.setPretty(true);\n"
+    main += f"  auto val = get_{case_name}();\n"
+    for i in range(len(case["param_types"])):
+        main += f"  auto ret{i} = oi::result::SizedResult(oi::introspect"
+        if "arg_types" in case:
+            main += f"<std::decay_t<{case['param_types'][i]}>>"
+        main += f"(std::get<{i}>(val)));\n"
+        main += f"  pr.print(ret{i});\n"
+    main += "}\n"
+
+    f.write(main)
+    f.write(')--";\n')
+
+    f.write(
+        f'  std::string configPrefix = R"--({config_prefix})--";\n'
+        f'  std::string configSuffix = R"--({config_suffix})--";\n'
+        f"  ba::io_context ctx;\n"
+        f"  auto target = runOilgenTarget({{\n"
+        f"    .ctx = ctx,\n"
+        f"    .targetSrc = targetSrc,\n"
+        f"  }}, std::move(configPrefix), std::move(configSuffix));\n\n"
+        f"  ASSERT_EQ(exit_code(target), {exit_code});\n"
+    )
+
+    key = "expect_json"
+    if "expect_json_v2" in case:
+        key = "expect_json_v2"
+    if key in case:
+        try:
+            json.loads(case[key])
+        except json.decoder.JSONDecodeError as error:
+            print(
+                f"\x1b[31m`expect_json` value for test case {config['suite']}.{case_name} was invalid JSON: {error}\x1b[0m",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+        f.write(
+            f"\n"
+            f"  std::stringstream expected_json_ss;\n"
+            f'  expected_json_ss << R"--({case[key]})--";\n'
+            f"  auto result_json_ss = std::stringstream(stdout_);\n"
+            f"  bpt::ptree expected_json, actual_json;\n"
+            f"  bpt::read_json(expected_json_ss, expected_json);\n"
+            f"  bpt::read_json(result_json_ss, actual_json);\n"
+            f"  compare_json(expected_json, actual_json);\n"
+        )
+
+    f.write(f"}}\n")
+
+
 def generate_skip(case, specific):
     possibly_skip = ""
     skip_reason = case.get("skip", False)
diff --git a/test/integration/runner_common.cpp b/test/integration/runner_common.cpp
index 3c8525a..c843c5d 100644
--- a/test/integration/runner_common.cpp
+++ b/test/integration/runner_common.cpp
@@ -11,6 +11,7 @@
 #include <iostream>
 #include <string>
 #include <utility>
+#include <range/v3/view/split.hpp>
 
 #include "oi/OIOpts.h"
 #include "oi/support/Toml.h"
@@ -447,3 +448,165 @@ Proc OilIntegration::runOilTarget(OilOpts opts,
               std::move(std_out),
               std::move(std_err)};
 }
+
+std::string OilgenIntegration::TmpDirStr() {
+  return std::string("/tmp/oilgen-integration-XXXXXX");
+}
+
+Proc OilgenIntegration::runOilgenTarget(OilgenOpts opts,
+                                        std::string configPrefix,
+                                        std::string configSuffix) {
+  // Run an oilgen test in three stages.
+  // 1. Generate the OIL implementation .o from the input source.
+  // 2. Compile the input source and link to the implementation.
+  // 3. Run the now complete target.
+
+  auto srcPath = workingDir / "input_src.cpp";
+  {
+    std::ofstream file{srcPath, std::ios_base::app};
+    file << opts.targetSrc;
+  }
+
+  std::string generatorExe =
+      std::string(OILGEN_EXE_PATH) + " " + srcPath.native() + " --output=oil_generated.o --debug-level=3 ";
+  if (auto prefix = writeCustomConfig("prefix", configPrefix)) {
+    generatorExe += "--config-file ";
+    generatorExe += *prefix;
+    generatorExe += " ";
+  }
+  generatorExe += "--config-file ";
+  generatorExe += configFile;
+  if (auto suffix = writeCustomConfig("suffix", configSuffix)) {
+    generatorExe += " ";
+    generatorExe += "--config-file ";
+    generatorExe += *suffix;
+  }
+
+
+  // TODO: get this from the CMake arguments for integration_test_target.cpp
+  // somehow
+  // clang-format off
+  std::vector<std::string> clangArgs{
+      "-DOIL_AOT_COMPILATION=1",
+      "--std=c++20",
+      "-resource-dir",
+      "/usr/lib64/clang/15.0.7",
+      "-fpic",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../include/c++/12",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../include/c++/12/x86_64-redhat-linux",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../include/c++/12/backward",
+      "-isystem",
+      "/usr/lib64/clang/15.0.7/include",
+      "-isystem",
+      "/usr/local/include",
+      "-isystem",
+      "/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/../../../../x86_64-redhat-linux/include",
+      "-isystem",
+      "/include",
+      "-isystem",
+      "/usr/include",
+      "-isystem",
+      "/data/users/jakehillion/object-introspection-sl/build/_deps/folly-src",
+      "-fdebug-compilation-dir=/data/users/jakehillion/Downloads/libclang-oilgen-testing",
+      "-fgnuc-version=4.2.1",
+      "-fcxx-exceptions",
+      "-fexceptions",
+      "-faddrsig",
+      "-D__GCC_HAVE_DWARF2_CFI_ASM=1",
+  };
+  // clang-format on
+  for(auto&& rng : TARGET_INCLUDE_DIRECTORIES | ranges::views::split(':')) {
+    clangArgs.emplace_back("-I");
+    clangArgs.emplace_back(&*rng.begin(), ranges::distance(rng));
+  }
+
+  for (const auto& arg : clangArgs) {
+    generatorExe += " --extra-arg=";
+    generatorExe += arg;
+  }
+
+  if (verbose) {
+    std::cerr << "Running: " << generatorExe << std::endl;
+  }
+
+  bp::child generatorProc{generatorExe, opts.ctx};
+  generatorProc.wait();
+  if (generatorProc.exit_code() != 0)
+    throw std::runtime_error("generation failed!");
+
+  std::string compilerExe =
+      std::string(CXX) +
+      " input_src.cpp oil_generated.o -o generated_target "
+      "/data/users/jakehillion/object-introspection-sl/oi/IntrospectionResult.cpp "
+      "/data/users/jakehillion/object-introspection-sl/oi/exporters/ParsedData.cpp ";
+  for (const auto& arg : clangArgs) {
+    compilerExe += ' ';
+    compilerExe += arg;
+  }
+
+  if (verbose) {
+    std::cerr << "Running: " << compilerExe << std::endl;
+  }
+
+  bp::child compilerProc{compilerExe, opts.ctx};
+  compilerProc.wait();
+  if (compilerProc.exit_code() != 0)
+    throw std::runtime_error("compilation failed");
+
+  std::string targetExe = "./generated_target";
+  if (verbose) {
+    std::cerr << "Running: " << targetExe << std::endl;
+  }
+
+  // Use tee to write the output to files. If verbose is on, also redirect the
+  // output to stderr.
+  bp::async_pipe std_out_pipe(opts.ctx), std_err_pipe(opts.ctx);
+  bp::child std_out, std_err;
+  if (verbose) {
+    // clang-format off
+    std_out = bp::child(bp::search_path("tee"),
+                        (workingDir / "stdout").string(),
+                        bp::std_in < std_out_pipe,
+                        bp::std_out > stderr,
+                        opts.ctx);
+    std_err = bp::child(bp::search_path("tee"),
+                        (workingDir / "stderr").string(),
+                        bp::std_in < std_err_pipe,
+                        bp::std_out > stderr,
+                        opts.ctx);
+    // clang-format on
+  } else {
+    // clang-format off
+    std_out = bp::child(bp::search_path("tee"),
+                        (workingDir / "stdout").string(),
+                        bp::std_in < std_out_pipe,
+                        bp::std_out > bp::null,
+                        opts.ctx);
+    std_err = bp::child(bp::search_path("tee"),
+                        (workingDir / "stderr").string(),
+                        bp::std_in < std_err_pipe,
+                        bp::std_out > bp::null,
+                        opts.ctx);
+    // clang-format on
+  }
+
+  /* Spawn `oid` with tracing on and IOs redirected */
+  // clang-format off
+  bp::child targetProc(
+      targetExe,
+      bp::std_in  < bp::null,
+      bp::std_out > std_out_pipe,
+      bp::std_err > std_err_pipe,
+      opts.ctx);
+  // clang-format on
+
+  return Proc{
+      opts.ctx,
+      std::move(targetProc),
+      std::move(std_out),
+      std::move(std_err),
+  };
+}
diff --git a/test/integration/runner_common.h b/test/integration/runner_common.h
index d2b5e57..8c6c8eb 100644
--- a/test/integration/runner_common.h
+++ b/test/integration/runner_common.h
@@ -22,6 +22,11 @@ struct OilOpts {
   std::string targetArgs;
 };
 
+struct OilgenOpts {
+  boost::asio::io_context& ctx;
+  std::string_view targetSrc;
+};
+
 struct Proc {
   boost::asio::io_context& ctx;
   boost::process::child proc;
@@ -81,3 +86,12 @@ class OilIntegration : public IntegrationBase {
                     std::string configPrefix,
                     std::string configSuffix);
 };
+
+class OilgenIntegration : public IntegrationBase {
+ protected:
+  std::string TmpDirStr() override;
+
+  Proc runOilgenTarget(OilgenOpts opts,
+                       std::string configPrefix,
+                       std::string configSuffix);
+};

From 66a4ed28df30a8bc5663f011d0eada33b2587962 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 3 Jan 2024 17:30:31 +0000
Subject: [PATCH 107/188] clangparser: add support for parents

TODO: Check the assumption that a "base" always has a Type that can be cast to
RecordType.
TODO: Check the assumption that a "base" always has a Decl that can be cast to
CXXRecordDecl.

Add basic support for class parents. Focus purely on bases for now and ignore
vbases.

Test Plan:
- Tested with a simple example. Base containing a long and a class containing a
  float. Both fields appear in the final flattened code.
---
 oi/type_graph/ClangTypeParser.cpp | 32 ++++++++++++++++++++++++++++++-
 oi/type_graph/ClangTypeParser.h   |  2 ++
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index dfa3a07..6b36104 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -17,8 +17,10 @@
 
 #include <clang/AST/ASTContext.h>
 #include <clang/AST/Decl.h>
+#include <clang/AST/DeclCXX.h>
 #include <clang/AST/DeclTemplate.h>
 #include <clang/AST/QualTypeNames.h>
+#include <clang/AST/RecordLayout.h>
 #include <clang/AST/Type.h>
 #include <clang/Basic/DiagnosticSema.h>
 #include <clang/Sema/Sema.h>
@@ -196,7 +198,7 @@ Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
       ty, kind, std::move(name), std::move(fqName), size, virtuality);
 
   enumerateClassTemplateParams(ty, c.templateParams);
-  // enumerateClassParents(type, c.parents);
+  enumerateClassParents(ty, c.parents);
   enumerateClassMembers(ty, c.members);
   // enumerateClassFunctions(type, c.functions);
 
@@ -275,6 +277,34 @@ std::optional<TemplateParam> ClangTypeParser::enumerateTemplateTemplateParam(
   }
 }
 
+void ClangTypeParser::enumerateClassParents(const clang::RecordType& ty,
+                                            std::vector<Parent>& parents) {
+  assert(parents.empty());
+
+  auto* decl = ty.getDecl();
+  auto* cxxDecl = llvm::dyn_cast<clang::CXXRecordDecl>(decl);
+  if (cxxDecl == nullptr)
+    return;
+
+  const auto& layout = decl->getASTContext().getASTRecordLayout(decl);
+  for (const auto& base : cxxDecl->bases()) {
+    auto baseType = base.getType().getDesugaredType(decl->getASTContext());
+    const auto* baseRecordType = llvm::dyn_cast<clang::RecordType>(&*baseType);
+    if (baseRecordType == nullptr)
+      continue;
+
+    auto* baseDecl = baseRecordType->getDecl();
+    auto* baseCxxDecl = llvm::dyn_cast<clang::CXXRecordDecl>(baseDecl);
+    if (baseCxxDecl == nullptr)
+      continue;
+
+    auto offset = layout.getBaseClassOffset(baseCxxDecl).getQuantity();
+    auto& ptype = enumerateType(*baseType);
+    Parent p{ptype, static_cast<uint64_t>(offset * 8)};
+    parents.push_back(p);
+  }
+}
+
 void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
                                             std::vector<Member>& members) {
   assert(members.empty());
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
index 9b27d64..98f0f67 100644
--- a/oi/type_graph/ClangTypeParser.h
+++ b/oi/type_graph/ClangTypeParser.h
@@ -50,6 +50,7 @@ class Array;
 class Class;
 class Enum;
 class Member;
+struct Parent;
 class Primitive;
 class Reference;
 class Type;
@@ -114,6 +115,7 @@ class ClangTypeParser {
   std::optional<TemplateParam> enumerateTemplateTemplateParam(
       const clang::TemplateName&);
 
+  void enumerateClassParents(const clang::RecordType&, std::vector<Parent>&);
   void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
 
   ContainerInfo* getContainerInfo(const std::string& fqName) const;

From 213dba4325c1fedfd65973de6f04fb23beaed5b7 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 4 Jan 2024 11:53:19 +0000
Subject: [PATCH 108/188] oilgen: add to integration test framework

TODO: Replace the references to local paths.

oilgen (the basis of Ahead Of Time compilation for OIL) has never been passed
through our large test suite and has instead had more focused testing and large
examples. When consuming DWARF information in a similar fashion to JIT OIL this
was okay, but with the new Clang AST based mechanism it means we have very
little coverage.

This change adds an oilgen test for every test case that has an oil test.
Relying on the build system to create the test target as before would make it
difficult to have failing tests, so we move the build into the integration test
runner. This involves:
1. Writing the input source to a file.
2. Consuming it with oilgen to get the implementation object file.
3. Compiling the input source and linking it with this file.
4. Running the newly created target.

This approach can give the full error message at any stage that fails and will
fail the test appropriately. The downside is the build system integration is
more difficult, as we need the correct compiler flags for the target and to use
the correct compiler. It would be very tricky to replicate this in a build
system that's not CMake, so we will likely only run these tests in open source.

Test plan:
- CI
---
 .circleci/config.yml               |   2 +-
 oi/type_graph/ClangTypeParser.cpp  |   4 +-
 oi/type_graph/Types.h              |   2 +-
 test/integration/CMakeLists.txt    |  11 +-
 test/integration/gen_tests.py      | 153 ++++++++++++++++++++------
 test/integration/runner_common.cpp | 165 +++++++++++++++++++++++++++++
 test/integration/runner_common.h   |  14 +++
 7 files changed, 314 insertions(+), 37 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index b91cc2c..a6c6b8f 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -14,7 +14,7 @@ workflows:
           name: test-gcc
           requires:
             - build-gcc
-          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
+          exclude_regex: "OilgenIntegration\\..*|.*inheritance_polymorphic.*|.*arrays_member_int0"
       - coverage:
           name: coverage
           requires:
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index 24d5835..dfa3a07 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -350,6 +350,7 @@ Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
     case clang::BuiltinType::WChar_U:
       return makeType<Primitive>(ty, Primitive::Kind::UInt32);
 
+    case clang::BuiltinType::Char8:
     case clang::BuiltinType::Char_S:
     case clang::BuiltinType::SChar:
       return makeType<Primitive>(ty, Primitive::Kind::Int8);
@@ -380,8 +381,9 @@ Primitive& ClangTypeParser::enumeratePrimitive(const clang::BuiltinType& ty) {
     case clang::BuiltinType::Float:
       return makeType<Primitive>(ty, Primitive::Kind::Float32);
     case clang::BuiltinType::Double:
-    case clang::BuiltinType::LongDouble:
       return makeType<Primitive>(ty, Primitive::Kind::Float64);
+    case clang::BuiltinType::LongDouble:
+      return makeType<Primitive>(ty, Primitive::Kind::Float128);
 
     case clang::BuiltinType::UInt128:
     case clang::BuiltinType::Int128:
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index 1d5dd79..81cab11 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -592,7 +592,7 @@ class Primitive : public Type {
     Float32,
     Float64,
     Float80,   // TODO worth including?
-    Float128,  // TODO can we generate this?
+    Float128,
     Bool,
 
     StubbedPointer,
diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 4d235b5..b4fe3de 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -58,12 +58,18 @@ target_link_libraries(integration_test_runner PRIVATE
   GTest::gmock_main
   Boost::headers
   ${Boost_LIBRARIES}
+  range-v3
   toml
 )
 target_compile_definitions(integration_test_runner PRIVATE
-  TARGET_EXE_PATH="${CMAKE_CURRENT_BINARY_DIR}/integration_test_target"
+  TARGET_EXE_PATH="$<TARGET_FILE:integration_test_target>"
   OID_EXE_PATH="$<TARGET_FILE:oid>"
-  CONFIG_FILE_PATH="${CMAKE_BINARY_DIR}/testing.oid.toml")
+  OILGEN_EXE_PATH="$<TARGET_FILE:oilgen>"
+  CONFIG_FILE_PATH="${CMAKE_BINARY_DIR}/testing.oid.toml"
+
+  CXX="${CMAKE_CXX_COMPILER}"
+  TARGET_INCLUDE_DIRECTORIES="$<JOIN:$<TARGET_PROPERTY:integration_test_target,INCLUDE_DIRECTORIES>,:>"
+)
 
 if (${THRIFT_FOUND})
   foreach(THRIFT_TEST IN LISTS THRIFT_TESTS)
@@ -85,6 +91,7 @@ if (${THRIFT_FOUND})
 
     add_custom_target(integration_test_thrift_sources_${THRIFT_TEST} DEPENDS ${THRIFT_TYPES_H})
     add_dependencies(integration_test_target integration_test_thrift_sources_${THRIFT_TEST})
+    add_dependencies(integration_test_runner integration_test_thrift_sources_${THRIFT_TEST})
     target_sources(integration_test_target PRIVATE ${THRIFT_DATA_CPP})
   endforeach()
 
diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py
index b72e371..d9fe054 100644
--- a/test/integration/gen_tests.py
+++ b/test/integration/gen_tests.py
@@ -51,6 +51,41 @@ def add_headers(f, custom_headers, thrift_headers):
         f.write(f'#include "{header}"\n')
 
 
+def add_test_getters(f, case_name, case):
+    param_types = ", ".join(
+        f"std::remove_cvref_t<{param}>" for param in case["param_types"]
+    )
+    if "arg_types" in case:
+        arg_types = ", ".join(case["arg_types"])
+    else:
+        arg_types = param_types
+
+    f.write(
+        f"\n"
+        f"  std::tuple<{arg_types}> get_{case_name}() {{\n"
+        f'{case["setup"]}\n'
+        f"  }}\n"
+    )
+
+
+def get_param_str(param, i):
+    if "]" in param:
+        # Array param
+
+        if ")" in param:
+            # "int(&)[5]" ->  "int (&a0)[5]"
+            start, end = param.split(")")
+            return f"{start}a{i}){end}"
+
+        # "int[5]" -> "int a0[5]"
+        # "int[5][10]" -> "int a0[5][10]"
+        type_name, array_size = param.split("[", 1)
+        return f"{type_name} a{i}[{array_size}"
+
+    # Non-array param, e.g. "int&" -> "int& a0"
+    return f"{param} a{i}"
+
+
 def add_test_setup(f, config):
     ns = get_namespace(config["suite"])
     # fmt: off
@@ -65,23 +100,6 @@ def add_test_setup(f, config):
     )
     # fmt: on
 
-    def get_param_str(param, i):
-        if "]" in param:
-            # Array param
-
-            if ")" in param:
-                # "int(&)[5]" ->  "int (&a0)[5]"
-                start, end = param.split(")")
-                return f"{start}a{i}){end}"
-
-            # "int[5]" -> "int a0[5]"
-            # "int[5][10]" -> "int a0[5][10]"
-            type_name, array_size = param.split("[", 1)
-            return f"{type_name} a{i}[{array_size}"
-
-        # Non-array param, e.g. "int&" -> "int& a0"
-        return f"{param} a{i}"
-
     def define_traceable_func(name, params, body):
         return (
             f"\n"
@@ -99,21 +117,7 @@ def add_test_setup(f, config):
             # target func for it
             continue
 
-        # generate getter for an object of this type
-        param_types = ", ".join(
-            f"std::remove_cvref_t<{param}>" for param in case["param_types"]
-        )
-        if "arg_types" in case:
-            arg_types = ", ".join(case["arg_types"])
-        else:
-            arg_types = param_types
-
-        f.write(
-            f"\n"
-            f"  std::tuple<{arg_types}> get_{case_name}() {{\n"
-            f'{case["setup"]}\n'
-            f"  }}\n"
-        )
+        add_test_getters(f, case_name, case)
 
         # generate oid and oil targets
         params_str = ", ".join(
@@ -266,6 +270,7 @@ def add_tests(f, config):
     for case_name, case in config["cases"].items():
         add_oid_integration_test(f, config, case_name, case)
         add_oil_integration_test(f, config, case_name, case)
+        add_oilgen_integration_test(f, config, case_name, case)
 
 
 def add_oid_integration_test(f, config, case_name, case):
@@ -400,6 +405,90 @@ def add_oil_integration_test(f, config, case_name, case):
     f.write(f"}}\n")
 
 
+def add_oilgen_integration_test(f, config, case_name, case):
+    case_str = get_case_name(config["suite"], case_name)
+    exit_code = case.get("expect_oil_exit_code", 0)
+
+    if "oil_disable" in case or "target_function" in case:
+        return
+
+    config_prefix = case.get("config_prefix", "")
+    config_suffix = case.get("config_suffix", "")
+
+    f.write(
+        f"\n"
+        f"TEST_F(OilgenIntegration, {case_str}) {{\n"
+        f"{generate_skip(case, 'oil')}"
+    )
+
+    f.write('  constexpr std::string_view targetSrc = R"--(')
+    headers = set(config.get("includes", []))
+    thrift_headers = [f"thrift/annotation/gen-cpp2/{config['suite']}_types.h"] if is_thrift_test(config) else []
+    add_headers(f, sorted(headers), thrift_headers)
+
+    f.write(
+        f"\n"
+        f'{config.get("raw_definitions", "")}\n'
+        f"#pragma clang diagnostic push\n"
+        f'#pragma clang diagnostic ignored "-Wunused-private-field"\n'
+        f'{config.get("definitions", "")}\n'
+        f"#pragma clang diagnostic pop\n"
+    )
+    add_test_getters(f, case_name, case)
+
+    main = "int main() {\n"
+    main += "  auto pr = oi::exporters::Json(std::cout);\n"
+    main += "  pr.setPretty(true);\n"
+    main += f"  auto val = get_{case_name}();\n"
+    for i in range(len(case["param_types"])):
+        main += f"  auto ret{i} = oi::result::SizedResult(oi::introspect"
+        if "arg_types" in case:
+            main += f"<std::decay_t<{case['param_types'][i]}>>"
+        main += f"(std::get<{i}>(val)));\n"
+        main += f"  pr.print(ret{i});\n"
+    main += "}\n"
+
+    f.write(main)
+    f.write(')--";\n')
+
+    f.write(
+        f'  std::string configPrefix = R"--({config_prefix})--";\n'
+        f'  std::string configSuffix = R"--({config_suffix})--";\n'
+        f"  ba::io_context ctx;\n"
+        f"  auto target = runOilgenTarget({{\n"
+        f"    .ctx = ctx,\n"
+        f"    .targetSrc = targetSrc,\n"
+        f"  }}, std::move(configPrefix), std::move(configSuffix));\n\n"
+        f"  ASSERT_EQ(exit_code(target), {exit_code});\n"
+    )
+
+    key = "expect_json"
+    if "expect_json_v2" in case:
+        key = "expect_json_v2"
+    if key in case:
+        try:
+            json.loads(case[key])
+        except json.decoder.JSONDecodeError as error:
+            print(
+                f"\x1b[31m`expect_json` value for test case {config['suite']}.{case_name} was invalid JSON: {error}\x1b[0m",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+        f.write(
+            f"\n"
+            f"  std::stringstream expected_json_ss;\n"
+            f'  expected_json_ss << R"--({case[key]})--";\n'
+            f"  auto result_json_ss = std::stringstream(stdout_);\n"
+            f"  bpt::ptree expected_json, actual_json;\n"
+            f"  bpt::read_json(expected_json_ss, expected_json);\n"
+            f"  bpt::read_json(result_json_ss, actual_json);\n"
+            f"  compare_json(expected_json, actual_json);\n"
+        )
+
+    f.write(f"}}\n")
+
+
 def generate_skip(case, specific):
     possibly_skip = ""
     skip_reason = case.get("skip", False)
diff --git a/test/integration/runner_common.cpp b/test/integration/runner_common.cpp
index 3c8525a..f67d3cc 100644
--- a/test/integration/runner_common.cpp
+++ b/test/integration/runner_common.cpp
@@ -4,11 +4,13 @@
 #include <boost/algorithm/string.hpp>
 #include <boost/asio.hpp>
 #include <boost/process.hpp>
+#include <boost/property_tree/json_parser.hpp>
 #include <cstdlib>
 #include <filesystem>
 #include <fstream>
 #include <ios>
 #include <iostream>
+#include <range/v3/view/split.hpp>
 #include <string>
 #include <utility>
 
@@ -447,3 +449,166 @@ Proc OilIntegration::runOilTarget(OilOpts opts,
               std::move(std_out),
               std::move(std_err)};
 }
+
+std::string OilgenIntegration::TmpDirStr() {
+  return std::string("/tmp/oilgen-integration-XXXXXX");
+}
+
+Proc OilgenIntegration::runOilgenTarget(OilgenOpts opts,
+                                        std::string configPrefix,
+                                        std::string configSuffix) {
+  // Run an oilgen test in three stages.
+  // 1. Fake up a compilation database from the real CMake one.
+  // 1. Generate the OIL implementation .o from the input source.
+  // 2. Compile the input source and link to the implementation.
+  // 3. Run the now complete target.
+
+  static constexpr std::string_view InputPath = "input_src.cpp";
+  static constexpr std::string_view ObjectPath = "oil_generated.o";
+  static constexpr std::string_view TargetPath = "generated_target";
+  static const std::string CompileCommandsOutPath = "compile_commands.json";
+
+  static const char* CompileCommandsInPath = "/data/users/jakehillion/object-introspection-sl/build/compile_commands.json";
+
+  {
+    std::ofstream file{std::string(InputPath), std::ios_base::app};
+    file << opts.targetSrc;
+  }
+
+  bpt::ptree compile_commands_in;
+  bpt::ptree compile_commands_out;
+  bpt::read_json(CompileCommandsInPath, compile_commands_in);
+  for (bpt::ptree::value_type& entry : compile_commands_in) {
+    if (!entry.second.get<std::string>("file").ends_with("integration_test_target.cpp"))
+      continue;
+
+    auto input = std::filesystem::absolute(InputPath).string();
+    entry.second.put("file", input);
+    compile_commands_out.add_child(bpt::ptree::path_type(input, ':'), entry.second);
+    break;
+  }
+  bpt::write_json(CompileCommandsOutPath, compile_commands_out);
+
+  std::string generatorExe{OILGEN_EXE_PATH};
+  generatorExe += " ";
+  generatorExe += InputPath;
+  generatorExe += " --output=";
+  generatorExe += ObjectPath;
+  generatorExe += " --debug-level=3";
+  if (auto prefix = writeCustomConfig("prefix", configPrefix)) {
+    generatorExe += " --config-file ";
+    generatorExe += *prefix;
+  }
+  generatorExe += " --config-file ";
+  generatorExe += configFile;
+  if (auto suffix = writeCustomConfig("suffix", configSuffix)) {
+    generatorExe += " --config-file ";
+    generatorExe += *suffix;
+  }
+
+
+  std::vector<std::string> clangArgs{
+      "-DOIL_AOT_COMPILATION=1",
+      "--std=c++20",
+      "-resource-dir",
+      "/usr/lib64/clang/15.0.7",
+  };
+
+  for(auto&& rng : TARGET_INCLUDE_DIRECTORIES | ranges::views::split(':')) {
+    clangArgs.emplace_back("-I");
+    clangArgs.emplace_back(&*rng.begin(), ranges::distance(rng));
+  }
+
+  for (const auto& arg : clangArgs) {
+    generatorExe += " --extra-arg=";
+    generatorExe += arg;
+  }
+
+  if (verbose) {
+    std::cerr << "Running: " << generatorExe << std::endl;
+  }
+
+  bp::child generatorProc{generatorExe, opts.ctx};
+  generatorProc.wait();
+  if (generatorProc.exit_code() != 0)
+    throw std::runtime_error("generation failed!");
+
+  std::string compilerExe{CXX};
+  compilerExe += " input_src.cpp ";
+  compilerExe += ObjectPath;
+  compilerExe += " -o ";
+  compilerExe += TargetPath;
+  compilerExe += " /data/users/jakehillion/object-introspection-sl/oi/IntrospectionResult.cpp";
+  compilerExe += " /data/users/jakehillion/object-introspection-sl/oi/exporters/ParsedData.cpp";
+
+  compilerExe += " -Wl,-rpath,/data/users/jakehillion/object-introspection-sl/extern/drgn/build/.libs:/data/users/jakehillion/object-introspection-sl/extern/drgn/build/velfutils/libdw:/data/users/jakehillion/object-introspection-sl/extern/drgn/build/velfutils/libelf:/data/users/jakehillion/object-introspection-sl/extern/drgn/build/velfutils/libdwelf:/home/jakehillion/fbsource/fbcode/third-party-buck/platform010/build/icu/lib:/home/jakehillion/fbsource/fbcode/third-party-buck/platform010/build/gflags/lib";
+  for (const auto& arg : clangArgs) {
+    compilerExe += ' ';
+    compilerExe += arg;
+  }
+
+  if (verbose) {
+    std::cerr << "Running: " << compilerExe << std::endl;
+  }
+
+  bp::child compilerProc{compilerExe, opts.ctx};
+  compilerProc.wait();
+  if (compilerProc.exit_code() != 0)
+    throw std::runtime_error("compilation failed");
+
+  std::string targetExe = "./";
+  targetExe += TargetPath;
+
+  if (verbose) {
+    std::cerr << "Running: " << targetExe << std::endl;
+  }
+
+  // Use tee to write the output to files. If verbose is on, also redirect the
+  // output to stderr.
+  bp::async_pipe std_out_pipe(opts.ctx), std_err_pipe(opts.ctx);
+  bp::child std_out, std_err;
+  if (verbose) {
+    // clang-format off
+    std_out = bp::child(bp::search_path("tee"),
+                        (workingDir / "stdout").string(),
+                        bp::std_in < std_out_pipe,
+                        bp::std_out > stderr,
+                        opts.ctx);
+    std_err = bp::child(bp::search_path("tee"),
+                        (workingDir / "stderr").string(),
+                        bp::std_in < std_err_pipe,
+                        bp::std_out > stderr,
+                        opts.ctx);
+    // clang-format on
+  } else {
+    // clang-format off
+    std_out = bp::child(bp::search_path("tee"),
+                        (workingDir / "stdout").string(),
+                        bp::std_in < std_out_pipe,
+                        bp::std_out > bp::null,
+                        opts.ctx);
+    std_err = bp::child(bp::search_path("tee"),
+                        (workingDir / "stderr").string(),
+                        bp::std_in < std_err_pipe,
+                        bp::std_out > bp::null,
+                        opts.ctx);
+    // clang-format on
+  }
+
+  /* Spawn `oid` with tracing on and IOs redirected */
+  // clang-format off
+  bp::child targetProc(
+      targetExe,
+      bp::std_in  < bp::null,
+      bp::std_out > std_out_pipe,
+      bp::std_err > std_err_pipe,
+      opts.ctx);
+  // clang-format on
+
+  return Proc{
+      opts.ctx,
+      std::move(targetProc),
+      std::move(std_out),
+      std::move(std_err),
+  };
+}
diff --git a/test/integration/runner_common.h b/test/integration/runner_common.h
index d2b5e57..8c6c8eb 100644
--- a/test/integration/runner_common.h
+++ b/test/integration/runner_common.h
@@ -22,6 +22,11 @@ struct OilOpts {
   std::string targetArgs;
 };
 
+struct OilgenOpts {
+  boost::asio::io_context& ctx;
+  std::string_view targetSrc;
+};
+
 struct Proc {
   boost::asio::io_context& ctx;
   boost::process::child proc;
@@ -81,3 +86,12 @@ class OilIntegration : public IntegrationBase {
                     std::string configPrefix,
                     std::string configSuffix);
 };
+
+class OilgenIntegration : public IntegrationBase {
+ protected:
+  std::string TmpDirStr() override;
+
+  Proc runOilgenTarget(OilgenOpts opts,
+                       std::string configPrefix,
+                       std::string configSuffix);
+};

From 587daae97aab8408ef564be258eb099a738d688e Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 4 Jan 2024 14:03:32 +0000
Subject: [PATCH 109/188] tbv2: use std::decay_t with smart pointers

CodeGen v2 permits template parameters to be qualified. This means that if we
call `make_field` with a template parameter it will be qualified. However, we
don't qualify the types when generating meta functions such as `NameProvider`
and `TypeHandler`. This means these qualified types don't match up with the
expected type.

Use `std::decay_t` when forwarding the type to `NameProvider` and `TypeHandler`
so they're always the base type that they were generated with. Most of this is
covered by `make_field`, but there are direct references to `TypeHandler<Ctx,
T>` in a lot of `TypeHandler::type` fields. Fix the problematic types manually
for now, there may need to be a better solution with meta functions for this in
the future.

Test Plan:
- CI
- Added a test for `std::unique_ptr<const uint64_t>` to exercise this. Failed
  before, passes after.
- Added a test for `std::unique_ptr<const std::vector<uint64_t>>` to test a
  non-primitive type. Failed before, passes after.
---
 oi/CodeGen.cpp                      |  8 +--
 test/integration/std_smart_ptr.toml | 97 +++++++++++++++++++++++++++++
 types/shrd_ptr_type.toml            |  2 +-
 types/uniq_ptr_type.toml            |  2 +-
 4 files changed, 103 insertions(+), 6 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 9614a1f..d9e4a38 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -1070,11 +1070,11 @@ template <typename Ctx, typename T>
 constexpr inst::Field make_field(std::string_view name) {
   return inst::Field{
     sizeof(T),
-    ExclusiveSizeProvider<T>::size,
+    ExclusiveSizeProvider<std::decay_t<T>>::size,
     name,
-    NameProvider<T>::names,
-    TypeHandler<Ctx, T>::fields,
-    TypeHandler<Ctx, T>::processors,
+    NameProvider<std::decay_t<T>>::names,
+    TypeHandler<Ctx, std::decay_t<T>>::fields,
+    TypeHandler<Ctx, std::decay_t<T>>::processors,
   };
 }
 )";
diff --git a/test/integration/std_smart_ptr.toml b/test/integration/std_smart_ptr.toml
index 8fc66bf..e14e398 100644
--- a/test/integration/std_smart_ptr.toml
+++ b/test/integration/std_smart_ptr.toml
@@ -31,6 +31,31 @@ definitions = '''
       }
     ]
     '''
+  [cases.unique_ptr_const_uint64_empty]
+    param_types = ["std::unique_ptr<const std::uint64_t>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 8,
+        "dynamicSize": 0,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 8
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 8,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.unique_ptr_uint64_present]
     param_types = ["std::unique_ptr<std::uint64_t>&"]
     setup = "return {std::make_unique<std::uint64_t>(64)};"
@@ -79,6 +104,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.unique_ptr_const_vector_empty]
+    param_types = ["std::unique_ptr<const std::vector<std::uint64_t>>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 8,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 24
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 8,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.unique_ptr_vector_present]
     param_types = ["std::unique_ptr<std::vector<std::uint64_t>>&"]
     setup = "return {std::make_unique<std::vector<std::uint64_t>>(std::initializer_list<std::uint64_t>({1,2,3,4,5}))};"
@@ -181,6 +230,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.shared_ptr_const_uint64_empty]
+    param_types = ["std::shared_ptr<const std::uint64_t>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 16,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 8
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 16,
+        "exclusiveSize": 16,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.shared_ptr_uint64_present]
     param_types = ["std::shared_ptr<std::uint64_t>&"]
     setup = "return std::make_shared<std::uint64_t>(64);"
@@ -232,6 +305,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.shared_ptr_const_vector_empty]
+    param_types = ["std::shared_ptr<const std::vector<std::uint64_t>>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 16,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 24
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 16,
+        "exclusiveSize": 16,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.shared_ptr_vector_present]
     param_types = ["std::shared_ptr<std::vector<std::uint64_t>>&"]
     setup = "return std::make_shared<std::vector<std::uint64_t>>(std::initializer_list<std::uint64_t>({1,2,3,4,5}));"
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index bb3a622..a7095ce 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -60,7 +60,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<Ctx, T0>::type>
+  typename TypeHandler<Ctx, std::decay_t<T0>>::type>
 """
 func = """
 #ifdef __GLIBCXX__
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index f723f80..68d20e7 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -61,7 +61,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<Ctx, T0>::type>
+  typename TypeHandler<Ctx, std::decay_t<T0>>::type>
 """
 func = """
 auto sum = std::get<ParsedData::Sum>(d.val);

From e559821e8e0fe428b8b105893cdc1deed49a42bc Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 4 Jan 2024 17:01:52 +0000
Subject: [PATCH 110/188] incomplete: name type in compiler errors

Summary:

We have a good type representation in the Type Graph of an incomplete type and
the underlying type that represents. However, this incomplete type still ends
up in the generated code as `void` which loses information. For example, a
container that can't contain void may fail to compile because it was
initialised with `void` but really its because the type it was supposed to be
initialised with (say, `Foo`) had incomplete debug information.

This change identifies that a type is incomplete in the output by generating it
as an incomplete type `struct Incomplete<struct Foo>`. This allows us to name
the type correctly in the TreeBuilder output and filter for incomplete types,
as well as getting appropriate compiler errors if it mustn't be incomplete.

Test Plan:
- CI
- Added a unit test to namegen.
- Enabled and added an extra pointers_incomplete test.

This change is tricky to test because it isn't really user visible. The types
still use their `inputName` which is unchanged in any successful output - this
change is used so the compiler fails with a more detailed error.
---
 oi/CodeGen.cpp                            |  9 ++++++
 oi/OITraceCode.cpp                        | 16 ++++++++++
 oi/type_graph/DrgnExporter.cpp            |  2 ++
 oi/type_graph/NameGen.cpp                 | 23 +++++++++++++
 oi/type_graph/NameGen.h                   |  1 +
 oi/type_graph/Printer.cpp                 |  4 ++-
 oi/type_graph/TopoSorter.cpp              |  4 +++
 oi/type_graph/TopoSorter.h                |  1 +
 oi/type_graph/Types.cpp                   |  2 --
 oi/type_graph/Types.h                     | 39 +++++++++++++++--------
 test/TypeGraphParser.cpp                  |  5 +--
 test/integration/pointers_incomplete.toml | 32 ++++++++++++++++---
 test/test_drgn_parser.cpp                 |  4 +--
 test/test_flattener.cpp                   |  2 +-
 test/test_name_gen.cpp                    | 13 ++++++++
 test/test_remove_members.cpp              |  2 +-
 types/shrd_ptr_type.toml                  |  6 ++--
 types/uniq_ptr_type.toml                  |  6 ++--
 18 files changed, 138 insertions(+), 33 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 9614a1f..63ac247 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -60,6 +60,7 @@ using type_graph::EnforceCompatibility;
 using type_graph::Enum;
 using type_graph::Flattener;
 using type_graph::IdentifyContainers;
+using type_graph::Incomplete;
 using type_graph::KeyCapture;
 using type_graph::Member;
 using type_graph::NameGen;
@@ -181,12 +182,20 @@ void genDeclsEnum(const Enum& e, std::string& code) {
   code += " {};\n";
 }
 
+void genDeclsIncomplete(const Incomplete& i, std::string& code) {
+  code += "template<> struct ";
+  code += i.name();
+  code += ";\n";
+}
+
 void genDecls(const TypeGraph& typeGraph, std::string& code) {
   for (const Type& t : typeGraph.finalTypes) {
     if (const auto* c = dynamic_cast<const Class*>(&t)) {
       genDeclsClass(*c, code);
     } else if (const auto* e = dynamic_cast<const Enum*>(&t)) {
       genDeclsEnum(*e, code);
+    } else if (const auto* i = dynamic_cast<const Incomplete*>(&t)) {
+      genDeclsIncomplete(*i, code);
     }
   }
 }
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index 31a873b..a100900 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -182,3 +182,19 @@ bool isStorageInline(const auto& c) {
   return (uintptr_t)std::data(c) < (uintptr_t)(&c + sizeof(c)) &&
          (uintptr_t)std::data(c) >= (uintptr_t)&c;
 }
+
+namespace OIInternal {
+namespace {
+
+template <typename T>
+struct Incomplete;
+
+template <typename T>
+constexpr bool oi_is_complete = true;
+template <>
+constexpr bool oi_is_complete<void> = false;
+template <typename T>
+constexpr bool oi_is_complete<Incomplete<T>> = false;
+
+}  // namespace
+}  // namespace OIInternal
diff --git a/oi/type_graph/DrgnExporter.cpp b/oi/type_graph/DrgnExporter.cpp
index df9722d..1e79fb6 100644
--- a/oi/type_graph/DrgnExporter.cpp
+++ b/oi/type_graph/DrgnExporter.cpp
@@ -100,6 +100,8 @@ drgn_type* DrgnExporter::visit(Container& c) {
     if (auto* p = dynamic_cast<const Primitive*>(&paramType);
         p && p->kind() == Primitive::Kind::Void) {
       return drgnType;
+    } else if (auto* p = dynamic_cast<const Incomplete*>(&paramType)) {
+      return drgnType;
     }
   }
 
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index 725ab59..0cc2a61 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -219,4 +219,27 @@ void NameGen::visit(CaptureKeys& c) {
   c.regenerateName();
 }
 
+void NameGen::visit(Incomplete& i) {
+  constexpr std::string_view kPrefix{"Incomplete<struct "};
+
+  RecursiveVisitor::visit(i);
+
+  std::string_view inputName = i.inputName();
+
+  std::string name;
+  name.reserve(kPrefix.size() + inputName.size() + 2);
+  name = kPrefix;
+
+  for (const unsigned char c : inputName) {
+    if (std::isalnum(c)) {
+      name += c;
+    } else {
+      name += '_';
+    }
+  }
+  name += ">";
+
+  i.setName(name);
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 4f1c258..927cef4 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -49,6 +49,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Reference& r) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
+  void visit(Incomplete& i) override;
 
   static const inline std::string AnonPrefix = "__oi_anon";
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index f598161..d8bac7c 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -37,7 +37,9 @@ void Printer::print(const Type& type) {
 }
 
 void Printer::visit(const Incomplete& i) {
-  prefix();
+  if (prefix(i))
+    return;
+
   out_ << "Incomplete";
   if (auto underlyingType = i.underlyingType()) {
     out_ << std::endl;
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index 6ca10af..ff2715a 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -139,6 +139,10 @@ void TopoSorter::visit(CaptureKeys& c) {
   sortedTypes_.push_back(c);
 }
 
+void TopoSorter::visit(Incomplete& i) {
+  sortedTypes_.push_back(i);
+}
+
 /*
  * A type graph may contain cycles, so we need to slightly tweak the standard
  * topological sorting algorithm. Cycles can only be introduced by certain
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 374e943..44f2364 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -48,6 +48,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
+  void visit(Incomplete& i) override;
 
  private:
   std::unordered_set<Type*> visited_;
diff --git a/oi/type_graph/Types.cpp b/oi/type_graph/Types.cpp
index c6219b5..9b53f49 100644
--- a/oi/type_graph/Types.cpp
+++ b/oi/type_graph/Types.cpp
@@ -35,8 +35,6 @@ namespace oi::detail::type_graph {
 OI_TYPE_LIST
 #undef X
 
-const std::string Incomplete::kName = "void";
-
 std::string Primitive::getName(Kind kind) {
   switch (kind) {
     case Kind::Int8:
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index 1d5dd79..f09f926 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -29,6 +29,8 @@
  * debugging.
  */
 
+#include <algorithm>
+#include <cctype>
 #include <cstddef>
 #include <map>
 #include <optional>
@@ -211,29 +213,33 @@ struct TemplateParam {
  */
 class Incomplete : public Type {
  public:
-  Incomplete(Type& underlyingType) : underlyingType_(underlyingType) {
+  Incomplete(NodeId id, Type& underlyingType)
+      : id_(id), underlyingType_(underlyingType) {
   }
 
-  Incomplete(std::string underlyingTypeName)
-      : underlyingType_(std::move(underlyingTypeName)) {
+  Incomplete(NodeId id, std::string underlyingTypeName)
+      : id_(id), underlyingType_(std::move(underlyingTypeName)) {
   }
 
-  static inline constexpr bool has_node_id = false;
+  static inline constexpr bool has_node_id = true;
 
   DECLARE_ACCEPT
 
   const std::string& name() const override {
-    return kName;
+    return name_;
   }
 
   std::string_view inputName() const override {
-    if (std::holds_alternative<std::string>(underlyingType_)) {
-      return std::get<std::string>(underlyingType_);
-    }
-
-    return std::get<std::reference_wrapper<Type>>(underlyingType_)
-        .get()
-        .inputName();
+    return std::visit(
+        [](const auto& el) -> std::string_view {
+          using T = std::decay_t<decltype(el)>;
+          if constexpr (std::is_same_v<T, std::string>) {
+            return el;
+          } else {
+            return el.get().inputName();
+          }
+        },
+        underlyingType_);
   }
 
   size_t size() const override {
@@ -245,7 +251,11 @@ class Incomplete : public Type {
   }
 
   NodeId id() const override {
-    return -1;
+    return id_;
+  }
+
+  void setName(std::string name) {
+    name_ = std::move(name);
   }
 
   std::optional<std::reference_wrapper<Type>> underlyingType() const {
@@ -257,8 +267,9 @@ class Incomplete : public Type {
   }
 
  private:
+  NodeId id_ = -1;
   std::variant<std::string, std::reference_wrapper<Type>> underlyingType_;
-  static const std::string kName;
+  std::string name_ = "void";
 };
 
 /*
diff --git a/test/TypeGraphParser.cpp b/test/TypeGraphParser.cpp
index f54fe08..d70236c 100644
--- a/test/TypeGraphParser.cpp
+++ b/test/TypeGraphParser.cpp
@@ -215,10 +215,11 @@ Type& TypeGraphParser::parseType(std::string_view& input, size_t rootIndent) {
       auto nameEndPos = line.find(']', nameStartPos);
       auto underlyingTypeName =
           line.substr(nameStartPos, nameEndPos - nameStartPos);
-      type = &typeGraph_.makeType<Incomplete>(std::string(underlyingTypeName));
+      type =
+          &typeGraph_.makeType<Incomplete>(id, std::string(underlyingTypeName));
     } else {
       auto& underlyingType = parseType(input, indent + 2);
-      type = &typeGraph_.makeType<Incomplete>(underlyingType);
+      type = &typeGraph_.makeType<Incomplete>(id, underlyingType);
     }
   } else if (nodeTypeName == "Class" || nodeTypeName == "Struct" ||
              nodeTypeName == "Union") {
diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml
index 0690511..2d34865 100644
--- a/test/integration/pointers_incomplete.toml
+++ b/test/integration/pointers_incomplete.toml
@@ -55,7 +55,6 @@ definitions = '''
     }]'''
 
   [cases.unique_ptr]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299
     param_types = ["const std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>&"]
     setup = '''
       auto raw_ptr = static_cast<IncompleteType*>(::operator new(5));
@@ -63,28 +62,29 @@ definitions = '''
         raw_ptr, &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
   [cases.unique_ptr_null]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299
     param_types = ["const std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>&"]
     setup = '''
       return std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>(
         nullptr, &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.shared_ptr]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300
     param_types = ["const std::shared_ptr<IncompleteType>&"]
     setup = '''
       auto raw_ptr = static_cast<IncompleteType*>(::operator new(5));
       return std::shared_ptr<IncompleteType>(raw_ptr , &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
   [cases.shared_ptr_null]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300
     param_types = ["const std::shared_ptr<IncompleteType>"]
     setup = "return nullptr;"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.containing_struct]
     oil_disable = "oil can't chase raw pointers safely"
@@ -116,3 +116,27 @@ definitions = '''
         }
       ]
     }]'''
+
+  [cases.containing_struct_no_follow]
+    param_types = ["const IncompleteTypeContainer&"]
+    setup = "return IncompleteTypeContainer{};"
+    expect_json = '''[{
+      "staticSize": 88,
+      "members": [
+        { "name": "ptrundef", "staticSize": 8 },
+        { "name": "__makePad1", "staticSize": 1 },
+        { "name": "shundef", "staticSize": 16 },
+        { "name": "__makePad2", "staticSize": 1 },
+        { "name": "shoptundef",
+          "staticSize": 24,
+          "length": 0,
+          "capacity": 1
+        },
+        { "name": "__makePad3", "staticSize": 1 },
+        { "name": "optundef",
+          "staticSize": 16,
+          "length": 0,
+          "capacity": 1
+        }
+      ]
+    }]'''
diff --git a/test/test_drgn_parser.cpp b/test/test_drgn_parser.cpp
index e8d03e3..c70f5a9 100644
--- a/test/test_drgn_parser.cpp
+++ b/test/test_drgn_parser.cpp
@@ -426,8 +426,8 @@ TEST_F(DrgnParserTest, PointerNoFollow) {
 
 TEST_F(DrgnParserTest, PointerIncomplete) {
   test("oid_test_case_pointers_incomplete_raw", R"(
-[0] Pointer
-      Incomplete: [IncompleteType]
+[1] Pointer
+[0]   Incomplete: [IncompleteType]
 )");
 }
 
diff --git a/test/test_flattener.cpp b/test/test_flattener.cpp
index 808f4ee..c8944bf 100644
--- a/test/test_flattener.cpp
+++ b/test/test_flattener.cpp
@@ -1007,7 +1007,7 @@ TEST(FlattenerTest, IncompleteParent) {
        R"(
 [0] Class: MyClass (size: 4)
       Parent (offset: 0)
-        Incomplete: [IncompleteParent]
+[1]     Incomplete: [IncompleteParent]
 )",
        R"(
 [0] Class: MyClass (size: 4)
diff --git a/test/test_name_gen.cpp b/test/test_name_gen.cpp
index d0cda26..a83c180 100644
--- a/test/test_name_gen.cpp
+++ b/test/test_name_gen.cpp
@@ -501,3 +501,16 @@ TEST(NameGenTest, AnonymousMembers) {
   EXPECT_EQ(myclass.members[0].inputName, "__oi_anon_0");
   EXPECT_EQ(myclass.members[1].inputName, "__oi_anon_1");
 }
+
+TEST(NameGenTest, IncompleteTypes) {
+  auto myincompletevector = Incomplete{0, "std::vector<int>"};
+
+  auto myint = Primitive{Primitive::Kind::Int32};
+  auto myincompleteint = Incomplete{1, myint};
+
+  NameGen nameGen;
+  nameGen.generateNames({myincompletevector, myincompleteint});
+
+  EXPECT_EQ(myincompletevector.name(), "Incomplete<struct std__vector_int_>");
+  EXPECT_EQ(myincompleteint.name(), "Incomplete<struct int32_t>");
+}
diff --git a/test/test_remove_members.cpp b/test/test_remove_members.cpp
index 0d4a54e..9eff4ed 100644
--- a/test/test_remove_members.cpp
+++ b/test/test_remove_members.cpp
@@ -191,7 +191,7 @@ TEST(RemoveMembersTest, Incomplete) {
       Member: a (offset: 0)
         Primitive: int32_t
       Member: b (offset: 4)
-        Incomplete: [MyIncompleteType]
+[1]     Incomplete: [MyIncompleteType]
       Member: c (offset: 8)
         Primitive: int32_t
 )",
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index bb3a622..3439bf3 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
         if (s_ptr && ctx.pointers.add((uintptr_t)(s_ptr.get()))) {
@@ -37,7 +37,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (!oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
@@ -85,7 +85,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index f723f80..f71cff4 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T,Deleter>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
         if (u_ptr && ctx.pointers.add((uintptr_t)(u_ptr.get()))) {
@@ -38,7 +38,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (!oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
@@ -71,7 +71,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);

From bb724f432bec8c853998073397830c5980163265 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 9 Jan 2024 13:56:17 +0000
Subject: [PATCH 111/188] tbv2: fix type names for std::optional

Type names of optional elements were accidentally left as todo. Update
`std::optional` to use `make_field` and correctly name its elements.

Test Plan:
- CI
- Updated the integration tests to test the names.
---
 test/integration/std_optional.toml | 6 ++++++
 types/optional_type.toml           | 9 +--------
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/test/integration/std_optional.toml b/test/integration/std_optional.toml
index 9da0271..b943df3 100644
--- a/test/integration/std_optional.toml
+++ b/test/integration/std_optional.toml
@@ -18,6 +18,7 @@ includes = ["optional", "cstdint", "vector"]
     expect_json_v2 = '''
     [
       {
+        "typeNames": ["std::optional<uint64_t>"],
         "staticSize": 16,
         "exclusiveSize": 16,
         "size": 16,
@@ -45,6 +46,7 @@ includes = ["optional", "cstdint", "vector"]
     expect_json_v2 = '''
     [
       {
+        "typeNames": ["std::optional<uint64_t>"],
         "staticSize": 16,
         "exclusiveSize": 8,
         "size": 16,
@@ -52,6 +54,7 @@ includes = ["optional", "cstdint", "vector"]
         "capacity": 1,
         "members": [
           {
+            "typeNames": ["uint64_t"],
             "staticSize": 8,
             "exclusiveSize": 8
           }
@@ -77,6 +80,7 @@ includes = ["optional", "cstdint", "vector"]
     expect_json_v2 = '''
     [
       {
+        "typeNames": ["std::optional<std::vector<uint64_t, std::allocator<uint64_t>>>"],
         "staticSize": 32,
         "exclusiveSize": 32,
         "size": 32,
@@ -112,6 +116,7 @@ includes = ["optional", "cstdint", "vector"]
     expect_json_v2 = '''
     [
       {
+        "typeNames": ["std::optional<std::vector<uint64_t, std::allocator<uint64_t>>>"],
         "staticSize": 32,
         "exclusiveSize": 8,
         "size": 72,
@@ -119,6 +124,7 @@ includes = ["optional", "cstdint", "vector"]
         "capacity": 1,
         "members": [
           {
+            "typeNames": ["std::vector<uint64_t, std::allocator<uint64_t>>"],
             "staticSize": 24,
             "exclusiveSize": 24,
             "size": 64,
diff --git a/types/optional_type.toml b/types/optional_type.toml
index eeba64e..2fc6c60 100644
--- a/types/optional_type.toml
+++ b/types/optional_type.toml
@@ -41,14 +41,7 @@ if (container.has_value()) {
 [[codegen.processor]]
 type = "types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, T0>::type>"
 func = """
-static constexpr std::array<std::string_view, 1> names{"TODO"};
-static constexpr auto elementField = inst::Field{
-  sizeof(T0),
-  "el",
-  names,
-  TypeHandler<Ctx, T0>::fields,
-  TypeHandler<Ctx, T0>::processors,
-};
+static constexpr auto elementField = make_field<Ctx, T0>("el");
 
 auto sum = std::get<ParsedData::Sum>(d.val);
 

From 5cd2443ddfa39b7ce12c9fb4dbf2ca1278a2affc Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 4 Jan 2024 17:01:52 +0000
Subject: [PATCH 112/188] incomplete: name type in compiler errors

Summary:

We have a good type representation in the Type Graph of an incomplete type and
the underlying type that represents. However, this incomplete type still ends
up in the generated code as `void` which loses information. For example, a
container that can't contain void may fail to compile because it was
initialised with `void` but really its because the type it was supposed to be
initialised with (say, `Foo`) had incomplete debug information.

This change identifies that a type is incomplete in the output by generating it
as an incomplete type `struct Incomplete<struct Foo>`. This allows us to name
the type correctly in the TreeBuilder output and filter for incomplete types,
as well as getting appropriate compiler errors if it mustn't be incomplete.

Test Plan:
- CI
- Added a unit test to namegen.
- Enabled and added an extra pointers_incomplete test.

This change is tricky to test because it isn't really user visible. The types
still use their `inputName` which is unchanged in any successful output - this
change is used so the compiler fails with a more detailed error.
---
 oi/CodeGen.cpp                            | 15 +++++----
 oi/OITraceCode.cpp                        | 16 ++++++++++
 oi/type_graph/DrgnExporter.cpp            |  2 ++
 oi/type_graph/NameGen.cpp                 | 23 +++++++++++++
 oi/type_graph/NameGen.h                   |  1 +
 oi/type_graph/Printer.cpp                 |  4 ++-
 oi/type_graph/TopoSorter.cpp              |  4 +++
 oi/type_graph/TopoSorter.h                |  1 +
 oi/type_graph/Types.cpp                   |  2 --
 oi/type_graph/Types.h                     | 39 +++++++++++++++--------
 test/TypeGraphParser.cpp                  |  5 +--
 test/integration/pointers_incomplete.toml | 32 ++++++++++++++++---
 test/test_drgn_parser.cpp                 |  4 +--
 test/test_flattener.cpp                   |  2 +-
 test/test_name_gen.cpp                    | 13 ++++++++
 test/test_remove_members.cpp              |  2 +-
 types/shrd_ptr_type.toml                  |  6 ++--
 types/uniq_ptr_type.toml                  |  6 ++--
 18 files changed, 137 insertions(+), 40 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 9614a1f..9866550 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -60,6 +60,7 @@ using type_graph::EnforceCompatibility;
 using type_graph::Enum;
 using type_graph::Flattener;
 using type_graph::IdentifyContainers;
+using type_graph::Incomplete;
 using type_graph::KeyCapture;
 using type_graph::Member;
 using type_graph::NameGen;
@@ -438,6 +439,13 @@ void addStandardGetSizeFuncDefs(std::string& code) {
     template<typename T>
     void getSizeType(/*const*/ T* s_ptr, size_t& returnArg)
     {
+      if constexpr (!oi_is_complete<T>) {
+        JLOG("incomplete ptr @");
+        JLOGPTR(s_ptr);
+        StoreData((uintptr_t)(s_ptr), returnArg);
+        return;
+      }
+
       JLOG("ptr val @");
       JLOGPTR(s_ptr);
       StoreData((uintptr_t)(s_ptr), returnArg);
@@ -449,13 +457,6 @@ void addStandardGetSizeFuncDefs(std::string& code) {
       }
     }
 
-    void getSizeType(/*const*/ void *s_ptr, size_t& returnArg)
-    {
-      JLOG("void ptr @");
-      JLOGPTR(s_ptr);
-      StoreData((uintptr_t)(s_ptr), returnArg);
-    }
-
     template <typename T, int N>
     void getSizeType(const OIArray<T,N>& container, size_t& returnArg)
     {
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index 31a873b..a100900 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -182,3 +182,19 @@ bool isStorageInline(const auto& c) {
   return (uintptr_t)std::data(c) < (uintptr_t)(&c + sizeof(c)) &&
          (uintptr_t)std::data(c) >= (uintptr_t)&c;
 }
+
+namespace OIInternal {
+namespace {
+
+template <typename T>
+struct Incomplete;
+
+template <typename T>
+constexpr bool oi_is_complete = true;
+template <>
+constexpr bool oi_is_complete<void> = false;
+template <typename T>
+constexpr bool oi_is_complete<Incomplete<T>> = false;
+
+}  // namespace
+}  // namespace OIInternal
diff --git a/oi/type_graph/DrgnExporter.cpp b/oi/type_graph/DrgnExporter.cpp
index df9722d..39292e5 100644
--- a/oi/type_graph/DrgnExporter.cpp
+++ b/oi/type_graph/DrgnExporter.cpp
@@ -100,6 +100,8 @@ drgn_type* DrgnExporter::visit(Container& c) {
     if (auto* p = dynamic_cast<const Primitive*>(&paramType);
         p && p->kind() == Primitive::Kind::Void) {
       return drgnType;
+    } else if (auto* i = dynamic_cast<const Incomplete*>(&paramType)) {
+      return drgnType;
     }
   }
 
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index 725ab59..0cc2a61 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -219,4 +219,27 @@ void NameGen::visit(CaptureKeys& c) {
   c.regenerateName();
 }
 
+void NameGen::visit(Incomplete& i) {
+  constexpr std::string_view kPrefix{"Incomplete<struct "};
+
+  RecursiveVisitor::visit(i);
+
+  std::string_view inputName = i.inputName();
+
+  std::string name;
+  name.reserve(kPrefix.size() + inputName.size() + 2);
+  name = kPrefix;
+
+  for (const unsigned char c : inputName) {
+    if (std::isalnum(c)) {
+      name += c;
+    } else {
+      name += '_';
+    }
+  }
+  name += ">";
+
+  i.setName(name);
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 4f1c258..927cef4 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -49,6 +49,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Reference& r) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
+  void visit(Incomplete& i) override;
 
   static const inline std::string AnonPrefix = "__oi_anon";
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index f598161..d8bac7c 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -37,7 +37,9 @@ void Printer::print(const Type& type) {
 }
 
 void Printer::visit(const Incomplete& i) {
-  prefix();
+  if (prefix(i))
+    return;
+
   out_ << "Incomplete";
   if (auto underlyingType = i.underlyingType()) {
     out_ << std::endl;
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index 6ca10af..ff2715a 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -139,6 +139,10 @@ void TopoSorter::visit(CaptureKeys& c) {
   sortedTypes_.push_back(c);
 }
 
+void TopoSorter::visit(Incomplete& i) {
+  sortedTypes_.push_back(i);
+}
+
 /*
  * A type graph may contain cycles, so we need to slightly tweak the standard
  * topological sorting algorithm. Cycles can only be introduced by certain
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 374e943..44f2364 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -48,6 +48,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
+  void visit(Incomplete& i) override;
 
  private:
   std::unordered_set<Type*> visited_;
diff --git a/oi/type_graph/Types.cpp b/oi/type_graph/Types.cpp
index c6219b5..9b53f49 100644
--- a/oi/type_graph/Types.cpp
+++ b/oi/type_graph/Types.cpp
@@ -35,8 +35,6 @@ namespace oi::detail::type_graph {
 OI_TYPE_LIST
 #undef X
 
-const std::string Incomplete::kName = "void";
-
 std::string Primitive::getName(Kind kind) {
   switch (kind) {
     case Kind::Int8:
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index 1d5dd79..f09f926 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -29,6 +29,8 @@
  * debugging.
  */
 
+#include <algorithm>
+#include <cctype>
 #include <cstddef>
 #include <map>
 #include <optional>
@@ -211,29 +213,33 @@ struct TemplateParam {
  */
 class Incomplete : public Type {
  public:
-  Incomplete(Type& underlyingType) : underlyingType_(underlyingType) {
+  Incomplete(NodeId id, Type& underlyingType)
+      : id_(id), underlyingType_(underlyingType) {
   }
 
-  Incomplete(std::string underlyingTypeName)
-      : underlyingType_(std::move(underlyingTypeName)) {
+  Incomplete(NodeId id, std::string underlyingTypeName)
+      : id_(id), underlyingType_(std::move(underlyingTypeName)) {
   }
 
-  static inline constexpr bool has_node_id = false;
+  static inline constexpr bool has_node_id = true;
 
   DECLARE_ACCEPT
 
   const std::string& name() const override {
-    return kName;
+    return name_;
   }
 
   std::string_view inputName() const override {
-    if (std::holds_alternative<std::string>(underlyingType_)) {
-      return std::get<std::string>(underlyingType_);
-    }
-
-    return std::get<std::reference_wrapper<Type>>(underlyingType_)
-        .get()
-        .inputName();
+    return std::visit(
+        [](const auto& el) -> std::string_view {
+          using T = std::decay_t<decltype(el)>;
+          if constexpr (std::is_same_v<T, std::string>) {
+            return el;
+          } else {
+            return el.get().inputName();
+          }
+        },
+        underlyingType_);
   }
 
   size_t size() const override {
@@ -245,7 +251,11 @@ class Incomplete : public Type {
   }
 
   NodeId id() const override {
-    return -1;
+    return id_;
+  }
+
+  void setName(std::string name) {
+    name_ = std::move(name);
   }
 
   std::optional<std::reference_wrapper<Type>> underlyingType() const {
@@ -257,8 +267,9 @@ class Incomplete : public Type {
   }
 
  private:
+  NodeId id_ = -1;
   std::variant<std::string, std::reference_wrapper<Type>> underlyingType_;
-  static const std::string kName;
+  std::string name_ = "void";
 };
 
 /*
diff --git a/test/TypeGraphParser.cpp b/test/TypeGraphParser.cpp
index f54fe08..d70236c 100644
--- a/test/TypeGraphParser.cpp
+++ b/test/TypeGraphParser.cpp
@@ -215,10 +215,11 @@ Type& TypeGraphParser::parseType(std::string_view& input, size_t rootIndent) {
       auto nameEndPos = line.find(']', nameStartPos);
       auto underlyingTypeName =
           line.substr(nameStartPos, nameEndPos - nameStartPos);
-      type = &typeGraph_.makeType<Incomplete>(std::string(underlyingTypeName));
+      type =
+          &typeGraph_.makeType<Incomplete>(id, std::string(underlyingTypeName));
     } else {
       auto& underlyingType = parseType(input, indent + 2);
-      type = &typeGraph_.makeType<Incomplete>(underlyingType);
+      type = &typeGraph_.makeType<Incomplete>(id, underlyingType);
     }
   } else if (nodeTypeName == "Class" || nodeTypeName == "Struct" ||
              nodeTypeName == "Union") {
diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml
index 0690511..2d34865 100644
--- a/test/integration/pointers_incomplete.toml
+++ b/test/integration/pointers_incomplete.toml
@@ -55,7 +55,6 @@ definitions = '''
     }]'''
 
   [cases.unique_ptr]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299
     param_types = ["const std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>&"]
     setup = '''
       auto raw_ptr = static_cast<IncompleteType*>(::operator new(5));
@@ -63,28 +62,29 @@ definitions = '''
         raw_ptr, &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
   [cases.unique_ptr_null]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299
     param_types = ["const std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>&"]
     setup = '''
       return std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>(
         nullptr, &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.shared_ptr]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300
     param_types = ["const std::shared_ptr<IncompleteType>&"]
     setup = '''
       auto raw_ptr = static_cast<IncompleteType*>(::operator new(5));
       return std::shared_ptr<IncompleteType>(raw_ptr , &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
   [cases.shared_ptr_null]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300
     param_types = ["const std::shared_ptr<IncompleteType>"]
     setup = "return nullptr;"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.containing_struct]
     oil_disable = "oil can't chase raw pointers safely"
@@ -116,3 +116,27 @@ definitions = '''
         }
       ]
     }]'''
+
+  [cases.containing_struct_no_follow]
+    param_types = ["const IncompleteTypeContainer&"]
+    setup = "return IncompleteTypeContainer{};"
+    expect_json = '''[{
+      "staticSize": 88,
+      "members": [
+        { "name": "ptrundef", "staticSize": 8 },
+        { "name": "__makePad1", "staticSize": 1 },
+        { "name": "shundef", "staticSize": 16 },
+        { "name": "__makePad2", "staticSize": 1 },
+        { "name": "shoptundef",
+          "staticSize": 24,
+          "length": 0,
+          "capacity": 1
+        },
+        { "name": "__makePad3", "staticSize": 1 },
+        { "name": "optundef",
+          "staticSize": 16,
+          "length": 0,
+          "capacity": 1
+        }
+      ]
+    }]'''
diff --git a/test/test_drgn_parser.cpp b/test/test_drgn_parser.cpp
index e8d03e3..c70f5a9 100644
--- a/test/test_drgn_parser.cpp
+++ b/test/test_drgn_parser.cpp
@@ -426,8 +426,8 @@ TEST_F(DrgnParserTest, PointerNoFollow) {
 
 TEST_F(DrgnParserTest, PointerIncomplete) {
   test("oid_test_case_pointers_incomplete_raw", R"(
-[0] Pointer
-      Incomplete: [IncompleteType]
+[1] Pointer
+[0]   Incomplete: [IncompleteType]
 )");
 }
 
diff --git a/test/test_flattener.cpp b/test/test_flattener.cpp
index 808f4ee..c8944bf 100644
--- a/test/test_flattener.cpp
+++ b/test/test_flattener.cpp
@@ -1007,7 +1007,7 @@ TEST(FlattenerTest, IncompleteParent) {
        R"(
 [0] Class: MyClass (size: 4)
       Parent (offset: 0)
-        Incomplete: [IncompleteParent]
+[1]     Incomplete: [IncompleteParent]
 )",
        R"(
 [0] Class: MyClass (size: 4)
diff --git a/test/test_name_gen.cpp b/test/test_name_gen.cpp
index d0cda26..a83c180 100644
--- a/test/test_name_gen.cpp
+++ b/test/test_name_gen.cpp
@@ -501,3 +501,16 @@ TEST(NameGenTest, AnonymousMembers) {
   EXPECT_EQ(myclass.members[0].inputName, "__oi_anon_0");
   EXPECT_EQ(myclass.members[1].inputName, "__oi_anon_1");
 }
+
+TEST(NameGenTest, IncompleteTypes) {
+  auto myincompletevector = Incomplete{0, "std::vector<int>"};
+
+  auto myint = Primitive{Primitive::Kind::Int32};
+  auto myincompleteint = Incomplete{1, myint};
+
+  NameGen nameGen;
+  nameGen.generateNames({myincompletevector, myincompleteint});
+
+  EXPECT_EQ(myincompletevector.name(), "Incomplete<struct std__vector_int_>");
+  EXPECT_EQ(myincompleteint.name(), "Incomplete<struct int32_t>");
+}
diff --git a/test/test_remove_members.cpp b/test/test_remove_members.cpp
index 0d4a54e..9eff4ed 100644
--- a/test/test_remove_members.cpp
+++ b/test/test_remove_members.cpp
@@ -191,7 +191,7 @@ TEST(RemoveMembersTest, Incomplete) {
       Member: a (offset: 0)
         Primitive: int32_t
       Member: b (offset: 4)
-        Incomplete: [MyIncompleteType]
+[1]     Incomplete: [MyIncompleteType]
       Member: c (offset: 8)
         Primitive: int32_t
 )",
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index bb3a622..3439bf3 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
         if (s_ptr && ctx.pointers.add((uintptr_t)(s_ptr.get()))) {
@@ -37,7 +37,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (!oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
@@ -85,7 +85,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index f723f80..f71cff4 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T,Deleter>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
         if (u_ptr && ctx.pointers.add((uintptr_t)(u_ptr.get()))) {
@@ -38,7 +38,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (!oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
@@ -71,7 +71,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);

From 5c2b41e4bb1a51fbcb90fe4ac222fabac98d5b8c Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 4 Jan 2024 17:01:52 +0000
Subject: [PATCH 113/188] incomplete: name type in compiler errors

Summary:

We have a good type representation in the Type Graph of an incomplete type and
the underlying type that represents. However, this incomplete type still ends
up in the generated code as `void` which loses information. For example, a
container that can't contain void may fail to compile because it was
initialised with `void` but really its because the type it was supposed to be
initialised with (say, `Foo`) had incomplete debug information.

This change identifies that a type is incomplete in the output by generating it
as an incomplete type `struct Incomplete<struct Foo>`. This allows us to name
the type correctly in the TreeBuilder output and filter for incomplete types,
as well as getting appropriate compiler errors if it mustn't be incomplete.

Test Plan:
- CI
- Added a unit test to namegen.
- Enabled and added an extra pointers_incomplete test.

This change is tricky to test because it isn't really user visible. The types
still use their `inputName` which is unchanged in any successful output - this
change is used so the compiler fails with a more detailed error.
---
 oi/CodeGen.cpp                            | 31 +++++++++---------
 oi/OITraceCode.cpp                        | 16 ++++++++++
 oi/type_graph/DrgnExporter.cpp            |  2 ++
 oi/type_graph/NameGen.cpp                 | 23 +++++++++++++
 oi/type_graph/NameGen.h                   |  1 +
 oi/type_graph/Printer.cpp                 |  4 ++-
 oi/type_graph/TopoSorter.cpp              |  4 +++
 oi/type_graph/TopoSorter.h                |  1 +
 oi/type_graph/Types.cpp                   |  2 --
 oi/type_graph/Types.h                     | 39 +++++++++++++++--------
 test/TypeGraphParser.cpp                  |  5 +--
 test/integration/pointers_incomplete.toml | 32 ++++++++++++++++---
 test/test_drgn_parser.cpp                 |  4 +--
 test/test_flattener.cpp                   |  2 +-
 test/test_name_gen.cpp                    | 13 ++++++++
 test/test_remove_members.cpp              |  2 +-
 types/shrd_ptr_type.toml                  |  6 ++--
 types/uniq_ptr_type.toml                  |  6 ++--
 18 files changed, 144 insertions(+), 49 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 9614a1f..84fb9b4 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -60,6 +60,7 @@ using type_graph::EnforceCompatibility;
 using type_graph::Enum;
 using type_graph::Flattener;
 using type_graph::IdentifyContainers;
+using type_graph::Incomplete;
 using type_graph::KeyCapture;
 using type_graph::Member;
 using type_graph::NameGen;
@@ -415,8 +416,6 @@ void addStandardGetSizeFuncDecls(std::string& code) {
     template<typename T>
     void getSizeType(/*const*/ T* s_ptr, size_t& returnArg);
 
-    void getSizeType(/*const*/ void *s_ptr, size_t& returnArg);
-
     template <typename T, int N>
     void getSizeType(const OIArray<T,N>& container, size_t& returnArg);
   )";
@@ -438,24 +437,24 @@ void addStandardGetSizeFuncDefs(std::string& code) {
     template<typename T>
     void getSizeType(/*const*/ T* s_ptr, size_t& returnArg)
     {
-      JLOG("ptr val @");
-      JLOGPTR(s_ptr);
-      StoreData((uintptr_t)(s_ptr), returnArg);
-      if (s_ptr && ctx.pointers.add((uintptr_t)s_ptr)) {
-        StoreData(1, returnArg);
-        getSizeType(*(s_ptr), returnArg);
+      if constexpr (!oi_is_complete<T>) {
+        JLOG("incomplete ptr @");
+        JLOGPTR(s_ptr);
+        StoreData((uintptr_t)(s_ptr), returnArg);
+        return;
       } else {
-        StoreData(0, returnArg);
+        JLOG("ptr val @");
+        JLOGPTR(s_ptr);
+        StoreData((uintptr_t)(s_ptr), returnArg);
+        if (s_ptr && ctx.pointers.add((uintptr_t)s_ptr)) {
+          StoreData(1, returnArg);
+          getSizeType(*(s_ptr), returnArg);
+        } else {
+          StoreData(0, returnArg);
+        }
       }
     }
 
-    void getSizeType(/*const*/ void *s_ptr, size_t& returnArg)
-    {
-      JLOG("void ptr @");
-      JLOGPTR(s_ptr);
-      StoreData((uintptr_t)(s_ptr), returnArg);
-    }
-
     template <typename T, int N>
     void getSizeType(const OIArray<T,N>& container, size_t& returnArg)
     {
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index 31a873b..a100900 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -182,3 +182,19 @@ bool isStorageInline(const auto& c) {
   return (uintptr_t)std::data(c) < (uintptr_t)(&c + sizeof(c)) &&
          (uintptr_t)std::data(c) >= (uintptr_t)&c;
 }
+
+namespace OIInternal {
+namespace {
+
+template <typename T>
+struct Incomplete;
+
+template <typename T>
+constexpr bool oi_is_complete = true;
+template <>
+constexpr bool oi_is_complete<void> = false;
+template <typename T>
+constexpr bool oi_is_complete<Incomplete<T>> = false;
+
+}  // namespace
+}  // namespace OIInternal
diff --git a/oi/type_graph/DrgnExporter.cpp b/oi/type_graph/DrgnExporter.cpp
index df9722d..39292e5 100644
--- a/oi/type_graph/DrgnExporter.cpp
+++ b/oi/type_graph/DrgnExporter.cpp
@@ -100,6 +100,8 @@ drgn_type* DrgnExporter::visit(Container& c) {
     if (auto* p = dynamic_cast<const Primitive*>(&paramType);
         p && p->kind() == Primitive::Kind::Void) {
       return drgnType;
+    } else if (auto* i = dynamic_cast<const Incomplete*>(&paramType)) {
+      return drgnType;
     }
   }
 
diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp
index 725ab59..0cc2a61 100644
--- a/oi/type_graph/NameGen.cpp
+++ b/oi/type_graph/NameGen.cpp
@@ -219,4 +219,27 @@ void NameGen::visit(CaptureKeys& c) {
   c.regenerateName();
 }
 
+void NameGen::visit(Incomplete& i) {
+  constexpr std::string_view kPrefix{"Incomplete<struct "};
+
+  RecursiveVisitor::visit(i);
+
+  std::string_view inputName = i.inputName();
+
+  std::string name;
+  name.reserve(kPrefix.size() + inputName.size() + 2);
+  name = kPrefix;
+
+  for (const unsigned char c : inputName) {
+    if (std::isalnum(c)) {
+      name += c;
+    } else {
+      name += '_';
+    }
+  }
+  name += ">";
+
+  i.setName(name);
+}
+
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/NameGen.h b/oi/type_graph/NameGen.h
index 4f1c258..927cef4 100644
--- a/oi/type_graph/NameGen.h
+++ b/oi/type_graph/NameGen.h
@@ -49,6 +49,7 @@ class NameGen final : public RecursiveVisitor {
   void visit(Reference& r) override;
   void visit(DummyAllocator& d) override;
   void visit(CaptureKeys& d) override;
+  void visit(Incomplete& i) override;
 
   static const inline std::string AnonPrefix = "__oi_anon";
 
diff --git a/oi/type_graph/Printer.cpp b/oi/type_graph/Printer.cpp
index f598161..d8bac7c 100644
--- a/oi/type_graph/Printer.cpp
+++ b/oi/type_graph/Printer.cpp
@@ -37,7 +37,9 @@ void Printer::print(const Type& type) {
 }
 
 void Printer::visit(const Incomplete& i) {
-  prefix();
+  if (prefix(i))
+    return;
+
   out_ << "Incomplete";
   if (auto underlyingType = i.underlyingType()) {
     out_ << std::endl;
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index 6ca10af..ff2715a 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -139,6 +139,10 @@ void TopoSorter::visit(CaptureKeys& c) {
   sortedTypes_.push_back(c);
 }
 
+void TopoSorter::visit(Incomplete& i) {
+  sortedTypes_.push_back(i);
+}
+
 /*
  * A type graph may contain cycles, so we need to slightly tweak the standard
  * topological sorting algorithm. Cycles can only be introduced by certain
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 374e943..44f2364 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -48,6 +48,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Pointer& p) override;
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
+  void visit(Incomplete& i) override;
 
  private:
   std::unordered_set<Type*> visited_;
diff --git a/oi/type_graph/Types.cpp b/oi/type_graph/Types.cpp
index c6219b5..9b53f49 100644
--- a/oi/type_graph/Types.cpp
+++ b/oi/type_graph/Types.cpp
@@ -35,8 +35,6 @@ namespace oi::detail::type_graph {
 OI_TYPE_LIST
 #undef X
 
-const std::string Incomplete::kName = "void";
-
 std::string Primitive::getName(Kind kind) {
   switch (kind) {
     case Kind::Int8:
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index 1d5dd79..f09f926 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -29,6 +29,8 @@
  * debugging.
  */
 
+#include <algorithm>
+#include <cctype>
 #include <cstddef>
 #include <map>
 #include <optional>
@@ -211,29 +213,33 @@ struct TemplateParam {
  */
 class Incomplete : public Type {
  public:
-  Incomplete(Type& underlyingType) : underlyingType_(underlyingType) {
+  Incomplete(NodeId id, Type& underlyingType)
+      : id_(id), underlyingType_(underlyingType) {
   }
 
-  Incomplete(std::string underlyingTypeName)
-      : underlyingType_(std::move(underlyingTypeName)) {
+  Incomplete(NodeId id, std::string underlyingTypeName)
+      : id_(id), underlyingType_(std::move(underlyingTypeName)) {
   }
 
-  static inline constexpr bool has_node_id = false;
+  static inline constexpr bool has_node_id = true;
 
   DECLARE_ACCEPT
 
   const std::string& name() const override {
-    return kName;
+    return name_;
   }
 
   std::string_view inputName() const override {
-    if (std::holds_alternative<std::string>(underlyingType_)) {
-      return std::get<std::string>(underlyingType_);
-    }
-
-    return std::get<std::reference_wrapper<Type>>(underlyingType_)
-        .get()
-        .inputName();
+    return std::visit(
+        [](const auto& el) -> std::string_view {
+          using T = std::decay_t<decltype(el)>;
+          if constexpr (std::is_same_v<T, std::string>) {
+            return el;
+          } else {
+            return el.get().inputName();
+          }
+        },
+        underlyingType_);
   }
 
   size_t size() const override {
@@ -245,7 +251,11 @@ class Incomplete : public Type {
   }
 
   NodeId id() const override {
-    return -1;
+    return id_;
+  }
+
+  void setName(std::string name) {
+    name_ = std::move(name);
   }
 
   std::optional<std::reference_wrapper<Type>> underlyingType() const {
@@ -257,8 +267,9 @@ class Incomplete : public Type {
   }
 
  private:
+  NodeId id_ = -1;
   std::variant<std::string, std::reference_wrapper<Type>> underlyingType_;
-  static const std::string kName;
+  std::string name_ = "void";
 };
 
 /*
diff --git a/test/TypeGraphParser.cpp b/test/TypeGraphParser.cpp
index f54fe08..d70236c 100644
--- a/test/TypeGraphParser.cpp
+++ b/test/TypeGraphParser.cpp
@@ -215,10 +215,11 @@ Type& TypeGraphParser::parseType(std::string_view& input, size_t rootIndent) {
       auto nameEndPos = line.find(']', nameStartPos);
       auto underlyingTypeName =
           line.substr(nameStartPos, nameEndPos - nameStartPos);
-      type = &typeGraph_.makeType<Incomplete>(std::string(underlyingTypeName));
+      type =
+          &typeGraph_.makeType<Incomplete>(id, std::string(underlyingTypeName));
     } else {
       auto& underlyingType = parseType(input, indent + 2);
-      type = &typeGraph_.makeType<Incomplete>(underlyingType);
+      type = &typeGraph_.makeType<Incomplete>(id, underlyingType);
     }
   } else if (nodeTypeName == "Class" || nodeTypeName == "Struct" ||
              nodeTypeName == "Union") {
diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml
index 0690511..2d34865 100644
--- a/test/integration/pointers_incomplete.toml
+++ b/test/integration/pointers_incomplete.toml
@@ -55,7 +55,6 @@ definitions = '''
     }]'''
 
   [cases.unique_ptr]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299
     param_types = ["const std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>&"]
     setup = '''
       auto raw_ptr = static_cast<IncompleteType*>(::operator new(5));
@@ -63,28 +62,29 @@ definitions = '''
         raw_ptr, &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
   [cases.unique_ptr_null]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299
     param_types = ["const std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>&"]
     setup = '''
       return std::unique_ptr<IncompleteType, decltype(&incomplete_type_deleter)>(
         nullptr, &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.shared_ptr]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300
     param_types = ["const std::shared_ptr<IncompleteType>&"]
     setup = '''
       auto raw_ptr = static_cast<IncompleteType*>(::operator new(5));
       return std::shared_ptr<IncompleteType>(raw_ptr , &incomplete_type_deleter);
     '''
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
   [cases.shared_ptr_null]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300
     param_types = ["const std::shared_ptr<IncompleteType>"]
     setup = "return nullptr;"
     expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]'
+    expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.containing_struct]
     oil_disable = "oil can't chase raw pointers safely"
@@ -116,3 +116,27 @@ definitions = '''
         }
       ]
     }]'''
+
+  [cases.containing_struct_no_follow]
+    param_types = ["const IncompleteTypeContainer&"]
+    setup = "return IncompleteTypeContainer{};"
+    expect_json = '''[{
+      "staticSize": 88,
+      "members": [
+        { "name": "ptrundef", "staticSize": 8 },
+        { "name": "__makePad1", "staticSize": 1 },
+        { "name": "shundef", "staticSize": 16 },
+        { "name": "__makePad2", "staticSize": 1 },
+        { "name": "shoptundef",
+          "staticSize": 24,
+          "length": 0,
+          "capacity": 1
+        },
+        { "name": "__makePad3", "staticSize": 1 },
+        { "name": "optundef",
+          "staticSize": 16,
+          "length": 0,
+          "capacity": 1
+        }
+      ]
+    }]'''
diff --git a/test/test_drgn_parser.cpp b/test/test_drgn_parser.cpp
index e8d03e3..c70f5a9 100644
--- a/test/test_drgn_parser.cpp
+++ b/test/test_drgn_parser.cpp
@@ -426,8 +426,8 @@ TEST_F(DrgnParserTest, PointerNoFollow) {
 
 TEST_F(DrgnParserTest, PointerIncomplete) {
   test("oid_test_case_pointers_incomplete_raw", R"(
-[0] Pointer
-      Incomplete: [IncompleteType]
+[1] Pointer
+[0]   Incomplete: [IncompleteType]
 )");
 }
 
diff --git a/test/test_flattener.cpp b/test/test_flattener.cpp
index 808f4ee..c8944bf 100644
--- a/test/test_flattener.cpp
+++ b/test/test_flattener.cpp
@@ -1007,7 +1007,7 @@ TEST(FlattenerTest, IncompleteParent) {
        R"(
 [0] Class: MyClass (size: 4)
       Parent (offset: 0)
-        Incomplete: [IncompleteParent]
+[1]     Incomplete: [IncompleteParent]
 )",
        R"(
 [0] Class: MyClass (size: 4)
diff --git a/test/test_name_gen.cpp b/test/test_name_gen.cpp
index d0cda26..a83c180 100644
--- a/test/test_name_gen.cpp
+++ b/test/test_name_gen.cpp
@@ -501,3 +501,16 @@ TEST(NameGenTest, AnonymousMembers) {
   EXPECT_EQ(myclass.members[0].inputName, "__oi_anon_0");
   EXPECT_EQ(myclass.members[1].inputName, "__oi_anon_1");
 }
+
+TEST(NameGenTest, IncompleteTypes) {
+  auto myincompletevector = Incomplete{0, "std::vector<int>"};
+
+  auto myint = Primitive{Primitive::Kind::Int32};
+  auto myincompleteint = Incomplete{1, myint};
+
+  NameGen nameGen;
+  nameGen.generateNames({myincompletevector, myincompleteint});
+
+  EXPECT_EQ(myincompletevector.name(), "Incomplete<struct std__vector_int_>");
+  EXPECT_EQ(myincompleteint.name(), "Incomplete<struct int32_t>");
+}
diff --git a/test/test_remove_members.cpp b/test/test_remove_members.cpp
index 0d4a54e..9eff4ed 100644
--- a/test/test_remove_members.cpp
+++ b/test/test_remove_members.cpp
@@ -191,7 +191,7 @@ TEST(RemoveMembersTest, Incomplete) {
       Member: a (offset: 0)
         Primitive: int32_t
       Member: b (offset: 4)
-        Incomplete: [MyIncompleteType]
+[1]     Incomplete: [MyIncompleteType]
       Member: c (offset: 8)
         Primitive: int32_t
 )",
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index bb3a622..3439bf3 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -21,7 +21,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(s_ptr.get()));
 
         if (s_ptr && ctx.pointers.add((uintptr_t)(s_ptr.get()))) {
@@ -37,7 +37,7 @@ void getSizeType(const %1%<T> &s_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (!oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
@@ -85,7 +85,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index f723f80..f71cff4 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -22,7 +22,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 {
     SAVE_SIZE(sizeof(%1%<T,Deleter>));
 
-    if constexpr (!std::is_void<T>::value) {
+    if constexpr (oi_is_complete<T>) {
         SAVE_DATA((uintptr_t)(u_ptr.get()));
 
         if (u_ptr && ctx.pointers.add((uintptr_t)(u_ptr.get()))) {
@@ -38,7 +38,7 @@ void getSizeType(const %1%<T,Deleter> &u_ptr, size_t& returnArg)
 traversal_func = """
 auto tail = returnArg.write((uintptr_t)container.get());
 
-if constexpr (std::is_void<T0>::value) {
+if constexpr (!oi_is_complete<T0>) {
   return tail.template delegate<0>(std::identity());
 } else {
   bool do_visit = container && ctx.pointers.add((uintptr_t)container.get());
@@ -71,7 +71,7 @@ el.container_stats.emplace(result::Element::ContainerStats {
 });
 
 // Must be in a `if constexpr` or the compiler will complain about make_field<Ctx, void>
-if constexpr (!std::is_void<T0>::value) {
+if constexpr (oi_is_complete<T0>) {
   if (sum.index == 1) {
     static constexpr auto element = make_field<Ctx, T0>("ptr_val");
     stack_ins(element);

From 08f173236bd604345b1bbebf06ab89c3c6875538 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 9 Jan 2024 15:32:37 +0000
Subject: [PATCH 114/188] tbv2: update tuple test

This test is a bit odd, but this change adds the full set of size/member checks
for the hierarchy for TreeBuilder v2 and enables it.

Closes #304

Test Plan:
- CI
---
 test/integration/std_tuple.toml | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/test/integration/std_tuple.toml b/test/integration/std_tuple.toml
index 4d63064..1e64375 100644
--- a/test/integration/std_tuple.toml
+++ b/test/integration/std_tuple.toml
@@ -9,7 +9,6 @@ struct Bar {
 '''
 [cases]
   [cases.uint64_uint64]
-    oil_skip = "std::tuple is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/304
     param_types = ["Bar&"]
     setup = '''
       Foo f;
@@ -35,3 +34,15 @@ struct Bar {
       }
     ]
     '''
+    expect_json_v2 = '''[
+      { "size":24, "staticSize":24, "exclusiveSize":0, "members":[
+        {"size":24, "staticSize":24, "exclusiveSize":8, "members":[
+          {"size":16, "staticSize":16, "exclusiveSize":0, "members": [
+            {"size":16, "staticSize":16, "exclusiveSize":0, "members": [
+              {"size":8, "staticSize":8, "exclusiveSize":8, "members":[]},
+              {"size":8, "staticSize":8, "exclusiveSize":8, "members":[]}
+            ]}
+          ]}
+        ]}
+      ]}
+    ]'''

From d275714eb3d32bc81063facebab718428f237b13 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 9 Jan 2024 15:49:37 +0000
Subject: [PATCH 115/188] tbv2: add support for std::reference_wrapper

Closes #307

Test plan:
- CI
- Updated and enabled tests.
---
 test/integration/std_reference_wrapper.toml | 24 +++++++++++--
 types/ref_wrapper_type.toml                 | 39 +++++++++++++++++++++
 2 files changed, 61 insertions(+), 2 deletions(-)

diff --git a/test/integration/std_reference_wrapper.toml b/test/integration/std_reference_wrapper.toml
index bb3dd52..feff729 100644
--- a/test/integration/std_reference_wrapper.toml
+++ b/test/integration/std_reference_wrapper.toml
@@ -1,12 +1,14 @@
 includes = ["functional"]
 [cases]
   [cases.int]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/307
     param_types = ["const std::reference_wrapper<int>&"]
     setup = "return std::ref(*new int(1));"
     expect_json = '[{"staticSize":8, "dynamicSize":4, "length":1, "capacity":1, "elementStaticSize":4}]'
+    expect_json_v2 = '''[{
+      "size":12, "staticSize":8, "exclusiveSize":8, "length":1, "capacity":1, "members":[
+        {"size":4, "staticSize":4, "exclusiveSize":4, "members":[]}
+    ]}]'''
   [cases.vector]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/307
     param_types = ["const std::vector<std::reference_wrapper<int>>&"]
     setup = "return {{std::ref(*new int(1)), std::ref(*new int(2)), std::ref(*new int(3))}};"
     expect_json = '''[{
@@ -20,3 +22,21 @@ includes = ["functional"]
         {"staticSize":8, "dynamicSize":4, "length":1, "capacity":1, "elementStaticSize":4},
         {"staticSize":8, "dynamicSize":4, "length":1, "capacity":1, "elementStaticSize":4}
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":60,
+      "staticSize":24,
+      "exclusiveSize":24,
+      "length":3,
+      "capacity":3,
+      "members":[
+        {"size":12, "staticSize":8, "exclusiveSize":8, "length":1, "capacity":1, "members":[
+          {"size":4, "staticSize":4, "exclusiveSize":4, "members":[]}
+        ]},
+        {"size":12, "staticSize":8, "exclusiveSize":8, "length":1, "capacity":1, "members":[
+          {"size":4, "staticSize":4, "exclusiveSize":4, "members":[]}
+        ]},
+        {"size":12, "staticSize":8, "exclusiveSize":8, "length":1, "capacity":1, "members":[
+          {"size":4, "staticSize":4, "exclusiveSize":4, "members":[]}
+        ]}
+      ]
+    }]'''
diff --git a/types/ref_wrapper_type.toml b/types/ref_wrapper_type.toml
index 9ed5094..380ff84 100644
--- a/types/ref_wrapper_type.toml
+++ b/types/ref_wrapper_type.toml
@@ -29,3 +29,42 @@ void getSizeType(const %1%<T> &ref, size_t& returnArg)
     }
 }
 """
+
+traversal_func = """
+auto tail = returnArg.write((uintptr_t)&(container.get()));
+
+if (ctx.pointers.add((uintptr_t)&container.get())) {
+  return tail.template delegate<1>([&ctx, &container](auto ret) {
+    return OIInternal::getSizeType<Ctx>(ctx, container.get(), ret);
+  });
+} else {
+  return tail.template delegate<0>(std::identity());
+}
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.pointer = std::get<ParsedData::VarInt>(d.val).value;
+"""
+
+[[codegen.processor]]
+type = """
+types::st::Sum<DB,
+  types::st::Unit<DB>,
+  typename TypeHandler<Ctx, T0>::type>
+"""
+func = """
+auto sum = std::get<ParsedData::Sum>(d.val);
+el.container_stats.emplace(result::Element::ContainerStats {
+  .capacity = 1,
+  .length = sum.index, // 0 if in a cycle, 1 otherwise
+});
+
+if constexpr (oi_is_complete<T0>) {
+  if (sum.index == 1) {
+    static constexpr auto element = make_field<Ctx, T0>("ref_val");
+    stack_ins(element);
+  }
+}
+"""

From 83223311813f51bb2b2dabbdcab13539a9e75cf5 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 10 Jan 2024 14:02:29 +0000
Subject: [PATCH 116/188] tbv2: add is_primitive to output

C++ has a concept of Primitive which holds in the type graph. However we don't
currently expose this information to the end user. Expose this from the OIL
iterator to allow future features like primitive rollups.

Test Plan:
- CI
- Updated simple tests.
---
 include/oi/exporters/Json.h  |  5 +++--
 include/oi/exporters/inst.h  | 16 +++++++++++-----
 include/oi/result/Element.h  |  1 +
 oi/CodeGen.cpp               |  6 +++++-
 oi/FuncGen.cpp               |  3 ++-
 oi/IntrospectionResult.cpp   |  1 +
 test/integration/simple.toml | 17 ++++++++++-------
 7 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h
index dd88ade..3dd37f4 100644
--- a/include/oi/exporters/Json.h
+++ b/include/oi/exporters/Json.h
@@ -98,8 +98,8 @@ inline void Json::printStringField(std::string_view name,
 inline void Json::printBoolField(std::string_view name,
                                  bool value,
                                  std::string_view indent) {
-  out_ << tab() << '"' << name << "\":" << space() << value << ',' << endl()
-       << indent;
+  out_ << tab() << '"' << name << "\":" << space() << (value ? "true" : "false")
+       << ',' << endl() << indent;
 }
 inline void Json::printUnsignedField(std::string_view name,
                                      uint64_t value,
@@ -159,6 +159,7 @@ inline void Json::printFields(const result::Element& el,
   }
   if (el.is_set_stats.has_value())
     printUnsignedField("is_set", el.is_set_stats->is_set, indent);
+  printBoolField("is_primitive", el.is_primitive, indent);
 }
 
 template <typename It>
diff --git a/include/oi/exporters/inst.h b/include/oi/exporters/inst.h
index 8397cd6..5484376 100644
--- a/include/oi/exporters/inst.h
+++ b/include/oi/exporters/inst.h
@@ -58,19 +58,22 @@ struct Field {
                   std::string_view name_,
                   const std::array<std::string_view, N0>& type_names_,
                   const std::array<Field, N1>& fields_,
-                  const std::array<ProcessorInst, N2>& processors_);
+                  const std::array<ProcessorInst, N2>& processors_,
+                  bool is_primitive_);
   template <size_t N0, size_t N1, size_t N2>
   constexpr Field(size_t static_size_,
                   std::string_view name_,
                   const std::array<std::string_view, N0>& type_names_,
                   const std::array<Field, N1>& fields_,
-                  const std::array<ProcessorInst, N2>& processors_)
+                  const std::array<ProcessorInst, N2>& processors_,
+                  bool is_primitive_)
       : Field(static_size_,
               static_size_,
               name_,
               type_names_,
               fields_,
-              processors_) {
+              processors_,
+              is_primitive_) {
   }
   constexpr Field(const Field&) = default;  // no idea why this is needed
 
@@ -80,6 +83,7 @@ struct Field {
   std::span<const std::string_view> type_names;
   std::span<const Field> fields;
   std::span<const ProcessorInst> processors;
+  bool is_primitive;
 };
 
 template <size_t N0, size_t N1, size_t N2>
@@ -88,13 +92,15 @@ constexpr Field::Field(size_t static_size_,
                        std::string_view name_,
                        const std::array<std::string_view, N0>& type_names_,
                        const std::array<Field, N1>& fields_,
-                       const std::array<ProcessorInst, N2>& processors_)
+                       const std::array<ProcessorInst, N2>& processors_,
+                       bool is_primitive_)
     : static_size(static_size_),
       exclusive_size(exclusive_size_),
       name(name_),
       type_names(type_names_),
       fields(fields_),
-      processors(processors_) {
+      processors(processors_),
+      is_primitive(is_primitive_) {
 }
 
 }  // namespace oi::exporters::inst
diff --git a/include/oi/result/Element.h b/include/oi/result/Element.h
index 1eea86b..c971236 100644
--- a/include/oi/result/Element.h
+++ b/include/oi/result/Element.h
@@ -52,6 +52,7 @@ struct Element {
       std::nullopt};
   std::optional<ContainerStats> container_stats;
   std::optional<IsSetStats> is_set_stats;
+  bool is_primitive;
 };
 
 }  // namespace oi::result
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 84fb9b4..8dc884e 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -798,12 +798,15 @@ void CodeGen::genClassTreeBuilderInstructions(const Class& c,
     if (m.name.starts_with(AddPadding::MemberPrefix))
       continue;
     std::string fullName = c.name() + "::" + m.name;
+    bool isPrimitive = dynamic_cast<const Primitive*>(&m.type());
     code += "      inst::Field{sizeof(" + fullName + "), " +
             std::to_string(calculateExclusiveSize(m.type())) + ",\"" +
             m.inputName + "\", member_" + std::to_string(index) +
             "_type_names, TypeHandler<Ctx, decltype(" + fullName +
             ")>::fields, TypeHandler<Ctx, decltype(" + fullName +
-            ")>::processors},\n";
+            ")>::processors, ";
+    code += isPrimitive ? "true" : "false";
+    code += "},\n";
   }
   code += "  };\n";
   code +=
@@ -1074,6 +1077,7 @@ constexpr inst::Field make_field(std::string_view name) {
     NameProvider<T>::names,
     TypeHandler<Ctx, T>::fields,
     TypeHandler<Ctx, T>::processors,
+    std::is_fundamental_v<T>,
   };
 }
 )";
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 0b1e293..5bacc71 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -399,7 +399,8 @@ const std::array<std::string_view, )";
       ", OIInternal::TypeHandler<FakeContext, "
       "OIInternal::__ROOT_TYPE__>::fields, "
       "OIInternal::TypeHandler<FakeContext, "
-      "OIInternal::__ROOT_TYPE__>::processors};\n";
+      "OIInternal::__ROOT_TYPE__>::processors, "
+      "std::is_fundamental_v<OIInternal::__ROOT_TYPE__>};\n";
   code += "} // namespace\n";
   code +=
       "extern const exporters::inst::Inst __attribute__((used, retain)) "
diff --git a/oi/IntrospectionResult.cpp b/oi/IntrospectionResult.cpp
index bc59f5e..5310001 100644
--- a/oi/IntrospectionResult.cpp
+++ b/oi/IntrospectionResult.cpp
@@ -78,6 +78,7 @@ IntrospectionResult::const_iterator::operator++() {
                 .exclusive_size = ty.exclusive_size,
                 .container_stats = std::nullopt,
                 .is_set_stats = std::nullopt,
+                .is_primitive = ty.is_primitive,
             };
 
             for (const auto& [dy, handler] : ty.processors) {
diff --git a/test/integration/simple.toml b/test/integration/simple.toml
index 0945f70..76e0f20 100644
--- a/test/integration/simple.toml
+++ b/test/integration/simple.toml
@@ -25,10 +25,11 @@ definitions = '''
       "dynamicSize":0,
       "exclusiveSize": 3,
       "size": 16,
+      "is_primitive": false,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
-        {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
-        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4, "is_primitive": true},
+        {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1, "is_primitive": true},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8, "is_primitive": true}
       ]}]'''
   [cases.class]
     param_types = ["const SimpleClass&"]
@@ -38,10 +39,11 @@ definitions = '''
       "dynamicSize":0,
       "exclusiveSize": 3,
       "size": 16,
+      "is_primitive": false,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
-        {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
-        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4, "is_primitive": true},
+        {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1, "is_primitive": true},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8, "is_primitive": true}
       ]}]'''
   [cases.union]
     param_types = ["const SimpleUnion&"]
@@ -50,5 +52,6 @@ definitions = '''
       "staticSize":8,
       "dynamicSize":0,
       "exclusiveSize":8,
-      "size":8
+      "size":8,
+      "is_primitive": false
     }]'''

From 609d8605ad6ae25ad5cc0a8ec98a66e5c760d141 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 10 Jan 2024 14:02:29 +0000
Subject: [PATCH 117/188] tbv2: add is_primitive to output

C++ has a concept of Primitive which holds in the type graph. However we don't
currently expose this information to the end user. Expose this from the OIL
iterator to allow future features like primitive rollups.

Test Plan:
- CI
- Updated simple tests.
---
 include/oi/exporters/Json.h  |  5 +++--
 include/oi/exporters/inst.h  | 16 +++++++++++-----
 include/oi/result/Element.h  |  1 +
 oi/CodeGen.cpp               |  6 +++++-
 oi/FuncGen.cpp               | 19 ++++--------------
 oi/IntrospectionResult.cpp   |  1 +
 test/integration/simple.toml | 37 ++++++++++++++++++++++++++++++------
 7 files changed, 56 insertions(+), 29 deletions(-)

diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h
index dd88ade..3dd37f4 100644
--- a/include/oi/exporters/Json.h
+++ b/include/oi/exporters/Json.h
@@ -98,8 +98,8 @@ inline void Json::printStringField(std::string_view name,
 inline void Json::printBoolField(std::string_view name,
                                  bool value,
                                  std::string_view indent) {
-  out_ << tab() << '"' << name << "\":" << space() << value << ',' << endl()
-       << indent;
+  out_ << tab() << '"' << name << "\":" << space() << (value ? "true" : "false")
+       << ',' << endl() << indent;
 }
 inline void Json::printUnsignedField(std::string_view name,
                                      uint64_t value,
@@ -159,6 +159,7 @@ inline void Json::printFields(const result::Element& el,
   }
   if (el.is_set_stats.has_value())
     printUnsignedField("is_set", el.is_set_stats->is_set, indent);
+  printBoolField("is_primitive", el.is_primitive, indent);
 }
 
 template <typename It>
diff --git a/include/oi/exporters/inst.h b/include/oi/exporters/inst.h
index 8397cd6..5484376 100644
--- a/include/oi/exporters/inst.h
+++ b/include/oi/exporters/inst.h
@@ -58,19 +58,22 @@ struct Field {
                   std::string_view name_,
                   const std::array<std::string_view, N0>& type_names_,
                   const std::array<Field, N1>& fields_,
-                  const std::array<ProcessorInst, N2>& processors_);
+                  const std::array<ProcessorInst, N2>& processors_,
+                  bool is_primitive_);
   template <size_t N0, size_t N1, size_t N2>
   constexpr Field(size_t static_size_,
                   std::string_view name_,
                   const std::array<std::string_view, N0>& type_names_,
                   const std::array<Field, N1>& fields_,
-                  const std::array<ProcessorInst, N2>& processors_)
+                  const std::array<ProcessorInst, N2>& processors_,
+                  bool is_primitive_)
       : Field(static_size_,
               static_size_,
               name_,
               type_names_,
               fields_,
-              processors_) {
+              processors_,
+              is_primitive_) {
   }
   constexpr Field(const Field&) = default;  // no idea why this is needed
 
@@ -80,6 +83,7 @@ struct Field {
   std::span<const std::string_view> type_names;
   std::span<const Field> fields;
   std::span<const ProcessorInst> processors;
+  bool is_primitive;
 };
 
 template <size_t N0, size_t N1, size_t N2>
@@ -88,13 +92,15 @@ constexpr Field::Field(size_t static_size_,
                        std::string_view name_,
                        const std::array<std::string_view, N0>& type_names_,
                        const std::array<Field, N1>& fields_,
-                       const std::array<ProcessorInst, N2>& processors_)
+                       const std::array<ProcessorInst, N2>& processors_,
+                       bool is_primitive_)
     : static_size(static_size_),
       exclusive_size(exclusive_size_),
       name(name_),
       type_names(type_names_),
       fields(fields_),
-      processors(processors_) {
+      processors(processors_),
+      is_primitive(is_primitive_) {
 }
 
 }  // namespace oi::exporters::inst
diff --git a/include/oi/result/Element.h b/include/oi/result/Element.h
index 1eea86b..c971236 100644
--- a/include/oi/result/Element.h
+++ b/include/oi/result/Element.h
@@ -52,6 +52,7 @@ struct Element {
       std::nullopt};
   std::optional<ContainerStats> container_stats;
   std::optional<IsSetStats> is_set_stats;
+  bool is_primitive;
 };
 
 }  // namespace oi::result
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 84fb9b4..cb5bf54 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -798,12 +798,15 @@ void CodeGen::genClassTreeBuilderInstructions(const Class& c,
     if (m.name.starts_with(AddPadding::MemberPrefix))
       continue;
     std::string fullName = c.name() + "::" + m.name;
+    bool isPrimitive = dynamic_cast<const Primitive*>(&m.type());
     code += "      inst::Field{sizeof(" + fullName + "), " +
             std::to_string(calculateExclusiveSize(m.type())) + ",\"" +
             m.inputName + "\", member_" + std::to_string(index) +
             "_type_names, TypeHandler<Ctx, decltype(" + fullName +
             ")>::fields, TypeHandler<Ctx, decltype(" + fullName +
-            ")>::processors},\n";
+            ")>::processors, ";
+    code += isPrimitive ? "true" : "false";
+    code += "},\n";
   }
   code += "  };\n";
   code +=
@@ -1074,6 +1077,7 @@ constexpr inst::Field make_field(std::string_view name) {
     NameProvider<T>::names,
     TypeHandler<Ctx, T>::fields,
     TypeHandler<Ctx, T>::processors,
+    std::is_fundamental_v<T>
   };
 }
 )";
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 0b1e293..f2441e5 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -399,7 +399,8 @@ const std::array<std::string_view, )";
       ", OIInternal::TypeHandler<FakeContext, "
       "OIInternal::__ROOT_TYPE__>::fields, "
       "OIInternal::TypeHandler<FakeContext, "
-      "OIInternal::__ROOT_TYPE__>::processors};\n";
+      "OIInternal::__ROOT_TYPE__>::processors, "
+      "std::is_fundamental_v<OIInternal::__ROOT_TYPE__>};\n";
   code += "} // namespace\n";
   code +=
       "extern const exporters::inst::Inst __attribute__((used, retain)) "
@@ -640,13 +641,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
         }
         static void process_pointer_content(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
           static constexpr std::array<std::string_view, 1> names{"TODO"};
-          static constexpr auto childField = inst::Field{
-            sizeof(T),
-            "*",
-            names,
-            TypeHandler<Ctx, T>::fields,
-            TypeHandler<Ctx, T>::processors,
-          };
+          static constexpr auto childField = make_field<Ctx, T>("*");
 
           const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
 
@@ -743,13 +738,7 @@ return tail.finish();
       .type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>",
       .func = R"(
 static constexpr std::array<std::string_view, 1> names{"TODO"};
-static constexpr auto childField = inst::Field{
-  sizeof(T0),
-  "[]",
-  names,
-  TypeHandler<Ctx, T0>::fields,
-  TypeHandler<Ctx, T0>::processors,
-};
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 el.exclusive_size = 0;
 el.container_stats.emplace(result::Element::ContainerStats{ .capacity = N0, .length = N0 });
diff --git a/oi/IntrospectionResult.cpp b/oi/IntrospectionResult.cpp
index bc59f5e..5310001 100644
--- a/oi/IntrospectionResult.cpp
+++ b/oi/IntrospectionResult.cpp
@@ -78,6 +78,7 @@ IntrospectionResult::const_iterator::operator++() {
                 .exclusive_size = ty.exclusive_size,
                 .container_stats = std::nullopt,
                 .is_set_stats = std::nullopt,
+                .is_primitive = ty.is_primitive,
             };
 
             for (const auto& [dy, handler] : ty.processors) {
diff --git a/test/integration/simple.toml b/test/integration/simple.toml
index 0945f70..47142f5 100644
--- a/test/integration/simple.toml
+++ b/test/integration/simple.toml
@@ -24,12 +24,21 @@ definitions = '''
       "staticSize":16,
       "dynamicSize":0,
       "exclusiveSize": 3,
-      "size": 16,
       "members":[
         {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
         {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize": 3,
+      "size": 16,
+      "is_primitive": false,
+      "members":[
+        {"name":"a", "staticSize":4, "exclusiveSize": 4, "size": 4, "is_primitive": true},
+        {"name":"b", "staticSize":1, "exclusiveSize": 1, "size": 1, "is_primitive": true},
+        {"name":"c", "staticSize":8, "exclusiveSize": 8, "size": 8, "is_primitive": true}
+      ]}]'''
   [cases.class]
     param_types = ["const SimpleClass&"]
     setup = "return {};"
@@ -38,10 +47,21 @@ definitions = '''
       "dynamicSize":0,
       "exclusiveSize": 3,
       "size": 16,
+      "is_primitive": false,
       "members":[
-        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
-        {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
-        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
+        {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4, "is_primitive": true},
+        {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1, "is_primitive": true},
+        {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8, "is_primitive": true}
+      ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize": 3,
+      "size": 16,
+      "is_primitive": false,
+      "members":[
+        {"name":"a", "staticSize":4, "exclusiveSize": 4, "size": 4, "is_primitive": true},
+        {"name":"b", "staticSize":1, "exclusiveSize": 1, "size": 1, "is_primitive": true},
+        {"name":"c", "staticSize":8, "exclusiveSize": 8, "size": 8, "is_primitive": true}
       ]}]'''
   [cases.union]
     param_types = ["const SimpleUnion&"]
@@ -49,6 +69,11 @@ definitions = '''
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
-      "exclusiveSize":8,
-      "size":8
+      "exclusiveSize":8
+    }]'''
+    expect_json_v2 = '''[{
+      "staticSize":8,
+      "exclusiveSize":8,
+      "size":8,
+      "is_primitive":false
     }]'''

From b5571ecd84a7ef3e9eb1ff48e63b4158ca1e6e68 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 10 Jan 2024 15:59:26 +0000
Subject: [PATCH 118/188] tbv2: add is_primitive to output

C++ has a concept of Primitive which holds in the type graph. However we don't
currently expose this information to the end user. Expose this from the OIL
iterator to allow future features like primitive rollups.

Test Plan:
- CI
- Updated simple tests.
---
 include/oi/exporters/Json.h            |  5 +++--
 include/oi/exporters/inst.h            | 16 ++++++++-----
 include/oi/result/Element.h            |  1 +
 oi/CodeGen.cpp                         |  6 ++++-
 oi/FuncGen.cpp                         | 19 ++++------------
 oi/IntrospectionResult.cpp             |  1 +
 test/integration/simple.toml           | 31 ++++++++++++++++++++++----
 types/f14_fast_map.toml                |  1 +
 types/f14_node_map.toml                |  1 +
 types/f14_value_map.toml               |  1 +
 types/f14_vector_map.toml              |  1 +
 types/list_type.toml                   |  1 +
 types/map_seq_type.toml                |  1 +
 types/multi_map_type.toml              |  1 +
 types/std_map_type.toml                |  1 +
 types/std_unordered_map_type.toml      |  1 +
 types/std_unordered_multimap_type.toml |  1 +
 17 files changed, 62 insertions(+), 27 deletions(-)

diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h
index dd88ade..3dd37f4 100644
--- a/include/oi/exporters/Json.h
+++ b/include/oi/exporters/Json.h
@@ -98,8 +98,8 @@ inline void Json::printStringField(std::string_view name,
 inline void Json::printBoolField(std::string_view name,
                                  bool value,
                                  std::string_view indent) {
-  out_ << tab() << '"' << name << "\":" << space() << value << ',' << endl()
-       << indent;
+  out_ << tab() << '"' << name << "\":" << space() << (value ? "true" : "false")
+       << ',' << endl() << indent;
 }
 inline void Json::printUnsignedField(std::string_view name,
                                      uint64_t value,
@@ -159,6 +159,7 @@ inline void Json::printFields(const result::Element& el,
   }
   if (el.is_set_stats.has_value())
     printUnsignedField("is_set", el.is_set_stats->is_set, indent);
+  printBoolField("is_primitive", el.is_primitive, indent);
 }
 
 template <typename It>
diff --git a/include/oi/exporters/inst.h b/include/oi/exporters/inst.h
index 8397cd6..5484376 100644
--- a/include/oi/exporters/inst.h
+++ b/include/oi/exporters/inst.h
@@ -58,19 +58,22 @@ struct Field {
                   std::string_view name_,
                   const std::array<std::string_view, N0>& type_names_,
                   const std::array<Field, N1>& fields_,
-                  const std::array<ProcessorInst, N2>& processors_);
+                  const std::array<ProcessorInst, N2>& processors_,
+                  bool is_primitive_);
   template <size_t N0, size_t N1, size_t N2>
   constexpr Field(size_t static_size_,
                   std::string_view name_,
                   const std::array<std::string_view, N0>& type_names_,
                   const std::array<Field, N1>& fields_,
-                  const std::array<ProcessorInst, N2>& processors_)
+                  const std::array<ProcessorInst, N2>& processors_,
+                  bool is_primitive_)
       : Field(static_size_,
               static_size_,
               name_,
               type_names_,
               fields_,
-              processors_) {
+              processors_,
+              is_primitive_) {
   }
   constexpr Field(const Field&) = default;  // no idea why this is needed
 
@@ -80,6 +83,7 @@ struct Field {
   std::span<const std::string_view> type_names;
   std::span<const Field> fields;
   std::span<const ProcessorInst> processors;
+  bool is_primitive;
 };
 
 template <size_t N0, size_t N1, size_t N2>
@@ -88,13 +92,15 @@ constexpr Field::Field(size_t static_size_,
                        std::string_view name_,
                        const std::array<std::string_view, N0>& type_names_,
                        const std::array<Field, N1>& fields_,
-                       const std::array<ProcessorInst, N2>& processors_)
+                       const std::array<ProcessorInst, N2>& processors_,
+                       bool is_primitive_)
     : static_size(static_size_),
       exclusive_size(exclusive_size_),
       name(name_),
       type_names(type_names_),
       fields(fields_),
-      processors(processors_) {
+      processors(processors_),
+      is_primitive(is_primitive_) {
 }
 
 }  // namespace oi::exporters::inst
diff --git a/include/oi/result/Element.h b/include/oi/result/Element.h
index 1eea86b..c971236 100644
--- a/include/oi/result/Element.h
+++ b/include/oi/result/Element.h
@@ -52,6 +52,7 @@ struct Element {
       std::nullopt};
   std::optional<ContainerStats> container_stats;
   std::optional<IsSetStats> is_set_stats;
+  bool is_primitive;
 };
 
 }  // namespace oi::result
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 84fb9b4..cb5bf54 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -798,12 +798,15 @@ void CodeGen::genClassTreeBuilderInstructions(const Class& c,
     if (m.name.starts_with(AddPadding::MemberPrefix))
       continue;
     std::string fullName = c.name() + "::" + m.name;
+    bool isPrimitive = dynamic_cast<const Primitive*>(&m.type());
     code += "      inst::Field{sizeof(" + fullName + "), " +
             std::to_string(calculateExclusiveSize(m.type())) + ",\"" +
             m.inputName + "\", member_" + std::to_string(index) +
             "_type_names, TypeHandler<Ctx, decltype(" + fullName +
             ")>::fields, TypeHandler<Ctx, decltype(" + fullName +
-            ")>::processors},\n";
+            ")>::processors, ";
+    code += isPrimitive ? "true" : "false";
+    code += "},\n";
   }
   code += "  };\n";
   code +=
@@ -1074,6 +1077,7 @@ constexpr inst::Field make_field(std::string_view name) {
     NameProvider<T>::names,
     TypeHandler<Ctx, T>::fields,
     TypeHandler<Ctx, T>::processors,
+    std::is_fundamental_v<T>
   };
 }
 )";
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 0b1e293..f2441e5 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -399,7 +399,8 @@ const std::array<std::string_view, )";
       ", OIInternal::TypeHandler<FakeContext, "
       "OIInternal::__ROOT_TYPE__>::fields, "
       "OIInternal::TypeHandler<FakeContext, "
-      "OIInternal::__ROOT_TYPE__>::processors};\n";
+      "OIInternal::__ROOT_TYPE__>::processors, "
+      "std::is_fundamental_v<OIInternal::__ROOT_TYPE__>};\n";
   code += "} // namespace\n";
   code +=
       "extern const exporters::inst::Inst __attribute__((used, retain)) "
@@ -640,13 +641,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
         }
         static void process_pointer_content(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
           static constexpr std::array<std::string_view, 1> names{"TODO"};
-          static constexpr auto childField = inst::Field{
-            sizeof(T),
-            "*",
-            names,
-            TypeHandler<Ctx, T>::fields,
-            TypeHandler<Ctx, T>::processors,
-          };
+          static constexpr auto childField = make_field<Ctx, T>("*");
 
           const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
 
@@ -743,13 +738,7 @@ return tail.finish();
       .type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>",
       .func = R"(
 static constexpr std::array<std::string_view, 1> names{"TODO"};
-static constexpr auto childField = inst::Field{
-  sizeof(T0),
-  "[]",
-  names,
-  TypeHandler<Ctx, T0>::fields,
-  TypeHandler<Ctx, T0>::processors,
-};
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 el.exclusive_size = 0;
 el.container_stats.emplace(result::Element::ContainerStats{ .capacity = N0, .length = N0 });
diff --git a/oi/IntrospectionResult.cpp b/oi/IntrospectionResult.cpp
index bc59f5e..5310001 100644
--- a/oi/IntrospectionResult.cpp
+++ b/oi/IntrospectionResult.cpp
@@ -78,6 +78,7 @@ IntrospectionResult::const_iterator::operator++() {
                 .exclusive_size = ty.exclusive_size,
                 .container_stats = std::nullopt,
                 .is_set_stats = std::nullopt,
+                .is_primitive = ty.is_primitive,
             };
 
             for (const auto& [dy, handler] : ty.processors) {
diff --git a/test/integration/simple.toml b/test/integration/simple.toml
index 0945f70..4557319 100644
--- a/test/integration/simple.toml
+++ b/test/integration/simple.toml
@@ -24,12 +24,21 @@ definitions = '''
       "staticSize":16,
       "dynamicSize":0,
       "exclusiveSize": 3,
-      "size": 16,
       "members":[
         {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
         {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize": 3,
+      "size": 16,
+      "is_primitive": false,
+      "members":[
+        {"name":"a", "staticSize":4, "exclusiveSize": 4, "size": 4, "is_primitive": true},
+        {"name":"b", "staticSize":1, "exclusiveSize": 1, "size": 1, "is_primitive": true},
+        {"name":"c", "staticSize":8, "exclusiveSize": 8, "size": 8, "is_primitive": true}
+      ]}]'''
   [cases.class]
     param_types = ["const SimpleClass&"]
     setup = "return {};"
@@ -37,18 +46,32 @@ definitions = '''
       "staticSize":16,
       "dynamicSize":0,
       "exclusiveSize": 3,
-      "size": 16,
       "members":[
         {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
         {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize": 3,
+      "size": 16,
+      "is_primitive": false,
+      "members":[
+        {"name":"a", "staticSize":4, "exclusiveSize": 4, "size": 4, "is_primitive": true},
+        {"name":"b", "staticSize":1, "exclusiveSize": 1, "size": 1, "is_primitive": true},
+        {"name":"c", "staticSize":8, "exclusiveSize": 8, "size": 8, "is_primitive": true}
+      ]}]'''
   [cases.union]
     param_types = ["const SimpleUnion&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
-      "exclusiveSize":8,
-      "size":8
+      "exclusiveSize":8
+    }]'''
+    expect_json_v2 = '''[{
+      "staticSize":8,
+      "exclusiveSize":8,
+      "size":8,
+      "is_primitive":false
     }]'''
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index 06c6f72..69a5173 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index de1dcda..cc89f06 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index c09bbf0..aa6aa59 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index a673bbd..f640dcc 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/list_type.toml b/types/list_type.toml
index 3e0da3c..3ece1d0 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -71,6 +71,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   child_field,
   std::array<inst::ProcessorInst, 0>{},
+  child_field.is_primitive,
 };
 static constexpr auto childField = make_field<Ctx, T0>("[]");
 
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index a09efff..f4281eb 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -91,6 +91,7 @@ static constexpr auto entry = inst::Field {
   std::array<std::string_view, 0>{},
   entryFields,
   processors,
+  entryFields[0].is_primitive && entryFields[1].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index a5d628f..4b068ba 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -118,6 +118,7 @@ static constexpr auto element = inst::Field {
   std::array<std::string_view, 0>{},
   elementFields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index ee652ec..b972b70 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -135,6 +135,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   processors,
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index 5b2402f..108731e 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -135,6 +135,7 @@ static constexpr auto element = inst::Field{
   std::array<std::string_view, 0>{},
   element_fields,
   processors,
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index a6f7671..8452154 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -121,6 +121,7 @@ static constexpr auto element = inst::Field{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)

From 9d122f8dd6e0ec9e9e11ee6a2f7eccf569a3d42a Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 11 Jan 2024 13:15:46 +0000
Subject: [PATCH 119/188] tbv2: re-add indirection to list

`std::list` was intended to have an extra layer to represent the overhead of
each list element similar to map implementations. This wasn't currently being
output which meant the size of each list element was being undercounted.

Add the extra layer of indirection to list elements and include this in the
tests.

Test Plan:
- CI
- Updated test cases
---
 test/integration/std_list.toml | 30 ++++++++++++++++++------------
 types/cxx11_list_type.toml     |  3 +--
 types/list_type.toml           |  3 +--
 3 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/test/integration/std_list.toml b/test/integration/std_list.toml
index 92c039a..2736603 100644
--- a/test/integration/std_list.toml
+++ b/test/integration/std_list.toml
@@ -18,19 +18,19 @@ definitions = '''
     param_types = ["const std::list<int>&"]
     setup = "return {{1,2,3}};"
     expect_json = '[{"staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity":3, "members":[
-      {"staticSize":4, "exclusiveSize":4, "size":4},
-      {"staticSize":4, "exclusiveSize":4, "size":4},
-      {"staticSize":4, "exclusiveSize":4, "size":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":96, "length":3, "capacity":3, "members":[
+      {"staticSize":24, "exclusiveSize":20, "size":24},
+      {"staticSize":24, "exclusiveSize":20, "size":24},
+      {"staticSize":24, "exclusiveSize":20, "size":24}
     ]}]'''
   [cases.struct_some]
     param_types = ["const std::list<SimpleStruct>&"]
     setup = "return {{{}, {}, {}}};"
     expect_json = '[{"staticSize":24, "dynamicSize":48, "length":3, "capacity":3, "elementStaticSize":16}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":72, "length":3, "capacity":3, "members":[
-      {"staticSize":16, "exclusiveSize":3, "size":16},
-      {"staticSize":16, "exclusiveSize":3, "size":16},
-      {"staticSize":16, "exclusiveSize":3, "size":16}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":120, "length":3, "capacity":3, "members":[
+      {"staticSize":32, "exclusiveSize":16, "size":32},
+      {"staticSize":32, "exclusiveSize":16, "size":32},
+      {"staticSize":32, "exclusiveSize":16, "size":32}
     ]}]'''
   [cases.list_int_empty]
     param_types = ["const std::list<std::list<int>>&"]
@@ -52,8 +52,14 @@ definitions = '''
         {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4},
         {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
       ]}]'''
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":120, "length":3, "capacity": 3, "members":[
-      {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity": 3},
-      {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity": 1},
-      {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity": 2}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":288, "length":3, "capacity": 3, "members":[
+      {"name":"[]", "staticSize":40, "exclusiveSize":16, "size":112, "members": [{
+        "name":"*", "staticSize":24, "exclusiveSize":24, "size":96, "length":3, "capacity": 3
+      }]},
+      {"name":"[]", "staticSize":40, "exclusiveSize":16, "size":64, "members": [{
+        "name":"*", "staticSize":24, "exclusiveSize":24, "size":48, "length":1, "capacity": 1
+      }]},
+      {"name":"[]", "staticSize":40, "exclusiveSize":16, "size":88, "members": [{
+        "name":"*", "staticSize":24, "exclusiveSize":24, "size":72, "length":2, "capacity": 2
+      }]}
     ]}]'''
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index 6788750..3285b95 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -72,7 +72,6 @@ static constexpr inst::Field element{
   child_field,
   std::array<inst::ProcessorInst, 0>{},
 };
-static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats{
@@ -81,5 +80,5 @@ el.container_stats.emplace(result::Element::ContainerStats{
 });
 el.exclusive_size += (el.container_stats->capacity - el.container_stats->length) * sizeof(T0);
 
-stack_ins(inst::Repeat{ list.length, childField });
+stack_ins(inst::Repeat{ list.length, element });
 """
diff --git a/types/list_type.toml b/types/list_type.toml
index 3e0da3c..86b798b 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -72,7 +72,6 @@ static constexpr inst::Field element{
   child_field,
   std::array<inst::ProcessorInst, 0>{},
 };
-static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats{
@@ -81,5 +80,5 @@ el.container_stats.emplace(result::Element::ContainerStats{
 });
 el.exclusive_size += (el.container_stats->capacity - el.container_stats->length) * sizeof(T0);
 
-stack_ins(inst::Repeat{ list.length, childField });
+stack_ins(inst::Repeat{ list.length, element });
 """

From ce6f316d67eed58923d51abd930e7df50f7c1f4c Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 11 Jan 2024 13:16:22 +0000
Subject: [PATCH 120/188] tbv2: add is_primitive to output

C++ has a concept of Primitive which holds in the type graph. However we don't
currently expose this information to the end user. Expose this from the OIL
iterator to allow future features like primitive rollups.

This affects containers like maps and `std::list` which have a fake `[]`
element with no type. They use this to group together the key/value in a map
and to account for any per element storage overhead. Currently the decision is
to make the fake `[]` element a primitive if all of its children are
primitives. This allows for more effective primitive rollups if that is
implemented. This implementation detail may be changed in future.

Test Plan:
- CI
- Updated simple tests.
---
 include/oi/exporters/Json.h            |  5 +++--
 include/oi/exporters/inst.h            | 16 ++++++++-----
 include/oi/result/Element.h            |  1 +
 oi/CodeGen.cpp                         |  6 ++++-
 oi/FuncGen.cpp                         | 19 ++++------------
 oi/IntrospectionResult.cpp             |  1 +
 test/integration/simple.toml           | 31 ++++++++++++++++++++++----
 types/cxx11_list_type.toml             |  1 +
 types/f14_fast_map.toml                |  1 +
 types/f14_node_map.toml                |  1 +
 types/f14_value_map.toml               |  1 +
 types/f14_vector_map.toml              |  1 +
 types/list_type.toml                   |  1 +
 types/map_seq_type.toml                |  1 +
 types/multi_map_type.toml              |  1 +
 types/std_map_type.toml                |  1 +
 types/std_unordered_map_type.toml      |  1 +
 types/std_unordered_multimap_type.toml |  1 +
 18 files changed, 63 insertions(+), 27 deletions(-)

diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h
index dd88ade..3dd37f4 100644
--- a/include/oi/exporters/Json.h
+++ b/include/oi/exporters/Json.h
@@ -98,8 +98,8 @@ inline void Json::printStringField(std::string_view name,
 inline void Json::printBoolField(std::string_view name,
                                  bool value,
                                  std::string_view indent) {
-  out_ << tab() << '"' << name << "\":" << space() << value << ',' << endl()
-       << indent;
+  out_ << tab() << '"' << name << "\":" << space() << (value ? "true" : "false")
+       << ',' << endl() << indent;
 }
 inline void Json::printUnsignedField(std::string_view name,
                                      uint64_t value,
@@ -159,6 +159,7 @@ inline void Json::printFields(const result::Element& el,
   }
   if (el.is_set_stats.has_value())
     printUnsignedField("is_set", el.is_set_stats->is_set, indent);
+  printBoolField("is_primitive", el.is_primitive, indent);
 }
 
 template <typename It>
diff --git a/include/oi/exporters/inst.h b/include/oi/exporters/inst.h
index 8397cd6..5484376 100644
--- a/include/oi/exporters/inst.h
+++ b/include/oi/exporters/inst.h
@@ -58,19 +58,22 @@ struct Field {
                   std::string_view name_,
                   const std::array<std::string_view, N0>& type_names_,
                   const std::array<Field, N1>& fields_,
-                  const std::array<ProcessorInst, N2>& processors_);
+                  const std::array<ProcessorInst, N2>& processors_,
+                  bool is_primitive_);
   template <size_t N0, size_t N1, size_t N2>
   constexpr Field(size_t static_size_,
                   std::string_view name_,
                   const std::array<std::string_view, N0>& type_names_,
                   const std::array<Field, N1>& fields_,
-                  const std::array<ProcessorInst, N2>& processors_)
+                  const std::array<ProcessorInst, N2>& processors_,
+                  bool is_primitive_)
       : Field(static_size_,
               static_size_,
               name_,
               type_names_,
               fields_,
-              processors_) {
+              processors_,
+              is_primitive_) {
   }
   constexpr Field(const Field&) = default;  // no idea why this is needed
 
@@ -80,6 +83,7 @@ struct Field {
   std::span<const std::string_view> type_names;
   std::span<const Field> fields;
   std::span<const ProcessorInst> processors;
+  bool is_primitive;
 };
 
 template <size_t N0, size_t N1, size_t N2>
@@ -88,13 +92,15 @@ constexpr Field::Field(size_t static_size_,
                        std::string_view name_,
                        const std::array<std::string_view, N0>& type_names_,
                        const std::array<Field, N1>& fields_,
-                       const std::array<ProcessorInst, N2>& processors_)
+                       const std::array<ProcessorInst, N2>& processors_,
+                       bool is_primitive_)
     : static_size(static_size_),
       exclusive_size(exclusive_size_),
       name(name_),
       type_names(type_names_),
       fields(fields_),
-      processors(processors_) {
+      processors(processors_),
+      is_primitive(is_primitive_) {
 }
 
 }  // namespace oi::exporters::inst
diff --git a/include/oi/result/Element.h b/include/oi/result/Element.h
index 1eea86b..c971236 100644
--- a/include/oi/result/Element.h
+++ b/include/oi/result/Element.h
@@ -52,6 +52,7 @@ struct Element {
       std::nullopt};
   std::optional<ContainerStats> container_stats;
   std::optional<IsSetStats> is_set_stats;
+  bool is_primitive;
 };
 
 }  // namespace oi::result
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 84fb9b4..cb5bf54 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -798,12 +798,15 @@ void CodeGen::genClassTreeBuilderInstructions(const Class& c,
     if (m.name.starts_with(AddPadding::MemberPrefix))
       continue;
     std::string fullName = c.name() + "::" + m.name;
+    bool isPrimitive = dynamic_cast<const Primitive*>(&m.type());
     code += "      inst::Field{sizeof(" + fullName + "), " +
             std::to_string(calculateExclusiveSize(m.type())) + ",\"" +
             m.inputName + "\", member_" + std::to_string(index) +
             "_type_names, TypeHandler<Ctx, decltype(" + fullName +
             ")>::fields, TypeHandler<Ctx, decltype(" + fullName +
-            ")>::processors},\n";
+            ")>::processors, ";
+    code += isPrimitive ? "true" : "false";
+    code += "},\n";
   }
   code += "  };\n";
   code +=
@@ -1074,6 +1077,7 @@ constexpr inst::Field make_field(std::string_view name) {
     NameProvider<T>::names,
     TypeHandler<Ctx, T>::fields,
     TypeHandler<Ctx, T>::processors,
+    std::is_fundamental_v<T>
   };
 }
 )";
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 0b1e293..f2441e5 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -399,7 +399,8 @@ const std::array<std::string_view, )";
       ", OIInternal::TypeHandler<FakeContext, "
       "OIInternal::__ROOT_TYPE__>::fields, "
       "OIInternal::TypeHandler<FakeContext, "
-      "OIInternal::__ROOT_TYPE__>::processors};\n";
+      "OIInternal::__ROOT_TYPE__>::processors, "
+      "std::is_fundamental_v<OIInternal::__ROOT_TYPE__>};\n";
   code += "} // namespace\n";
   code +=
       "extern const exporters::inst::Inst __attribute__((used, retain)) "
@@ -640,13 +641,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
         }
         static void process_pointer_content(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
           static constexpr std::array<std::string_view, 1> names{"TODO"};
-          static constexpr auto childField = inst::Field{
-            sizeof(T),
-            "*",
-            names,
-            TypeHandler<Ctx, T>::fields,
-            TypeHandler<Ctx, T>::processors,
-          };
+          static constexpr auto childField = make_field<Ctx, T>("*");
 
           const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
 
@@ -743,13 +738,7 @@ return tail.finish();
       .type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>",
       .func = R"(
 static constexpr std::array<std::string_view, 1> names{"TODO"};
-static constexpr auto childField = inst::Field{
-  sizeof(T0),
-  "[]",
-  names,
-  TypeHandler<Ctx, T0>::fields,
-  TypeHandler<Ctx, T0>::processors,
-};
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 el.exclusive_size = 0;
 el.container_stats.emplace(result::Element::ContainerStats{ .capacity = N0, .length = N0 });
diff --git a/oi/IntrospectionResult.cpp b/oi/IntrospectionResult.cpp
index bc59f5e..5310001 100644
--- a/oi/IntrospectionResult.cpp
+++ b/oi/IntrospectionResult.cpp
@@ -78,6 +78,7 @@ IntrospectionResult::const_iterator::operator++() {
                 .exclusive_size = ty.exclusive_size,
                 .container_stats = std::nullopt,
                 .is_set_stats = std::nullopt,
+                .is_primitive = ty.is_primitive,
             };
 
             for (const auto& [dy, handler] : ty.processors) {
diff --git a/test/integration/simple.toml b/test/integration/simple.toml
index 0945f70..4557319 100644
--- a/test/integration/simple.toml
+++ b/test/integration/simple.toml
@@ -24,12 +24,21 @@ definitions = '''
       "staticSize":16,
       "dynamicSize":0,
       "exclusiveSize": 3,
-      "size": 16,
       "members":[
         {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
         {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize": 3,
+      "size": 16,
+      "is_primitive": false,
+      "members":[
+        {"name":"a", "staticSize":4, "exclusiveSize": 4, "size": 4, "is_primitive": true},
+        {"name":"b", "staticSize":1, "exclusiveSize": 1, "size": 1, "is_primitive": true},
+        {"name":"c", "staticSize":8, "exclusiveSize": 8, "size": 8, "is_primitive": true}
+      ]}]'''
   [cases.class]
     param_types = ["const SimpleClass&"]
     setup = "return {};"
@@ -37,18 +46,32 @@ definitions = '''
       "staticSize":16,
       "dynamicSize":0,
       "exclusiveSize": 3,
-      "size": 16,
       "members":[
         {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
         {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize": 3,
+      "size": 16,
+      "is_primitive": false,
+      "members":[
+        {"name":"a", "staticSize":4, "exclusiveSize": 4, "size": 4, "is_primitive": true},
+        {"name":"b", "staticSize":1, "exclusiveSize": 1, "size": 1, "is_primitive": true},
+        {"name":"c", "staticSize":8, "exclusiveSize": 8, "size": 8, "is_primitive": true}
+      ]}]'''
   [cases.union]
     param_types = ["const SimpleUnion&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
-      "exclusiveSize":8,
-      "size":8
+      "exclusiveSize":8
+    }]'''
+    expect_json_v2 = '''[{
+      "staticSize":8,
+      "exclusiveSize":8,
+      "size":8,
+      "is_primitive":false
     }]'''
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index 3285b95..6b73704 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -71,6 +71,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   child_field,
   std::array<inst::ProcessorInst, 0>{},
+  child_field[0].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index 06c6f72..69a5173 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index de1dcda..cc89f06 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index c09bbf0..aa6aa59 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index a673bbd..f640dcc 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/list_type.toml b/types/list_type.toml
index 86b798b..106a371 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -71,6 +71,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   child_field,
   std::array<inst::ProcessorInst, 0>{},
+  child_field[0].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index a09efff..f4281eb 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -91,6 +91,7 @@ static constexpr auto entry = inst::Field {
   std::array<std::string_view, 0>{},
   entryFields,
   processors,
+  entryFields[0].is_primitive && entryFields[1].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index a5d628f..e313417 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -118,6 +118,7 @@ static constexpr auto element = inst::Field {
   std::array<std::string_view, 0>{},
   elementFields,
   std::array<inst::ProcessorInst, 0>{},
+  elementFields[0].is_primitive && elementFields[1].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index ee652ec..b972b70 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -135,6 +135,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   processors,
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index 5b2402f..108731e 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -135,6 +135,7 @@ static constexpr auto element = inst::Field{
   std::array<std::string_view, 0>{},
   element_fields,
   processors,
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index a6f7671..8452154 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -121,6 +121,7 @@ static constexpr auto element = inst::Field{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)

From 6d8775e2486015bd11c30f6ee41617a89e426220 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 11 Jan 2024 13:15:46 +0000
Subject: [PATCH 121/188] tbv2: re-add indirection to list

`std::list` was intended to have an extra layer to represent the overhead of
each list element similar to map implementations. This wasn't currently being
output which meant the size of each list element was being undercounted.

Add the extra layer of indirection to list elements and include this in the
tests.

Test Plan:
- CI
- Updated test cases
---
 test/integration/std_list.toml               | 30 ++++++++++++--------
 test/integration/std_list_del_allocator.toml |  6 ++--
 types/cxx11_list_type.toml                   |  3 +-
 types/list_type.toml                         |  3 +-
 4 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/test/integration/std_list.toml b/test/integration/std_list.toml
index 92c039a..2736603 100644
--- a/test/integration/std_list.toml
+++ b/test/integration/std_list.toml
@@ -18,19 +18,19 @@ definitions = '''
     param_types = ["const std::list<int>&"]
     setup = "return {{1,2,3}};"
     expect_json = '[{"staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity":3, "members":[
-      {"staticSize":4, "exclusiveSize":4, "size":4},
-      {"staticSize":4, "exclusiveSize":4, "size":4},
-      {"staticSize":4, "exclusiveSize":4, "size":4}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":96, "length":3, "capacity":3, "members":[
+      {"staticSize":24, "exclusiveSize":20, "size":24},
+      {"staticSize":24, "exclusiveSize":20, "size":24},
+      {"staticSize":24, "exclusiveSize":20, "size":24}
     ]}]'''
   [cases.struct_some]
     param_types = ["const std::list<SimpleStruct>&"]
     setup = "return {{{}, {}, {}}};"
     expect_json = '[{"staticSize":24, "dynamicSize":48, "length":3, "capacity":3, "elementStaticSize":16}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":72, "length":3, "capacity":3, "members":[
-      {"staticSize":16, "exclusiveSize":3, "size":16},
-      {"staticSize":16, "exclusiveSize":3, "size":16},
-      {"staticSize":16, "exclusiveSize":3, "size":16}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":120, "length":3, "capacity":3, "members":[
+      {"staticSize":32, "exclusiveSize":16, "size":32},
+      {"staticSize":32, "exclusiveSize":16, "size":32},
+      {"staticSize":32, "exclusiveSize":16, "size":32}
     ]}]'''
   [cases.list_int_empty]
     param_types = ["const std::list<std::list<int>>&"]
@@ -52,8 +52,14 @@ definitions = '''
         {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4},
         {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
       ]}]'''
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":120, "length":3, "capacity": 3, "members":[
-      {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity": 3},
-      {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity": 1},
-      {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity": 2}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":288, "length":3, "capacity": 3, "members":[
+      {"name":"[]", "staticSize":40, "exclusiveSize":16, "size":112, "members": [{
+        "name":"*", "staticSize":24, "exclusiveSize":24, "size":96, "length":3, "capacity": 3
+      }]},
+      {"name":"[]", "staticSize":40, "exclusiveSize":16, "size":64, "members": [{
+        "name":"*", "staticSize":24, "exclusiveSize":24, "size":48, "length":1, "capacity": 1
+      }]},
+      {"name":"[]", "staticSize":40, "exclusiveSize":16, "size":88, "members": [{
+        "name":"*", "staticSize":24, "exclusiveSize":24, "size":72, "length":2, "capacity": 2
+      }]}
     ]}]'''
diff --git a/test/integration/std_list_del_allocator.toml b/test/integration/std_list_del_allocator.toml
index c85d3ab..ba3a1d9 100644
--- a/test/integration/std_list_del_allocator.toml
+++ b/test/integration/std_list_del_allocator.toml
@@ -60,8 +60,8 @@ includes = ["list"]
     expect_json_v2 = '''[{
       "staticSize": 48,
       "exclusiveSize": 0,
-      "size": 60,
+      "size": 120,
       "members": [
-        {"name": "v1", "staticSize": 24, "exclusiveSize": 24, "size": 28, "length": 1, "capacity": 1},
-        {"name": "v2", "staticSize": 24, "exclusiveSize": 24, "size": 32, "length": 2, "capacity": 2}
+        {"name": "v1", "staticSize": 24, "exclusiveSize": 24, "size": 48, "length": 1, "capacity": 1},
+        {"name": "v2", "staticSize": 24, "exclusiveSize": 24, "size": 72, "length": 2, "capacity": 2}
       ]}]'''
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index 6788750..3285b95 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -72,7 +72,6 @@ static constexpr inst::Field element{
   child_field,
   std::array<inst::ProcessorInst, 0>{},
 };
-static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats{
@@ -81,5 +80,5 @@ el.container_stats.emplace(result::Element::ContainerStats{
 });
 el.exclusive_size += (el.container_stats->capacity - el.container_stats->length) * sizeof(T0);
 
-stack_ins(inst::Repeat{ list.length, childField });
+stack_ins(inst::Repeat{ list.length, element });
 """
diff --git a/types/list_type.toml b/types/list_type.toml
index 3e0da3c..86b798b 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -72,7 +72,6 @@ static constexpr inst::Field element{
   child_field,
   std::array<inst::ProcessorInst, 0>{},
 };
-static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
 el.container_stats.emplace(result::Element::ContainerStats{
@@ -81,5 +80,5 @@ el.container_stats.emplace(result::Element::ContainerStats{
 });
 el.exclusive_size += (el.container_stats->capacity - el.container_stats->length) * sizeof(T0);
 
-stack_ins(inst::Repeat{ list.length, childField });
+stack_ins(inst::Repeat{ list.length, element });
 """

From 9d4862ae14e6e0b5f8617763485f5668807dae79 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 11 Jan 2024 13:18:35 +0000
Subject: [PATCH 122/188] tbv2: add is_primitive to output

C++ has a concept of Primitive which holds in the type graph. However we don't
currently expose this information to the end user. Expose this from the OIL
iterator to allow future features like primitive rollups.

This affects containers like maps and `std::list` which have a fake `[]`
element with no type. They use this to group together the key/value in a map
and to account for any per element storage overhead. Currently the decision is
to make the fake `[]` element a primitive if all of its children are
primitives. This allows for more effective primitive rollups if that is
implemented. This implementation detail may be changed in future.

Test Plan:
- CI
- Updated simple tests.
---
 include/oi/exporters/Json.h            |  5 +++--
 include/oi/exporters/inst.h            | 16 ++++++++-----
 include/oi/result/Element.h            |  1 +
 oi/CodeGen.cpp                         |  6 ++++-
 oi/FuncGen.cpp                         | 19 ++++------------
 oi/IntrospectionResult.cpp             |  1 +
 test/integration/simple.toml           | 31 ++++++++++++++++++++++----
 types/cxx11_list_type.toml             |  1 +
 types/f14_fast_map.toml                |  1 +
 types/f14_node_map.toml                |  1 +
 types/f14_value_map.toml               |  1 +
 types/f14_vector_map.toml              |  1 +
 types/list_type.toml                   |  1 +
 types/map_seq_type.toml                |  1 +
 types/multi_map_type.toml              |  1 +
 types/std_map_type.toml                |  1 +
 types/std_unordered_map_type.toml      |  1 +
 types/std_unordered_multimap_type.toml |  1 +
 18 files changed, 63 insertions(+), 27 deletions(-)

diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h
index dd88ade..3dd37f4 100644
--- a/include/oi/exporters/Json.h
+++ b/include/oi/exporters/Json.h
@@ -98,8 +98,8 @@ inline void Json::printStringField(std::string_view name,
 inline void Json::printBoolField(std::string_view name,
                                  bool value,
                                  std::string_view indent) {
-  out_ << tab() << '"' << name << "\":" << space() << value << ',' << endl()
-       << indent;
+  out_ << tab() << '"' << name << "\":" << space() << (value ? "true" : "false")
+       << ',' << endl() << indent;
 }
 inline void Json::printUnsignedField(std::string_view name,
                                      uint64_t value,
@@ -159,6 +159,7 @@ inline void Json::printFields(const result::Element& el,
   }
   if (el.is_set_stats.has_value())
     printUnsignedField("is_set", el.is_set_stats->is_set, indent);
+  printBoolField("is_primitive", el.is_primitive, indent);
 }
 
 template <typename It>
diff --git a/include/oi/exporters/inst.h b/include/oi/exporters/inst.h
index 8397cd6..5484376 100644
--- a/include/oi/exporters/inst.h
+++ b/include/oi/exporters/inst.h
@@ -58,19 +58,22 @@ struct Field {
                   std::string_view name_,
                   const std::array<std::string_view, N0>& type_names_,
                   const std::array<Field, N1>& fields_,
-                  const std::array<ProcessorInst, N2>& processors_);
+                  const std::array<ProcessorInst, N2>& processors_,
+                  bool is_primitive_);
   template <size_t N0, size_t N1, size_t N2>
   constexpr Field(size_t static_size_,
                   std::string_view name_,
                   const std::array<std::string_view, N0>& type_names_,
                   const std::array<Field, N1>& fields_,
-                  const std::array<ProcessorInst, N2>& processors_)
+                  const std::array<ProcessorInst, N2>& processors_,
+                  bool is_primitive_)
       : Field(static_size_,
               static_size_,
               name_,
               type_names_,
               fields_,
-              processors_) {
+              processors_,
+              is_primitive_) {
   }
   constexpr Field(const Field&) = default;  // no idea why this is needed
 
@@ -80,6 +83,7 @@ struct Field {
   std::span<const std::string_view> type_names;
   std::span<const Field> fields;
   std::span<const ProcessorInst> processors;
+  bool is_primitive;
 };
 
 template <size_t N0, size_t N1, size_t N2>
@@ -88,13 +92,15 @@ constexpr Field::Field(size_t static_size_,
                        std::string_view name_,
                        const std::array<std::string_view, N0>& type_names_,
                        const std::array<Field, N1>& fields_,
-                       const std::array<ProcessorInst, N2>& processors_)
+                       const std::array<ProcessorInst, N2>& processors_,
+                       bool is_primitive_)
     : static_size(static_size_),
       exclusive_size(exclusive_size_),
       name(name_),
       type_names(type_names_),
       fields(fields_),
-      processors(processors_) {
+      processors(processors_),
+      is_primitive(is_primitive_) {
 }
 
 }  // namespace oi::exporters::inst
diff --git a/include/oi/result/Element.h b/include/oi/result/Element.h
index 1eea86b..c971236 100644
--- a/include/oi/result/Element.h
+++ b/include/oi/result/Element.h
@@ -52,6 +52,7 @@ struct Element {
       std::nullopt};
   std::optional<ContainerStats> container_stats;
   std::optional<IsSetStats> is_set_stats;
+  bool is_primitive;
 };
 
 }  // namespace oi::result
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 84fb9b4..cb5bf54 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -798,12 +798,15 @@ void CodeGen::genClassTreeBuilderInstructions(const Class& c,
     if (m.name.starts_with(AddPadding::MemberPrefix))
       continue;
     std::string fullName = c.name() + "::" + m.name;
+    bool isPrimitive = dynamic_cast<const Primitive*>(&m.type());
     code += "      inst::Field{sizeof(" + fullName + "), " +
             std::to_string(calculateExclusiveSize(m.type())) + ",\"" +
             m.inputName + "\", member_" + std::to_string(index) +
             "_type_names, TypeHandler<Ctx, decltype(" + fullName +
             ")>::fields, TypeHandler<Ctx, decltype(" + fullName +
-            ")>::processors},\n";
+            ")>::processors, ";
+    code += isPrimitive ? "true" : "false";
+    code += "},\n";
   }
   code += "  };\n";
   code +=
@@ -1074,6 +1077,7 @@ constexpr inst::Field make_field(std::string_view name) {
     NameProvider<T>::names,
     TypeHandler<Ctx, T>::fields,
     TypeHandler<Ctx, T>::processors,
+    std::is_fundamental_v<T>
   };
 }
 )";
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 0b1e293..f2441e5 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -399,7 +399,8 @@ const std::array<std::string_view, )";
       ", OIInternal::TypeHandler<FakeContext, "
       "OIInternal::__ROOT_TYPE__>::fields, "
       "OIInternal::TypeHandler<FakeContext, "
-      "OIInternal::__ROOT_TYPE__>::processors};\n";
+      "OIInternal::__ROOT_TYPE__>::processors, "
+      "std::is_fundamental_v<OIInternal::__ROOT_TYPE__>};\n";
   code += "} // namespace\n";
   code +=
       "extern const exporters::inst::Inst __attribute__((used, retain)) "
@@ -640,13 +641,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
         }
         static void process_pointer_content(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
           static constexpr std::array<std::string_view, 1> names{"TODO"};
-          static constexpr auto childField = inst::Field{
-            sizeof(T),
-            "*",
-            names,
-            TypeHandler<Ctx, T>::fields,
-            TypeHandler<Ctx, T>::processors,
-          };
+          static constexpr auto childField = make_field<Ctx, T>("*");
 
           const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
 
@@ -743,13 +738,7 @@ return tail.finish();
       .type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>",
       .func = R"(
 static constexpr std::array<std::string_view, 1> names{"TODO"};
-static constexpr auto childField = inst::Field{
-  sizeof(T0),
-  "[]",
-  names,
-  TypeHandler<Ctx, T0>::fields,
-  TypeHandler<Ctx, T0>::processors,
-};
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 el.exclusive_size = 0;
 el.container_stats.emplace(result::Element::ContainerStats{ .capacity = N0, .length = N0 });
diff --git a/oi/IntrospectionResult.cpp b/oi/IntrospectionResult.cpp
index bc59f5e..5310001 100644
--- a/oi/IntrospectionResult.cpp
+++ b/oi/IntrospectionResult.cpp
@@ -78,6 +78,7 @@ IntrospectionResult::const_iterator::operator++() {
                 .exclusive_size = ty.exclusive_size,
                 .container_stats = std::nullopt,
                 .is_set_stats = std::nullopt,
+                .is_primitive = ty.is_primitive,
             };
 
             for (const auto& [dy, handler] : ty.processors) {
diff --git a/test/integration/simple.toml b/test/integration/simple.toml
index 0945f70..4557319 100644
--- a/test/integration/simple.toml
+++ b/test/integration/simple.toml
@@ -24,12 +24,21 @@ definitions = '''
       "staticSize":16,
       "dynamicSize":0,
       "exclusiveSize": 3,
-      "size": 16,
       "members":[
         {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
         {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize": 3,
+      "size": 16,
+      "is_primitive": false,
+      "members":[
+        {"name":"a", "staticSize":4, "exclusiveSize": 4, "size": 4, "is_primitive": true},
+        {"name":"b", "staticSize":1, "exclusiveSize": 1, "size": 1, "is_primitive": true},
+        {"name":"c", "staticSize":8, "exclusiveSize": 8, "size": 8, "is_primitive": true}
+      ]}]'''
   [cases.class]
     param_types = ["const SimpleClass&"]
     setup = "return {};"
@@ -37,18 +46,32 @@ definitions = '''
       "staticSize":16,
       "dynamicSize":0,
       "exclusiveSize": 3,
-      "size": 16,
       "members":[
         {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
         {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize": 3,
+      "size": 16,
+      "is_primitive": false,
+      "members":[
+        {"name":"a", "staticSize":4, "exclusiveSize": 4, "size": 4, "is_primitive": true},
+        {"name":"b", "staticSize":1, "exclusiveSize": 1, "size": 1, "is_primitive": true},
+        {"name":"c", "staticSize":8, "exclusiveSize": 8, "size": 8, "is_primitive": true}
+      ]}]'''
   [cases.union]
     param_types = ["const SimpleUnion&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
-      "exclusiveSize":8,
-      "size":8
+      "exclusiveSize":8
+    }]'''
+    expect_json_v2 = '''[{
+      "staticSize":8,
+      "exclusiveSize":8,
+      "size":8,
+      "is_primitive":false
     }]'''
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index 3285b95..6b73704 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -71,6 +71,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   child_field,
   std::array<inst::ProcessorInst, 0>{},
+  child_field[0].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index 06c6f72..69a5173 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index de1dcda..cc89f06 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index c09bbf0..aa6aa59 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index a673bbd..f640dcc 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/list_type.toml b/types/list_type.toml
index 86b798b..106a371 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -71,6 +71,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   child_field,
   std::array<inst::ProcessorInst, 0>{},
+  child_field[0].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index a09efff..f4281eb 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -91,6 +91,7 @@ static constexpr auto entry = inst::Field {
   std::array<std::string_view, 0>{},
   entryFields,
   processors,
+  entryFields[0].is_primitive && entryFields[1].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index a5d628f..e313417 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -118,6 +118,7 @@ static constexpr auto element = inst::Field {
   std::array<std::string_view, 0>{},
   elementFields,
   std::array<inst::ProcessorInst, 0>{},
+  elementFields[0].is_primitive && elementFields[1].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index ee652ec..b972b70 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -135,6 +135,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   processors,
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index 5b2402f..108731e 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -135,6 +135,7 @@ static constexpr auto element = inst::Field{
   std::array<std::string_view, 0>{},
   element_fields,
   processors,
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index a6f7671..8452154 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -121,6 +121,7 @@ static constexpr auto element = inst::Field{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)

From 22ed863f75a52919bf149f71a0b1a40caf26d9c7 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 11 Jan 2024 15:11:57 +0000
Subject: [PATCH 123/188] tbv2: correctly account for list overhead

`std::list` has per element overhead for the individual heap allocations. This
was already calculated in the container implementation but not used. Allocate
the overhead of each element in the `std::list` to the `std::list` itself as
with other sequential containers.

Test Plan:
- CI
- Updated test cases
---
 test/integration/std_list.toml               | 12 ++++++------
 test/integration/std_list_del_allocator.toml |  6 +++---
 types/cxx11_list_type.toml                   | 13 +------------
 types/list_type.toml                         | 13 +------------
 4 files changed, 11 insertions(+), 33 deletions(-)

diff --git a/test/integration/std_list.toml b/test/integration/std_list.toml
index 92c039a..562ccc9 100644
--- a/test/integration/std_list.toml
+++ b/test/integration/std_list.toml
@@ -18,7 +18,7 @@ definitions = '''
     param_types = ["const std::list<int>&"]
     setup = "return {{1,2,3}};"
     expect_json = '[{"staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity":3, "members":[
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":84, "size":96, "length":3, "capacity":3, "members":[
       {"staticSize":4, "exclusiveSize":4, "size":4},
       {"staticSize":4, "exclusiveSize":4, "size":4},
       {"staticSize":4, "exclusiveSize":4, "size":4}
@@ -27,7 +27,7 @@ definitions = '''
     param_types = ["const std::list<SimpleStruct>&"]
     setup = "return {{{}, {}, {}}};"
     expect_json = '[{"staticSize":24, "dynamicSize":48, "length":3, "capacity":3, "elementStaticSize":16}]'
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":72, "length":3, "capacity":3, "members":[
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":72, "size":120, "length":3, "capacity":3, "members":[
       {"staticSize":16, "exclusiveSize":3, "size":16},
       {"staticSize":16, "exclusiveSize":3, "size":16},
       {"staticSize":16, "exclusiveSize":3, "size":16}
@@ -52,8 +52,8 @@ definitions = '''
         {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4},
         {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4}
       ]}]'''
-    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "size":120, "length":3, "capacity": 3, "members":[
-      {"staticSize":24, "exclusiveSize":24, "size":36, "length":3, "capacity": 3},
-      {"staticSize":24, "exclusiveSize":24, "size":28, "length":1, "capacity": 1},
-      {"staticSize":24, "exclusiveSize":24, "size":32, "length":2, "capacity": 2}
+    expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":72, "size":288, "length":3, "capacity": 3, "members":[
+      {"staticSize":24, "exclusiveSize":84, "size":96, "length":3, "capacity": 3},
+      {"staticSize":24, "exclusiveSize":44, "size":48, "length":1, "capacity": 1},
+      {"staticSize":24, "exclusiveSize":64, "size":72, "length":2, "capacity": 2}
     ]}]'''
diff --git a/test/integration/std_list_del_allocator.toml b/test/integration/std_list_del_allocator.toml
index c85d3ab..9298902 100644
--- a/test/integration/std_list_del_allocator.toml
+++ b/test/integration/std_list_del_allocator.toml
@@ -60,8 +60,8 @@ includes = ["list"]
     expect_json_v2 = '''[{
       "staticSize": 48,
       "exclusiveSize": 0,
-      "size": 60,
+      "size": 120,
       "members": [
-        {"name": "v1", "staticSize": 24, "exclusiveSize": 24, "size": 28, "length": 1, "capacity": 1},
-        {"name": "v2", "staticSize": 24, "exclusiveSize": 24, "size": 32, "length": 2, "capacity": 2}
+        {"name": "v1", "staticSize": 24, "exclusiveSize": 44, "size": 48, "length": 1, "capacity": 1},
+        {"name": "v2", "staticSize": 24, "exclusiveSize": 64, "size": 72, "length": 2, "capacity": 2}
       ]}]'''
diff --git a/types/cxx11_list_type.toml b/types/cxx11_list_type.toml
index 6788750..e55f145 100644
--- a/types/cxx11_list_type.toml
+++ b/types/cxx11_list_type.toml
@@ -61,17 +61,6 @@ static constexpr size_t element_size = sizeof(std::_List_node<T0>);
 static_assert(false && "No known element_size for list. See types/cxx11_list_type.toml");
 #endif
 
-static constexpr std::array<inst::Field, 1> child_field{
-  make_field<Ctx, T0>("*"),
-};
-static constexpr inst::Field element{
-  element_size,
-  element_size - sizeof(T0),
-  "[]",
-  std::array<std::string_view, 0>{},
-  child_field,
-  std::array<inst::ProcessorInst, 0>{},
-};
 static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
@@ -79,7 +68,7 @@ el.container_stats.emplace(result::Element::ContainerStats{
   .capacity = list.length,
   .length = list.length,
 });
-el.exclusive_size += (el.container_stats->capacity - el.container_stats->length) * sizeof(T0);
+el.exclusive_size += el.container_stats->length * (element_size - sizeof(T0));
 
 stack_ins(inst::Repeat{ list.length, childField });
 """
diff --git a/types/list_type.toml b/types/list_type.toml
index 3e0da3c..1161cca 100644
--- a/types/list_type.toml
+++ b/types/list_type.toml
@@ -61,17 +61,6 @@ static constexpr size_t element_size = sizeof(std::_List_node<T0>);
 static_assert(false && "No known element_size for list. See types/cxx11_list_type.toml");
 #endif
 
-static constexpr std::array<inst::Field, 1> child_field{
-  make_field<Ctx, T0>("*"),
-};
-static constexpr inst::Field element{
-  element_size,
-  element_size - sizeof(T0),
-  "[]",
-  std::array<std::string_view, 0>{},
-  child_field,
-  std::array<inst::ProcessorInst, 0>{},
-};
 static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 auto list = std::get<ParsedData::List>(d.val);
@@ -79,7 +68,7 @@ el.container_stats.emplace(result::Element::ContainerStats{
   .capacity = list.length,
   .length = list.length,
 });
-el.exclusive_size += (el.container_stats->capacity - el.container_stats->length) * sizeof(T0);
+el.exclusive_size += el.container_stats->length * (element_size - sizeof(T0));
 
 stack_ins(inst::Repeat{ list.length, childField });
 """

From f5f48c99efc2c7a898bf1fbc5b2955f6611f9255 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 11 Jan 2024 15:25:01 +0000
Subject: [PATCH 124/188] tbv2: name array member types correctly

Array members are currently being named "TODO" (whoops). Include arrays in
TopoSorter so each one can have a `NameProvider` generated in CodeGen. Then
pass array elements through `make_field`.

Test plan:
- CI
- Add array member names to an array test.
---
 oi/FuncGen.cpp               | 9 +--------
 oi/type_graph/TopoSorter.cpp | 5 +++++
 oi/type_graph/TopoSorter.h   | 1 +
 test/integration/arrays.toml | 4 ++--
 4 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 0b1e293..97d2707 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -742,14 +742,7 @@ return tail.finish();
   oiArray.codegen.processors.emplace_back(ContainerInfo::Processor{
       .type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>",
       .func = R"(
-static constexpr std::array<std::string_view, 1> names{"TODO"};
-static constexpr auto childField = inst::Field{
-  sizeof(T0),
-  "[]",
-  names,
-  TypeHandler<Ctx, T0>::fields,
-  TypeHandler<Ctx, T0>::processors,
-};
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 el.exclusive_size = 0;
 el.container_stats.emplace(result::Element::ContainerStats{ .capacity = N0, .length = N0 });
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index ff2715a..1be10f5 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -143,6 +143,11 @@ void TopoSorter::visit(Incomplete& i) {
   sortedTypes_.push_back(i);
 }
 
+void TopoSorter::visit(Array& a) {
+  RecursiveVisitor::visit(a);
+  sortedTypes_.push_back(a);
+}
+
 /*
  * A type graph may contain cycles, so we need to slightly tweak the standard
  * topological sorting algorithm. Cycles can only be introduced by certain
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 44f2364..5c78ad1 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -49,6 +49,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
   void visit(Incomplete& i) override;
+  void visit(Array& i) override;
 
  private:
   std::unordered_set<Type*> visited_;
diff --git a/test/integration/arrays.toml b/test/integration/arrays.toml
index 25e542a..a4fd3b6 100644
--- a/test/integration/arrays.toml
+++ b/test/integration/arrays.toml
@@ -88,8 +88,8 @@ definitions = '''
     expect_json_v2 = '''[
       {"staticSize":24, "exclusiveSize":0, "size":24, "members":[
         {"staticSize":24, "exclusiveSize":0, "size":24, "length":2, "capacity":2, "members":[
-          {"staticSize":12, "exclusiveSize":0, "size":12, "length":3, "capacity":3},
-          {"staticSize":12, "exclusiveSize":0, "size":12, "length":3, "capacity":3}]
+          {"typeNames":["int32_t[3]"], "staticSize":12, "exclusiveSize":0, "size":12, "length":3, "capacity":3},
+          {"typeNames":["int32_t[3]"], "staticSize":12, "exclusiveSize":0, "size":12, "length":3, "capacity":3}]
       }]}]'''
   [cases.direct_int10]
     skip = "Direct array arguments don't work"

From 719b33ed95ac8fd2cf538cab433da51ab171c458 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 11 Jan 2024 15:25:27 +0000
Subject: [PATCH 125/188] tbv2: add is_primitive to output

C++ has a concept of Primitive which holds in the type graph. However we don't
currently expose this information to the end user. Expose this from the OIL
iterator to allow future features like primitive rollups.

This affects containers like maps and `std::list` which have a fake `[]`
element with no type. They use this to group together the key/value in a map
and to account for any per element storage overhead. Currently the decision is
to make the fake `[]` element a primitive if all of its children are
primitives. This allows for more effective primitive rollups if that is
implemented. This implementation detail may be changed in future.

Test Plan:
- CI
- Updated simple tests.
---
 include/oi/exporters/Json.h            |  5 +++--
 include/oi/exporters/inst.h            | 16 ++++++++-----
 include/oi/result/Element.h            |  1 +
 oi/CodeGen.cpp                         |  6 ++++-
 oi/FuncGen.cpp                         | 11 +++------
 oi/IntrospectionResult.cpp             |  1 +
 test/integration/simple.toml           | 31 ++++++++++++++++++++++----
 types/f14_fast_map.toml                |  1 +
 types/f14_node_map.toml                |  1 +
 types/f14_value_map.toml               |  1 +
 types/f14_vector_map.toml              |  1 +
 types/map_seq_type.toml                |  1 +
 types/multi_map_type.toml              |  1 +
 types/std_map_type.toml                |  1 +
 types/std_unordered_map_type.toml      |  1 +
 types/std_unordered_multimap_type.toml |  1 +
 16 files changed, 60 insertions(+), 20 deletions(-)

diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h
index dd88ade..3dd37f4 100644
--- a/include/oi/exporters/Json.h
+++ b/include/oi/exporters/Json.h
@@ -98,8 +98,8 @@ inline void Json::printStringField(std::string_view name,
 inline void Json::printBoolField(std::string_view name,
                                  bool value,
                                  std::string_view indent) {
-  out_ << tab() << '"' << name << "\":" << space() << value << ',' << endl()
-       << indent;
+  out_ << tab() << '"' << name << "\":" << space() << (value ? "true" : "false")
+       << ',' << endl() << indent;
 }
 inline void Json::printUnsignedField(std::string_view name,
                                      uint64_t value,
@@ -159,6 +159,7 @@ inline void Json::printFields(const result::Element& el,
   }
   if (el.is_set_stats.has_value())
     printUnsignedField("is_set", el.is_set_stats->is_set, indent);
+  printBoolField("is_primitive", el.is_primitive, indent);
 }
 
 template <typename It>
diff --git a/include/oi/exporters/inst.h b/include/oi/exporters/inst.h
index 8397cd6..5484376 100644
--- a/include/oi/exporters/inst.h
+++ b/include/oi/exporters/inst.h
@@ -58,19 +58,22 @@ struct Field {
                   std::string_view name_,
                   const std::array<std::string_view, N0>& type_names_,
                   const std::array<Field, N1>& fields_,
-                  const std::array<ProcessorInst, N2>& processors_);
+                  const std::array<ProcessorInst, N2>& processors_,
+                  bool is_primitive_);
   template <size_t N0, size_t N1, size_t N2>
   constexpr Field(size_t static_size_,
                   std::string_view name_,
                   const std::array<std::string_view, N0>& type_names_,
                   const std::array<Field, N1>& fields_,
-                  const std::array<ProcessorInst, N2>& processors_)
+                  const std::array<ProcessorInst, N2>& processors_,
+                  bool is_primitive_)
       : Field(static_size_,
               static_size_,
               name_,
               type_names_,
               fields_,
-              processors_) {
+              processors_,
+              is_primitive_) {
   }
   constexpr Field(const Field&) = default;  // no idea why this is needed
 
@@ -80,6 +83,7 @@ struct Field {
   std::span<const std::string_view> type_names;
   std::span<const Field> fields;
   std::span<const ProcessorInst> processors;
+  bool is_primitive;
 };
 
 template <size_t N0, size_t N1, size_t N2>
@@ -88,13 +92,15 @@ constexpr Field::Field(size_t static_size_,
                        std::string_view name_,
                        const std::array<std::string_view, N0>& type_names_,
                        const std::array<Field, N1>& fields_,
-                       const std::array<ProcessorInst, N2>& processors_)
+                       const std::array<ProcessorInst, N2>& processors_,
+                       bool is_primitive_)
     : static_size(static_size_),
       exclusive_size(exclusive_size_),
       name(name_),
       type_names(type_names_),
       fields(fields_),
-      processors(processors_) {
+      processors(processors_),
+      is_primitive(is_primitive_) {
 }
 
 }  // namespace oi::exporters::inst
diff --git a/include/oi/result/Element.h b/include/oi/result/Element.h
index 1eea86b..c971236 100644
--- a/include/oi/result/Element.h
+++ b/include/oi/result/Element.h
@@ -52,6 +52,7 @@ struct Element {
       std::nullopt};
   std::optional<ContainerStats> container_stats;
   std::optional<IsSetStats> is_set_stats;
+  bool is_primitive;
 };
 
 }  // namespace oi::result
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 84fb9b4..cb5bf54 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -798,12 +798,15 @@ void CodeGen::genClassTreeBuilderInstructions(const Class& c,
     if (m.name.starts_with(AddPadding::MemberPrefix))
       continue;
     std::string fullName = c.name() + "::" + m.name;
+    bool isPrimitive = dynamic_cast<const Primitive*>(&m.type());
     code += "      inst::Field{sizeof(" + fullName + "), " +
             std::to_string(calculateExclusiveSize(m.type())) + ",\"" +
             m.inputName + "\", member_" + std::to_string(index) +
             "_type_names, TypeHandler<Ctx, decltype(" + fullName +
             ")>::fields, TypeHandler<Ctx, decltype(" + fullName +
-            ")>::processors},\n";
+            ")>::processors, ";
+    code += isPrimitive ? "true" : "false";
+    code += "},\n";
   }
   code += "  };\n";
   code +=
@@ -1074,6 +1077,7 @@ constexpr inst::Field make_field(std::string_view name) {
     NameProvider<T>::names,
     TypeHandler<Ctx, T>::fields,
     TypeHandler<Ctx, T>::processors,
+    std::is_fundamental_v<T>
   };
 }
 )";
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 97d2707..5ac3abf 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -399,7 +399,8 @@ const std::array<std::string_view, )";
       ", OIInternal::TypeHandler<FakeContext, "
       "OIInternal::__ROOT_TYPE__>::fields, "
       "OIInternal::TypeHandler<FakeContext, "
-      "OIInternal::__ROOT_TYPE__>::processors};\n";
+      "OIInternal::__ROOT_TYPE__>::processors, "
+      "std::is_fundamental_v<OIInternal::__ROOT_TYPE__>};\n";
   code += "} // namespace\n";
   code +=
       "extern const exporters::inst::Inst __attribute__((used, retain)) "
@@ -640,13 +641,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
         }
         static void process_pointer_content(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
           static constexpr std::array<std::string_view, 1> names{"TODO"};
-          static constexpr auto childField = inst::Field{
-            sizeof(T),
-            "*",
-            names,
-            TypeHandler<Ctx, T>::fields,
-            TypeHandler<Ctx, T>::processors,
-          };
+          static constexpr auto childField = make_field<Ctx, T>("*");
 
           const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
 
diff --git a/oi/IntrospectionResult.cpp b/oi/IntrospectionResult.cpp
index bc59f5e..5310001 100644
--- a/oi/IntrospectionResult.cpp
+++ b/oi/IntrospectionResult.cpp
@@ -78,6 +78,7 @@ IntrospectionResult::const_iterator::operator++() {
                 .exclusive_size = ty.exclusive_size,
                 .container_stats = std::nullopt,
                 .is_set_stats = std::nullopt,
+                .is_primitive = ty.is_primitive,
             };
 
             for (const auto& [dy, handler] : ty.processors) {
diff --git a/test/integration/simple.toml b/test/integration/simple.toml
index 0945f70..4557319 100644
--- a/test/integration/simple.toml
+++ b/test/integration/simple.toml
@@ -24,12 +24,21 @@ definitions = '''
       "staticSize":16,
       "dynamicSize":0,
       "exclusiveSize": 3,
-      "size": 16,
       "members":[
         {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
         {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize": 3,
+      "size": 16,
+      "is_primitive": false,
+      "members":[
+        {"name":"a", "staticSize":4, "exclusiveSize": 4, "size": 4, "is_primitive": true},
+        {"name":"b", "staticSize":1, "exclusiveSize": 1, "size": 1, "is_primitive": true},
+        {"name":"c", "staticSize":8, "exclusiveSize": 8, "size": 8, "is_primitive": true}
+      ]}]'''
   [cases.class]
     param_types = ["const SimpleClass&"]
     setup = "return {};"
@@ -37,18 +46,32 @@ definitions = '''
       "staticSize":16,
       "dynamicSize":0,
       "exclusiveSize": 3,
-      "size": 16,
       "members":[
         {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
         {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize": 3,
+      "size": 16,
+      "is_primitive": false,
+      "members":[
+        {"name":"a", "staticSize":4, "exclusiveSize": 4, "size": 4, "is_primitive": true},
+        {"name":"b", "staticSize":1, "exclusiveSize": 1, "size": 1, "is_primitive": true},
+        {"name":"c", "staticSize":8, "exclusiveSize": 8, "size": 8, "is_primitive": true}
+      ]}]'''
   [cases.union]
     param_types = ["const SimpleUnion&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
-      "exclusiveSize":8,
-      "size":8
+      "exclusiveSize":8
+    }]'''
+    expect_json_v2 = '''[{
+      "staticSize":8,
+      "exclusiveSize":8,
+      "size":8,
+      "is_primitive":false
     }]'''
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index 06c6f72..69a5173 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index de1dcda..cc89f06 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index c09bbf0..aa6aa59 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index a673bbd..f640dcc 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index a09efff..f4281eb 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -91,6 +91,7 @@ static constexpr auto entry = inst::Field {
   std::array<std::string_view, 0>{},
   entryFields,
   processors,
+  entryFields[0].is_primitive && entryFields[1].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index a5d628f..e313417 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -118,6 +118,7 @@ static constexpr auto element = inst::Field {
   std::array<std::string_view, 0>{},
   elementFields,
   std::array<inst::ProcessorInst, 0>{},
+  elementFields[0].is_primitive && elementFields[1].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index ee652ec..b972b70 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -135,6 +135,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   processors,
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index 5b2402f..108731e 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -135,6 +135,7 @@ static constexpr auto element = inst::Field{
   std::array<std::string_view, 0>{},
   element_fields,
   processors,
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index a6f7671..8452154 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -121,6 +121,7 @@ static constexpr auto element = inst::Field{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)

From 5c78c1d8d773605d4f9f1d79106cc45f3d6d7b2a Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 11 Jan 2024 16:11:18 +0000
Subject: [PATCH 126/188] tbv2: remove unnecessary copy in Element

`IntrospectionResult::const_iterator` iterates through the `Element`s in an
`IntrospectionResult`. `Element` currently copies the `type_path` which is a
`std::vector<string_view>` every time the iterator is incremented. This is
unnecessary as the data in the vector only changes slightly between iterations.

This change changes the `type_path` field in `Element` to a
`std::span<const std::string_view>`. Doing this previously caused SEGVs because
of the iterator's potential to be copied. To make it possible we do two things:
1. Make all copies explicit using a clone interface as in `ContainerInfo`. This
   prevents accidental copies of an expensive structure.
2. After calling the copy constructor in `clone()` update the `span` in `next_`
   to point at the newly copied structure.

Moves are fine because the `span` points at the allocation of the `vector`, not
the vector itself.

Test plan:
- CI
- `FILTER='OilIntegration.*' make test`
- Ran `OilgenIntegration.std_vector_vector_int_some` which SEGVd with the
  `span` change before and now doesn't. This now passes cleanly with ASAN
  enabled on the target, though isn't available in `main` (only works on my
  machine).
---
 include/oi/IntrospectionResult-inl.h | 16 +++++++++++++++-
 include/oi/IntrospectionResult.h     | 10 ++++++++++
 include/oi/result/Element.h          |  3 +--
 include/oi/result/SizedResult-inl.h  |  9 ++++++---
 oi/IntrospectionResult.cpp           |  5 ++---
 5 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/include/oi/IntrospectionResult-inl.h b/include/oi/IntrospectionResult-inl.h
index 13928b5..a0866a7 100644
--- a/include/oi/IntrospectionResult-inl.h
+++ b/include/oi/IntrospectionResult-inl.h
@@ -45,7 +45,9 @@ inline IntrospectionResult::const_iterator IntrospectionResult::begin() const {
   return cbegin();
 }
 inline IntrospectionResult::const_iterator IntrospectionResult::cbegin() const {
-  return ++const_iterator{buf_.cbegin(), inst_};
+  auto it = const_iterator{buf_.cbegin(), inst_};
+  ++it;
+  return it;
 }
 inline IntrospectionResult::const_iterator IntrospectionResult::end() const {
   return cend();
@@ -70,6 +72,18 @@ inline const result::Element* IntrospectionResult::const_iterator::operator->()
   return &*next_;
 }
 
+inline IntrospectionResult::const_iterator
+IntrospectionResult::const_iterator::clone() const {
+  auto ret{*this};
+
+  // Fix the self referential type_path_ field in next_
+  if (ret.next_.has_value()) {
+    ret.next_->type_path = ret.type_path_;
+  }
+
+  return ret;
+}
+
 inline bool IntrospectionResult::const_iterator::operator==(
     const IntrospectionResult::const_iterator& that) const {
   // Case 1: The next data to read differs, thus the iterators are different.
diff --git a/include/oi/IntrospectionResult.h b/include/oi/IntrospectionResult.h
index 091028f..1675423 100644
--- a/include/oi/IntrospectionResult.h
+++ b/include/oi/IntrospectionResult.h
@@ -43,6 +43,16 @@ class IntrospectionResult {
     const_iterator& operator++();
     const_iterator operator++(int);
 
+   private:
+    const_iterator(const const_iterator&) = default;
+    const_iterator& operator=(const const_iterator& other) = default;
+
+   public:
+    const_iterator(const_iterator&&) = default;
+    const_iterator& operator=(const_iterator&&) = default;
+    // Explicit interface for copying
+    const_iterator clone() const;
+
    private:
     using stack_t =
         std::stack<exporters::inst::Inst, std::vector<exporters::inst::Inst>>;
diff --git a/include/oi/result/Element.h b/include/oi/result/Element.h
index 1eea86b..5610c1a 100644
--- a/include/oi/result/Element.h
+++ b/include/oi/result/Element.h
@@ -41,8 +41,7 @@ struct Element {
   };
 
   std::string_view name;
-  std::vector<std::string_view>
-      type_path;  // TODO: should be span<const std::string_view>
+  std::span<const std::string_view> type_path;
   std::span<const std::string_view> type_names;
   size_t static_size;
   size_t exclusive_size;
diff --git a/include/oi/result/SizedResult-inl.h b/include/oi/result/SizedResult-inl.h
index f86ff0c..94418a4 100644
--- a/include/oi/result/SizedResult-inl.h
+++ b/include/oi/result/SizedResult-inl.h
@@ -35,7 +35,9 @@ SizedResult<Res>::SizedResult(Res res) : res_{std::move(res)} {
 
 template <typename Res>
 typename SizedResult<Res>::const_iterator SizedResult<Res>::begin() const {
-  return ++const_iterator{res_.begin(), res_.end()};
+  const_iterator it{res_.begin(), res_.end()};
+  ++it;
+  return it;
 }
 template <typename Res>
 typename SizedResult<Res>::const_iterator SizedResult<Res>::end() const {
@@ -44,7 +46,7 @@ typename SizedResult<Res>::const_iterator SizedResult<Res>::end() const {
 
 template <typename Res>
 SizedResult<Res>::const_iterator::const_iterator(It it, const It& end)
-    : data_{it} {
+    : data_{it.clone()} {
   struct StackEntry {
     size_t index;
     size_t depth;
@@ -75,7 +77,8 @@ SizedResult<Res>::const_iterator::const_iterator(It it, const It& end)
 }
 
 template <typename Res>
-SizedResult<Res>::const_iterator::const_iterator(It end) : data_{end} {
+SizedResult<Res>::const_iterator::const_iterator(It end)
+    : data_{std::move(end)} {
 }
 
 template <typename Res>
diff --git a/oi/IntrospectionResult.cpp b/oi/IntrospectionResult.cpp
index bc59f5e..b8a6814 100644
--- a/oi/IntrospectionResult.cpp
+++ b/oi/IntrospectionResult.cpp
@@ -70,7 +70,7 @@ IntrospectionResult::const_iterator::operator++() {
           if constexpr (std::is_same_v<T, exporters::inst::Field>) {
             type_path_.emplace_back(ty.name);
             stack_.emplace(exporters::inst::PopTypePath{});
-            next_ = result::Element{
+            next_.emplace(result::Element{
                 .name = ty.name,
                 .type_path = type_path_,
                 .type_names = ty.type_names,
@@ -78,7 +78,7 @@ IntrospectionResult::const_iterator::operator++() {
                 .exclusive_size = ty.exclusive_size,
                 .container_stats = std::nullopt,
                 .is_set_stats = std::nullopt,
-            };
+            });
 
             for (const auto& [dy, handler] : ty.processors) {
               auto parsed = exporters::ParsedData::parse(data_, dy);
@@ -93,7 +93,6 @@ IntrospectionResult::const_iterator::operator++() {
                       .second;
 
               type_path_.back() = new_name_ref;
-              next_->type_path.back() = new_name_ref;
               next_->name = new_name_ref;
             }
 

From 3653decf8e9ff23a37cc5daa1fbe82092a857e01 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 11 Jan 2024 16:23:22 +0000
Subject: [PATCH 127/188] tbv2: name array member types correctly

Array members are currently being named "TODO" (whoops). Include arrays in
TopoSorter so each one can have a `NameProvider` generated in CodeGen. Then
pass array elements through `make_field`.

Test plan:
- CI
- Add array member names to an array test.
---
 oi/FuncGen.cpp               | 9 +--------
 oi/type_graph/TopoSorter.cpp | 5 +++++
 oi/type_graph/TopoSorter.h   | 1 +
 test/integration/arrays.toml | 4 ++--
 test/test_topo_sorter.cpp    | 5 ++++-
 5 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 0b1e293..97d2707 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -742,14 +742,7 @@ return tail.finish();
   oiArray.codegen.processors.emplace_back(ContainerInfo::Processor{
       .type = "types::st::List<DB, typename TypeHandler<Ctx, T0>::type>",
       .func = R"(
-static constexpr std::array<std::string_view, 1> names{"TODO"};
-static constexpr auto childField = inst::Field{
-  sizeof(T0),
-  "[]",
-  names,
-  TypeHandler<Ctx, T0>::fields,
-  TypeHandler<Ctx, T0>::processors,
-};
+static constexpr auto childField = make_field<Ctx, T0>("[]");
 
 el.exclusive_size = 0;
 el.container_stats.emplace(result::Element::ContainerStats{ .capacity = N0, .length = N0 });
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index ff2715a..1be10f5 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -143,6 +143,11 @@ void TopoSorter::visit(Incomplete& i) {
   sortedTypes_.push_back(i);
 }
 
+void TopoSorter::visit(Array& a) {
+  RecursiveVisitor::visit(a);
+  sortedTypes_.push_back(a);
+}
+
 /*
  * A type graph may contain cycles, so we need to slightly tweak the standard
  * topological sorting algorithm. Cycles can only be introduced by certain
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 44f2364..5c78ad1 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -49,6 +49,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
   void visit(Incomplete& i) override;
+  void visit(Array& i) override;
 
  private:
   std::unordered_set<Type*> visited_;
diff --git a/test/integration/arrays.toml b/test/integration/arrays.toml
index 25e542a..a4fd3b6 100644
--- a/test/integration/arrays.toml
+++ b/test/integration/arrays.toml
@@ -88,8 +88,8 @@ definitions = '''
     expect_json_v2 = '''[
       {"staticSize":24, "exclusiveSize":0, "size":24, "members":[
         {"staticSize":24, "exclusiveSize":0, "size":24, "length":2, "capacity":2, "members":[
-          {"staticSize":12, "exclusiveSize":0, "size":12, "length":3, "capacity":3},
-          {"staticSize":12, "exclusiveSize":0, "size":12, "length":3, "capacity":3}]
+          {"typeNames":["int32_t[3]"], "staticSize":12, "exclusiveSize":0, "size":12, "length":3, "capacity":3},
+          {"typeNames":["int32_t[3]"], "staticSize":12, "exclusiveSize":0, "size":12, "length":3, "capacity":3}]
       }]}]'''
   [cases.direct_int10]
     skip = "Direct array arguments don't work"
diff --git a/test/test_topo_sorter.cpp b/test/test_topo_sorter.cpp
index 969c6aa..c97801f 100644
--- a/test/test_topo_sorter.cpp
+++ b/test/test_topo_sorter.cpp
@@ -211,7 +211,10 @@ TEST(TopoSorterTest, Arrays) {
   auto myclass = Class{0, Class::Kind::Class, "MyClass", 69};
   auto myarray = Array{1, myclass, 10};
 
-  test({myarray}, "MyClass\n");
+  test({myarray}, R"(
+MyClass
+OIArray<MyClass, 10>
+)");
 }
 
 TEST(TopoSorterTest, Typedef) {

From 98ce9d37c6e0edfa9c670347841967fce26fe609 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 11 Jan 2024 16:23:22 +0000
Subject: [PATCH 128/188] tbv2: add is_primitive to output

C++ has a concept of Primitive which holds in the type graph. However we don't
currently expose this information to the end user. Expose this from the OIL
iterator to allow future features like primitive rollups.

This affects containers like maps and `std::list` which have a fake `[]`
element with no type. They use this to group together the key/value in a map
and to account for any per element storage overhead. Currently the decision is
to make the fake `[]` element a primitive if all of its children are
primitives. This allows for more effective primitive rollups if that is
implemented. This implementation detail may be changed in future.

Test Plan:
- CI
- Updated simple tests.
---
 include/oi/exporters/Json.h            |  5 +++--
 include/oi/exporters/inst.h            | 16 ++++++++-----
 include/oi/result/Element.h            |  1 +
 oi/CodeGen.cpp                         |  6 ++++-
 oi/FuncGen.cpp                         | 11 +++------
 oi/IntrospectionResult.cpp             |  1 +
 test/integration/simple.toml           | 31 ++++++++++++++++++++++----
 types/f14_fast_map.toml                |  1 +
 types/f14_node_map.toml                |  1 +
 types/f14_value_map.toml               |  1 +
 types/f14_vector_map.toml              |  1 +
 types/map_seq_type.toml                |  1 +
 types/multi_map_type.toml              |  1 +
 types/std_map_type.toml                |  1 +
 types/std_unordered_map_type.toml      |  1 +
 types/std_unordered_multimap_type.toml |  1 +
 16 files changed, 60 insertions(+), 20 deletions(-)

diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h
index dd88ade..3dd37f4 100644
--- a/include/oi/exporters/Json.h
+++ b/include/oi/exporters/Json.h
@@ -98,8 +98,8 @@ inline void Json::printStringField(std::string_view name,
 inline void Json::printBoolField(std::string_view name,
                                  bool value,
                                  std::string_view indent) {
-  out_ << tab() << '"' << name << "\":" << space() << value << ',' << endl()
-       << indent;
+  out_ << tab() << '"' << name << "\":" << space() << (value ? "true" : "false")
+       << ',' << endl() << indent;
 }
 inline void Json::printUnsignedField(std::string_view name,
                                      uint64_t value,
@@ -159,6 +159,7 @@ inline void Json::printFields(const result::Element& el,
   }
   if (el.is_set_stats.has_value())
     printUnsignedField("is_set", el.is_set_stats->is_set, indent);
+  printBoolField("is_primitive", el.is_primitive, indent);
 }
 
 template <typename It>
diff --git a/include/oi/exporters/inst.h b/include/oi/exporters/inst.h
index 8397cd6..5484376 100644
--- a/include/oi/exporters/inst.h
+++ b/include/oi/exporters/inst.h
@@ -58,19 +58,22 @@ struct Field {
                   std::string_view name_,
                   const std::array<std::string_view, N0>& type_names_,
                   const std::array<Field, N1>& fields_,
-                  const std::array<ProcessorInst, N2>& processors_);
+                  const std::array<ProcessorInst, N2>& processors_,
+                  bool is_primitive_);
   template <size_t N0, size_t N1, size_t N2>
   constexpr Field(size_t static_size_,
                   std::string_view name_,
                   const std::array<std::string_view, N0>& type_names_,
                   const std::array<Field, N1>& fields_,
-                  const std::array<ProcessorInst, N2>& processors_)
+                  const std::array<ProcessorInst, N2>& processors_,
+                  bool is_primitive_)
       : Field(static_size_,
               static_size_,
               name_,
               type_names_,
               fields_,
-              processors_) {
+              processors_,
+              is_primitive_) {
   }
   constexpr Field(const Field&) = default;  // no idea why this is needed
 
@@ -80,6 +83,7 @@ struct Field {
   std::span<const std::string_view> type_names;
   std::span<const Field> fields;
   std::span<const ProcessorInst> processors;
+  bool is_primitive;
 };
 
 template <size_t N0, size_t N1, size_t N2>
@@ -88,13 +92,15 @@ constexpr Field::Field(size_t static_size_,
                        std::string_view name_,
                        const std::array<std::string_view, N0>& type_names_,
                        const std::array<Field, N1>& fields_,
-                       const std::array<ProcessorInst, N2>& processors_)
+                       const std::array<ProcessorInst, N2>& processors_,
+                       bool is_primitive_)
     : static_size(static_size_),
       exclusive_size(exclusive_size_),
       name(name_),
       type_names(type_names_),
       fields(fields_),
-      processors(processors_) {
+      processors(processors_),
+      is_primitive(is_primitive_) {
 }
 
 }  // namespace oi::exporters::inst
diff --git a/include/oi/result/Element.h b/include/oi/result/Element.h
index 1eea86b..c971236 100644
--- a/include/oi/result/Element.h
+++ b/include/oi/result/Element.h
@@ -52,6 +52,7 @@ struct Element {
       std::nullopt};
   std::optional<ContainerStats> container_stats;
   std::optional<IsSetStats> is_set_stats;
+  bool is_primitive;
 };
 
 }  // namespace oi::result
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 84fb9b4..cb5bf54 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -798,12 +798,15 @@ void CodeGen::genClassTreeBuilderInstructions(const Class& c,
     if (m.name.starts_with(AddPadding::MemberPrefix))
       continue;
     std::string fullName = c.name() + "::" + m.name;
+    bool isPrimitive = dynamic_cast<const Primitive*>(&m.type());
     code += "      inst::Field{sizeof(" + fullName + "), " +
             std::to_string(calculateExclusiveSize(m.type())) + ",\"" +
             m.inputName + "\", member_" + std::to_string(index) +
             "_type_names, TypeHandler<Ctx, decltype(" + fullName +
             ")>::fields, TypeHandler<Ctx, decltype(" + fullName +
-            ")>::processors},\n";
+            ")>::processors, ";
+    code += isPrimitive ? "true" : "false";
+    code += "},\n";
   }
   code += "  };\n";
   code +=
@@ -1074,6 +1077,7 @@ constexpr inst::Field make_field(std::string_view name) {
     NameProvider<T>::names,
     TypeHandler<Ctx, T>::fields,
     TypeHandler<Ctx, T>::processors,
+    std::is_fundamental_v<T>
   };
 }
 )";
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 97d2707..5ac3abf 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -399,7 +399,8 @@ const std::array<std::string_view, )";
       ", OIInternal::TypeHandler<FakeContext, "
       "OIInternal::__ROOT_TYPE__>::fields, "
       "OIInternal::TypeHandler<FakeContext, "
-      "OIInternal::__ROOT_TYPE__>::processors};\n";
+      "OIInternal::__ROOT_TYPE__>::processors, "
+      "std::is_fundamental_v<OIInternal::__ROOT_TYPE__>};\n";
   code += "} // namespace\n";
   code +=
       "extern const exporters::inst::Inst __attribute__((used, retain)) "
@@ -640,13 +641,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
         }
         static void process_pointer_content(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
           static constexpr std::array<std::string_view, 1> names{"TODO"};
-          static constexpr auto childField = inst::Field{
-            sizeof(T),
-            "*",
-            names,
-            TypeHandler<Ctx, T>::fields,
-            TypeHandler<Ctx, T>::processors,
-          };
+          static constexpr auto childField = make_field<Ctx, T>("*");
 
           const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
 
diff --git a/oi/IntrospectionResult.cpp b/oi/IntrospectionResult.cpp
index bc59f5e..5310001 100644
--- a/oi/IntrospectionResult.cpp
+++ b/oi/IntrospectionResult.cpp
@@ -78,6 +78,7 @@ IntrospectionResult::const_iterator::operator++() {
                 .exclusive_size = ty.exclusive_size,
                 .container_stats = std::nullopt,
                 .is_set_stats = std::nullopt,
+                .is_primitive = ty.is_primitive,
             };
 
             for (const auto& [dy, handler] : ty.processors) {
diff --git a/test/integration/simple.toml b/test/integration/simple.toml
index 0945f70..4557319 100644
--- a/test/integration/simple.toml
+++ b/test/integration/simple.toml
@@ -24,12 +24,21 @@ definitions = '''
       "staticSize":16,
       "dynamicSize":0,
       "exclusiveSize": 3,
-      "size": 16,
       "members":[
         {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
         {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize": 3,
+      "size": 16,
+      "is_primitive": false,
+      "members":[
+        {"name":"a", "staticSize":4, "exclusiveSize": 4, "size": 4, "is_primitive": true},
+        {"name":"b", "staticSize":1, "exclusiveSize": 1, "size": 1, "is_primitive": true},
+        {"name":"c", "staticSize":8, "exclusiveSize": 8, "size": 8, "is_primitive": true}
+      ]}]'''
   [cases.class]
     param_types = ["const SimpleClass&"]
     setup = "return {};"
@@ -37,18 +46,32 @@ definitions = '''
       "staticSize":16,
       "dynamicSize":0,
       "exclusiveSize": 3,
-      "size": 16,
       "members":[
         {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
         {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize": 3,
+      "size": 16,
+      "is_primitive": false,
+      "members":[
+        {"name":"a", "staticSize":4, "exclusiveSize": 4, "size": 4, "is_primitive": true},
+        {"name":"b", "staticSize":1, "exclusiveSize": 1, "size": 1, "is_primitive": true},
+        {"name":"c", "staticSize":8, "exclusiveSize": 8, "size": 8, "is_primitive": true}
+      ]}]'''
   [cases.union]
     param_types = ["const SimpleUnion&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
-      "exclusiveSize":8,
-      "size":8
+      "exclusiveSize":8
+    }]'''
+    expect_json_v2 = '''[{
+      "staticSize":8,
+      "exclusiveSize":8,
+      "size":8,
+      "is_primitive":false
     }]'''
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index 06c6f72..69a5173 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index de1dcda..cc89f06 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index c09bbf0..aa6aa59 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index a673bbd..f640dcc 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index a09efff..f4281eb 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -91,6 +91,7 @@ static constexpr auto entry = inst::Field {
   std::array<std::string_view, 0>{},
   entryFields,
   processors,
+  entryFields[0].is_primitive && entryFields[1].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index a5d628f..e313417 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -118,6 +118,7 @@ static constexpr auto element = inst::Field {
   std::array<std::string_view, 0>{},
   elementFields,
   std::array<inst::ProcessorInst, 0>{},
+  elementFields[0].is_primitive && elementFields[1].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index ee652ec..b972b70 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -135,6 +135,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   processors,
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index 5b2402f..108731e 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -135,6 +135,7 @@ static constexpr auto element = inst::Field{
   std::array<std::string_view, 0>{},
   element_fields,
   processors,
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index a6f7671..8452154 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -121,6 +121,7 @@ static constexpr auto element = inst::Field{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)

From 166bb5eeacac920e44383a140a18e5bd2e88c251 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 11 Jan 2024 17:58:25 +0000
Subject: [PATCH 129/188] type_graph: avoid overwriting explicitly set
 alignment

Previously AlignmentCalc calculates the alignment and sets packing for every
type except a member with explicit alignment. Change this to check whether an
alignment has been previously set for a type before calculating it. Use this in
ClangTypeParser where the full alignment of the type is available.

Remove explicitly aligning members by the type because that was previously
reserved for members with explicit alignment. AlignmentCalc will correctly
align a member to the underlying type without this. Explicit member alignment
is still missing, as before this change.

Test plan:
- CI
- Too little. Gets further into a production type than without this change.
---
 oi/type_graph/AlignmentCalc.cpp   | 30 ++++++++++++++++--------------
 oi/type_graph/ClangTypeParser.cpp |  4 ++--
 2 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/oi/type_graph/AlignmentCalc.cpp b/oi/type_graph/AlignmentCalc.cpp
index cf23eff..9abad71 100644
--- a/oi/type_graph/AlignmentCalc.cpp
+++ b/oi/type_graph/AlignmentCalc.cpp
@@ -50,24 +50,26 @@ void AlignmentCalc::accept(Type& type) {
 void AlignmentCalc::visit(Class& c) {
   RecursiveVisitor::visit(c);
 
-  uint64_t alignment = 1;
-  for (auto& member : c.members) {
-    if (member.align == 0) {
-      // If the member does not have an explicit alignment, calculate it from
-      // the member's type.
-      accept(member.type());
-      member.align = member.type().align();
-    }
-    alignment = std::max(alignment, member.align);
+  if (c.align() == 0) {
+    uint64_t alignment = 1;
+    for (auto& member : c.members) {
+      if (member.align == 0) {
+        // If the member does not have an explicit alignment, calculate it from
+        // the member's type.
+        accept(member.type());
+        member.align = member.type().align();
+      }
+      alignment = std::max(alignment, member.align);
 
-    if (member.align != 0 && (member.bitOffset / 8) % member.align != 0) {
-      // Mark as packed if members are not aligned
-      c.setPacked();
+      if (member.align != 0 && (member.bitOffset / 8) % member.align != 0) {
+        // Mark as packed if members are not aligned
+        c.setPacked();
+      }
     }
+
+    c.setAlign(alignment);
   }
 
-  c.setAlign(alignment);
-
   if (c.size() % c.align() != 0) {
     // Mark as packed if there is no tail padding
     c.setPacked();
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index 24d5835..3a13903 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -175,7 +175,7 @@ Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
   if (auto* info = getContainerInfo(fqName)) {
     auto& c = makeType<Container>(ty, *info, size, nullptr);
     enumerateClassTemplateParams(ty, c.templateParams);
-    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)));
+    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)) / 8);
     return c;
   }
 
@@ -194,6 +194,7 @@ Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
 
   auto& c = makeType<Class>(
       ty, kind, std::move(name), std::move(fqName), size, virtuality);
+  c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)) / 8);
 
   enumerateClassTemplateParams(ty, c.templateParams);
   // enumerateClassParents(type, c.parents);
@@ -294,7 +295,6 @@ void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
 
     auto& mtype = enumerateType(*qualType);
     Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
-    m.align = decl->getASTContext().getTypeAlign(qualType) / 8;
     members.push_back(m);
   }
 

From 18005b4505ef8a182c7e213670594b71009a59e8 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 11 Jan 2024 17:59:06 +0000
Subject: [PATCH 130/188] tbv2: remove unnecessary copy in Element

`IntrospectionResult::const_iterator` iterates through the `Element`s in an
`IntrospectionResult`. `Element` currently copies the `type_path` which is a
`std::vector<string_view>` every time the iterator is incremented. This is
unnecessary as the data in the vector only changes slightly between iterations.

This change changes the `type_path` field in `Element` to a
`std::span<const std::string_view>`. Doing this previously caused SEGVs because
of the iterator's potential to be copied. To make it possible we do two things:
1. Make all copies explicit using a clone interface as in `ContainerInfo`. This
   prevents accidental copies of an expensive structure.
2. After calling the copy constructor in `clone()` update the `span` in `next_`
   to point at the newly copied structure.

Moves are fine because the `span` points at the allocation of the `vector`, not
the vector itself.

Test plan:
- CI
- `FILTER='OilIntegration.*' make test`
- Ran `OilgenIntegration.std_vector_vector_int_some` which SEGVd with the
  `span` change before and now doesn't. This now passes cleanly with ASAN
  enabled on the target, though isn't available in `main` (only works on my
  machine).
---
 include/oi/IntrospectionResult-inl.h | 16 +++++++++++++++-
 include/oi/IntrospectionResult.h     | 10 ++++++++++
 include/oi/result/Element.h          |  3 +--
 include/oi/result/SizedResult-inl.h  |  9 ++++++---
 oi/IntrospectionResult.cpp           |  5 ++---
 5 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/include/oi/IntrospectionResult-inl.h b/include/oi/IntrospectionResult-inl.h
index 13928b5..a0866a7 100644
--- a/include/oi/IntrospectionResult-inl.h
+++ b/include/oi/IntrospectionResult-inl.h
@@ -45,7 +45,9 @@ inline IntrospectionResult::const_iterator IntrospectionResult::begin() const {
   return cbegin();
 }
 inline IntrospectionResult::const_iterator IntrospectionResult::cbegin() const {
-  return ++const_iterator{buf_.cbegin(), inst_};
+  auto it = const_iterator{buf_.cbegin(), inst_};
+  ++it;
+  return it;
 }
 inline IntrospectionResult::const_iterator IntrospectionResult::end() const {
   return cend();
@@ -70,6 +72,18 @@ inline const result::Element* IntrospectionResult::const_iterator::operator->()
   return &*next_;
 }
 
+inline IntrospectionResult::const_iterator
+IntrospectionResult::const_iterator::clone() const {
+  auto ret{*this};
+
+  // Fix the self referential type_path_ field in next_
+  if (ret.next_.has_value()) {
+    ret.next_->type_path = ret.type_path_;
+  }
+
+  return ret;
+}
+
 inline bool IntrospectionResult::const_iterator::operator==(
     const IntrospectionResult::const_iterator& that) const {
   // Case 1: The next data to read differs, thus the iterators are different.
diff --git a/include/oi/IntrospectionResult.h b/include/oi/IntrospectionResult.h
index 091028f..1675423 100644
--- a/include/oi/IntrospectionResult.h
+++ b/include/oi/IntrospectionResult.h
@@ -43,6 +43,16 @@ class IntrospectionResult {
     const_iterator& operator++();
     const_iterator operator++(int);
 
+   private:
+    const_iterator(const const_iterator&) = default;
+    const_iterator& operator=(const const_iterator& other) = default;
+
+   public:
+    const_iterator(const_iterator&&) = default;
+    const_iterator& operator=(const_iterator&&) = default;
+    // Explicit interface for copying
+    const_iterator clone() const;
+
    private:
     using stack_t =
         std::stack<exporters::inst::Inst, std::vector<exporters::inst::Inst>>;
diff --git a/include/oi/result/Element.h b/include/oi/result/Element.h
index 1eea86b..5610c1a 100644
--- a/include/oi/result/Element.h
+++ b/include/oi/result/Element.h
@@ -41,8 +41,7 @@ struct Element {
   };
 
   std::string_view name;
-  std::vector<std::string_view>
-      type_path;  // TODO: should be span<const std::string_view>
+  std::span<const std::string_view> type_path;
   std::span<const std::string_view> type_names;
   size_t static_size;
   size_t exclusive_size;
diff --git a/include/oi/result/SizedResult-inl.h b/include/oi/result/SizedResult-inl.h
index f86ff0c..94418a4 100644
--- a/include/oi/result/SizedResult-inl.h
+++ b/include/oi/result/SizedResult-inl.h
@@ -35,7 +35,9 @@ SizedResult<Res>::SizedResult(Res res) : res_{std::move(res)} {
 
 template <typename Res>
 typename SizedResult<Res>::const_iterator SizedResult<Res>::begin() const {
-  return ++const_iterator{res_.begin(), res_.end()};
+  const_iterator it{res_.begin(), res_.end()};
+  ++it;
+  return it;
 }
 template <typename Res>
 typename SizedResult<Res>::const_iterator SizedResult<Res>::end() const {
@@ -44,7 +46,7 @@ typename SizedResult<Res>::const_iterator SizedResult<Res>::end() const {
 
 template <typename Res>
 SizedResult<Res>::const_iterator::const_iterator(It it, const It& end)
-    : data_{it} {
+    : data_{it.clone()} {
   struct StackEntry {
     size_t index;
     size_t depth;
@@ -75,7 +77,8 @@ SizedResult<Res>::const_iterator::const_iterator(It it, const It& end)
 }
 
 template <typename Res>
-SizedResult<Res>::const_iterator::const_iterator(It end) : data_{end} {
+SizedResult<Res>::const_iterator::const_iterator(It end)
+    : data_{std::move(end)} {
 }
 
 template <typename Res>
diff --git a/oi/IntrospectionResult.cpp b/oi/IntrospectionResult.cpp
index bc59f5e..b8a6814 100644
--- a/oi/IntrospectionResult.cpp
+++ b/oi/IntrospectionResult.cpp
@@ -70,7 +70,7 @@ IntrospectionResult::const_iterator::operator++() {
           if constexpr (std::is_same_v<T, exporters::inst::Field>) {
             type_path_.emplace_back(ty.name);
             stack_.emplace(exporters::inst::PopTypePath{});
-            next_ = result::Element{
+            next_.emplace(result::Element{
                 .name = ty.name,
                 .type_path = type_path_,
                 .type_names = ty.type_names,
@@ -78,7 +78,7 @@ IntrospectionResult::const_iterator::operator++() {
                 .exclusive_size = ty.exclusive_size,
                 .container_stats = std::nullopt,
                 .is_set_stats = std::nullopt,
-            };
+            });
 
             for (const auto& [dy, handler] : ty.processors) {
               auto parsed = exporters::ParsedData::parse(data_, dy);
@@ -93,7 +93,6 @@ IntrospectionResult::const_iterator::operator++() {
                       .second;
 
               type_path_.back() = new_name_ref;
-              next_->type_path.back() = new_name_ref;
               next_->name = new_name_ref;
             }
 

From af1dc4acdc843955ef19b9b9c8db8ac382c67608 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 12 Jan 2024 21:43:30 +0000
Subject: [PATCH 131/188] test: add features field to integration tests

Previously we tested different feature flags by using the `cli_options` field
in the test `.toml`. This works for OID but doesn't work for JIT OIL and won't
work for AoT OIL when those tests get added.

This change adds a new higher level `features` field to the test `.toml` which
adds the features to the config file as a prefix. This works with all methods
of generation.

Change the existing `cli_options` features to `features` except for where
they're testing something specific. Enable tests that were previously disabled
for OIL but only didn't work because of not being able to enable features.
Change pointer tests that are currently broken for OIL from `oil_disable` to
`oil_skip` - they can work, but codegen is broken for them at the minute.

Test plan:
- CI
- `make test` is no worse
---
 test/integration/README.md                    | 12 +++++++-
 test/integration/anonymous.toml               | 19 ++++++------
 test/integration/arrays.toml                  |  2 +-
 test/integration/bitfields.toml               | 14 ++++-----
 test/integration/cycles.toml                  |  8 ++---
 test/integration/gen_tests.py                 | 19 +++++++++---
 test/integration/inheritance_polymorphic.toml | 12 ++++----
 .../inheritance_polymorphic_diamond.toml      | 20 ++++++-------
 ...eritance_polymorphic_non_dynamic_base.toml | 12 ++++----
 test/integration/pointers.toml                | 26 +++++++++--------
 test/integration/pointers_function.toml       | 10 +++----
 test/integration/pointers_incomplete.toml     | 29 +++++++++----------
 test/integration/runner_common.h              |  2 ++
 test/integration/std_smart_ptr.toml           |  4 +--
 test/integration/thrift_isset.toml            | 12 ++++----
 test/integration/thrift_isset_missing.toml    |  6 ++--
 test/integration/thrift_namespaces.toml       |  2 +-
 17 files changed, 115 insertions(+), 94 deletions(-)

diff --git a/test/integration/README.md b/test/integration/README.md
index 4d7d734..08759aa 100644
--- a/test/integration/README.md
+++ b/test/integration/README.md
@@ -201,13 +201,23 @@ definitions = '''
 
     Implies `oil_disable`.
 
+  - `features`
+
+    Append this list of features to the configuration. This works for all types
+    of test.
+
+    Example:
+    ```
+    features = ["chase-raw-pointers"]
+    ```
+
   - `cli_options`
 
     Additional command line arguments passed to oid.
 
     Example:
     ```
-    cli_options = ["-fchase-raw-pointers"]
+    cli_options = ["-Ftype-graph"]
     ```
 
   - `skip`, `oid_skip`, `oil_skip`
diff --git a/test/integration/anonymous.toml b/test/integration/anonymous.toml
index 15b9f10..70600d1 100644
--- a/test/integration/anonymous.toml
+++ b/test/integration/anonymous.toml
@@ -84,7 +84,7 @@ definitions = '''
   [cases.regular_struct]
     param_types = ["const Node&"]
     setup = "return Node{1, 2, 3};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":12,
       "dynamicSize":0,
@@ -97,7 +97,7 @@ definitions = '''
       ]}]'''
 
   [cases.anon_struct]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const AnonStructContainer&"]
     setup = '''
       return AnonStructContainer{
@@ -106,7 +106,7 @@ definitions = '''
         }
       };
     '''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 8,
       "dynamicSize": 12,
@@ -143,10 +143,10 @@ definitions = '''
           new Node{1, 2, 3}
         }
       };'''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
 
   [cases.anon_typedef]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const AnonTypedefContainer&"]
     setup = '''
       return AnonTypedefContainer{
@@ -154,7 +154,7 @@ definitions = '''
           .node = new Node{1, 2, 3}
         }
       };'''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 8,
       "dynamicSize": 12,
@@ -196,7 +196,6 @@ definitions = '''
   [cases.anon_union]
     param_types = ["const AnonUnionContainer&"]
     setup = 'return AnonUnionContainer{ .a = 3 };'
-    cli_options = ["-fchase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 24,
       "dynamicSize": 0,
@@ -218,10 +217,10 @@ definitions = '''
     }]'''
 
   [cases.nested_anon_struct]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const NestedAnonContainer&"]
+    features = ["chase-raw-pointers"]
     setup = 'return NestedAnonContainer{.m = { .v = {.as = {new Node{1, 2, 3}}}}};'
-    cli_options = ["-fchase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 80,
       "dynamicSize": 12,
@@ -290,7 +289,7 @@ definitions = '''
   #     result.ns[3].nodes.resize(22);
   #     return result;
   #   '''
-  #   cli_options = ["-fchase-raw-pointers"]
+  #   features = ["chase-raw-pointers"]
   #   expect_json = '''[{
   #     "staticSize": 104,
   #     "dynamicSize": 556,
diff --git a/test/integration/arrays.toml b/test/integration/arrays.toml
index 25e542a..9178559 100644
--- a/test/integration/arrays.toml
+++ b/test/integration/arrays.toml
@@ -76,7 +76,7 @@ definitions = '''
         "elementStaticSize":4
       }]}]'''
   [cases.multidim]
-    cli_options = ["-ftype-graph"]
+    
     param_types = ["const MultiDim&"]
     setup = "return {};"
     expect_json = '''[
diff --git a/test/integration/bitfields.toml b/test/integration/bitfields.toml
index bd1e052..6f66647 100644
--- a/test/integration/bitfields.toml
+++ b/test/integration/bitfields.toml
@@ -53,7 +53,7 @@ definitions = '''
 # if they were regular primitives.
 [cases]
   [cases.single]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["Single&"]
     setup = "return {};"
@@ -62,7 +62,7 @@ definitions = '''
         {"staticSize":4, "dynamicSize":0, "exclusiveSize":4}
     ]}]'''
   [cases.within_bytes]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["WithinBytes&"]
     setup = "return {};"
@@ -73,7 +73,7 @@ definitions = '''
         {"staticSize":1, "dynamicSize":0, "exclusiveSize":1}
     ]}]'''
   [cases.straddle_bytes]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["StraddleBytes&"]
     setup = "return {};"
@@ -84,7 +84,7 @@ definitions = '''
         {"staticSize":1, "dynamicSize":0, "exclusiveSize":1}
     ]}]'''
   [cases.mixed]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["Mixed&"]
     setup = "return {};"
@@ -98,13 +98,13 @@ definitions = '''
     ]}]'''
   [cases.more_bits_than_type] # TODO member sizes are wrong
     skip = "drgn errors out"
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["MoreBitsThanType&"]
     setup = "return {};"
     expect_json = '"TODO"'
   [cases.zero_bits]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["ZeroBits&"]
     setup = "return {};"
@@ -114,7 +114,7 @@ definitions = '''
         {"staticSize":1, "dynamicSize":0, "exclusiveSize":1}
     ]}]'''
   [cases.enum]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["Enum&"]
     setup = "return {};"
diff --git a/test/integration/cycles.toml b/test/integration/cycles.toml
index d6f6daf..77b0d0b 100644
--- a/test/integration/cycles.toml
+++ b/test/integration/cycles.toml
@@ -23,7 +23,7 @@ definitions = '''
 '''
 [cases]
   [cases.raw_ptr]
-    oil_disable = "oil can't chase pointers safely"
+    oil_skip = "cycles are broken" # https://github.com/facebookexperimental/object-introspection/issues/293
     param_types = ["RawNode*"]
     setup = '''
       RawNode *first  = new RawNode{1, nullptr};
@@ -34,7 +34,7 @@ definitions = '''
       third->next = first;
       return first;
     '''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''
     [
       {
@@ -96,7 +96,7 @@ definitions = '''
     '''
 
   [cases.raw_ptr_wrapped]
-    oil_disable = "oil can't chase pointers safely"
+    oil_skip = "cycles are broken" # https://github.com/facebookexperimental/object-introspection/issues/293
     param_types = ["Wrapper<RawNode*>&"]
     setup = '''
       RawNode *first  = new RawNode{1, nullptr};
@@ -107,7 +107,7 @@ definitions = '''
       third->next = first;
       return Wrapper<RawNode*>(first);
     '''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 8,
       "dynamicSize": 48,
diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py
index b72e371..5dc349b 100644
--- a/test/integration/gen_tests.py
+++ b/test/integration/gen_tests.py
@@ -268,6 +268,19 @@ def add_tests(f, config):
         add_oil_integration_test(f, config, case_name, case)
 
 
+def get_config_strings(case):
+    config_prefix = case.get("config_prefix", "")
+    if "features" in case:
+        feature_config = "features = [";
+        for f in case.get("features", []):
+            feature_config += '"' + f + '"'
+        feature_config += "]\n"
+        config_prefix = feature_config + config_prefix
+    config_suffix = case.get("config_suffix", "")
+
+    return (config_prefix, config_suffix)
+
+
 def add_oid_integration_test(f, config, case_name, case):
     probe_type = case.get("type", "entry")
     args = case.get("args", "arg0")
@@ -283,8 +296,7 @@ def add_oid_integration_test(f, config, case_name, case):
         "{" + ", ".join(f'"{option}"' for option in case.get("cli_options", ())) + "}"
     )
 
-    config_prefix = case.get("config_prefix", "")
-    config_suffix = case.get("config_suffix", "")
+    config_prefix, config_suffix = get_config_strings(case)
 
     f.write(
         f"\n"
@@ -356,8 +368,7 @@ def add_oil_integration_test(f, config, case_name, case):
     if "oil_disable" in case or "target_function" in case:
         return
 
-    config_prefix = case.get("config_prefix", "")
-    config_suffix = case.get("config_suffix", "")
+    config_prefix, config_suffix = get_config_strings(case)
 
     f.write(
         f"\n"
diff --git a/test/integration/inheritance_polymorphic.toml b/test/integration/inheritance_polymorphic.toml
index fe50462..0b9e577 100644
--- a/test/integration/inheritance_polymorphic.toml
+++ b/test/integration/inheritance_polymorphic.toml
@@ -24,7 +24,7 @@ definitions = '''
 [cases]
   [cases.a_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["A"]
     setup = "return {};"
@@ -39,7 +39,7 @@ definitions = '''
 
   [cases.b_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["B"]
     setup = '''
@@ -58,7 +58,7 @@ definitions = '''
       ]}]'''
   [cases.b_as_b]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const B&"]
     arg_types = ["B"]
     setup = '''
@@ -78,7 +78,7 @@ definitions = '''
 
   [cases.c_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["C"]
     setup = '''
@@ -98,7 +98,7 @@ definitions = '''
       ]}]'''
   [cases.c_as_b]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const B&"]
     arg_types = ["C"]
     setup = '''
@@ -118,7 +118,7 @@ definitions = '''
       ]}]'''
   [cases.c_as_c]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const C&"]
     arg_types = ["C"]
     setup = '''
diff --git a/test/integration/inheritance_polymorphic_diamond.toml b/test/integration/inheritance_polymorphic_diamond.toml
index ec9a0e1..cff0638 100644
--- a/test/integration/inheritance_polymorphic_diamond.toml
+++ b/test/integration/inheritance_polymorphic_diamond.toml
@@ -31,7 +31,7 @@ definitions = '''
 [cases]
   [cases.root_as_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Root&"]
     arg_types = ["Root"]
     setup = "return {};"
@@ -46,7 +46,7 @@ definitions = '''
 
   [cases.middle1_as_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Root&"]
     arg_types = ["Middle1"]
     setup = '''
@@ -65,7 +65,7 @@ definitions = '''
       ]}]'''
   [cases.middle1_as_middle1]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Middle1&"]
     arg_types = ["ns_inheritance_polymorphic_diamond::Middle1"]
     setup = '''
@@ -85,7 +85,7 @@ definitions = '''
 
   [cases.middle2_as_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Root&"]
     arg_types = ["Middle2"]
     setup = '''
@@ -104,7 +104,7 @@ definitions = '''
       ]}]'''
   [cases.middle2_as_middle2]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Middle2&"]
     arg_types = ["Middle2"]
     setup = '''
@@ -124,7 +124,7 @@ definitions = '''
 
   [cases.child_as_middle1_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     # We need to explicitly cast from Child to Middle1 before going to root to
     # resolve the diamond problem
     param_types = ["const Root&"]
@@ -150,7 +150,7 @@ definitions = '''
       ]}]'''
   [cases.child_as_middle2_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     # We need to explicitly cast from Child to Middle2 before going to root to
     # resolve the diamond problem
     param_types = ["const Root&"]
@@ -176,7 +176,7 @@ definitions = '''
       ]}]'''
   [cases.child_as_middle1]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Middle1&"]
     arg_types = ["Child"]
     setup = '''
@@ -200,7 +200,7 @@ definitions = '''
       ]}]'''
   [cases.child_as_middle2]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Middle2&"]
     arg_types = ["Child"]
     setup = '''
@@ -224,7 +224,7 @@ definitions = '''
       ]}]'''
   [cases.child_as_child]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Child&"]
     arg_types = ["Child"]
     setup = '''
diff --git a/test/integration/inheritance_polymorphic_non_dynamic_base.toml b/test/integration/inheritance_polymorphic_non_dynamic_base.toml
index fff0199..e482e92 100644
--- a/test/integration/inheritance_polymorphic_non_dynamic_base.toml
+++ b/test/integration/inheritance_polymorphic_non_dynamic_base.toml
@@ -22,7 +22,7 @@ definitions = '''
 [cases]
   [cases.a_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["A"]
     setup = "return {};"
@@ -54,7 +54,7 @@ definitions = '''
 
   [cases.b_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["B"]
     setup = '''
@@ -71,7 +71,7 @@ definitions = '''
       ]}]'''
   [cases.b_as_b]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const B&"]
     arg_types = ["B"]
     setup = '''
@@ -109,7 +109,7 @@ definitions = '''
 
   [cases.c_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["C"]
     setup = '''
@@ -126,7 +126,7 @@ definitions = '''
       ]}]'''
   [cases.c_as_b]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const B&"]
     arg_types = ["C"]
     setup = '''
@@ -146,7 +146,7 @@ definitions = '''
       ]}]'''
   [cases.c_as_c]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const C&"]
     arg_types = ["C"]
     setup = '''
diff --git a/test/integration/pointers.toml b/test/integration/pointers.toml
index 71bf2ad..cfd3627 100644
--- a/test/integration/pointers.toml
+++ b/test/integration/pointers.toml
@@ -17,7 +17,7 @@ definitions = '''
     skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19
     param_types = ["int*"]
     setup = "return new int(1);"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "typeName": "int *",
       "staticSize": 8,
@@ -59,7 +59,7 @@ definitions = '''
     skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19
     param_types = ["void*"]
     setup = "return new int(1);"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "typeName": "void *",
       "staticSize": 8,
@@ -95,7 +95,7 @@ definitions = '''
     skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19
     param_types = ["std::vector<int>*"]
     setup = "return new std::vector<int>{1,2,3};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "typeName": "std::vector<int> *",
       "staticSize": 8,
@@ -134,10 +134,10 @@ definitions = '''
 
 
   [cases.struct_primitive_ptrs]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, new int(0), new int(0)};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":4,
@@ -161,9 +161,10 @@ definitions = '''
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_primitive_ptrs_null]
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, nullptr, nullptr};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
@@ -177,10 +178,10 @@ definitions = '''
 
 
   [cases.struct_vector_ptr]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{new std::vector<int>{1,2,3}};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":36,
@@ -199,9 +200,10 @@ definitions = '''
         {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_vector_ptr_null]
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{nullptr};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
@@ -213,10 +215,10 @@ definitions = '''
 
 
   [cases.vector_of_pointers]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const std::vector<int*>&"]
     setup = "return {{new int(1), nullptr, new int(3)}};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":32,
@@ -258,7 +260,7 @@ definitions = '''
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.feature_config]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const std::vector<int*>&"]
     setup = "return {{new int(1), nullptr, new int(3)}};"
     config_prefix = 'features = ["chase-raw-pointers"]'
diff --git a/test/integration/pointers_function.toml b/test/integration/pointers_function.toml
index a62ef9a..1c7934f 100644
--- a/test/integration/pointers_function.toml
+++ b/test/integration/pointers_function.toml
@@ -29,11 +29,10 @@ definitions = '''
       }]
     }]'''
   [cases.raw_chase] # We should never chase function pointers
-    oid_skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22
-    oil_disable = "oil can't chase raw pointers safely"
+    skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22
     param_types = ["const FuncPtrStruct&"]
     setup = "return {{myFunction}};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 8,
       "dynamicSize": 0,
@@ -73,11 +72,10 @@ definitions = '''
       "NOT": "members"
     }]'''
   [cases.std_function_chase] # We should never chase function pointers
-    oid_skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22
-    oil_disable = "oil can't chase raw pointers safely"
+    skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22
     param_types = ["std::function<void(int)> &"]
     setup = "return myFunction;"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "typeName": "function<void (int)>",
       "staticSize": 32,
diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml
index 2d34865..87af9ad 100644
--- a/test/integration/pointers_incomplete.toml
+++ b/test/integration/pointers_incomplete.toml
@@ -19,38 +19,37 @@ definitions = '''
 '''
 [cases]
   [cases.raw]
-    oil_disable = "oil can't chase raw pointers safely"
     oid_skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oil_skip = "oil codegen fails on top level incomplete objects" # https://github.com/facebookexperimental/object-introspection/issues/459
     param_types = ["IncompleteType*"]
     setup = "return static_cast<IncompleteType*>(::operator new(5));"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
-      "typeName": "IncompleteType *",
-      "staticSize": 8,
+      "typeName": "IncompleteType",
+      "staticSize": 0,
       "dynamicSize": 0,
-      "NOT": {"pointer": 0},
       "NOT": "members"
     }]'''
   [cases.raw_no_follow]
-    skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oid_skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oil_skip = "oil codegen fails on top level incomplete objects" # https://github.com/facebookexperimental/object-introspection/issues/459
     param_types = ["IncompleteType*"]
     setup = "return static_cast<IncompleteType*>(::operator new(5));"
     expect_json = '''[{
-      "typeName": "IncompleteType *",
-      "staticSize": 8,
+      "typeName": "IncompleteType",
+      "staticSize": 0,
       "dynamicSize": 0,
-      "NOT": {"pointer": 0},
       "NOT": "members"
     }]'''
   [cases.raw_null]
-    skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oid_skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oil_skip = "oil codegen fails on top level incomplete objects" # https://github.com/facebookexperimental/object-introspection/issues/459
     param_types = ["IncompleteType*"]
     setup = "return nullptr;"
     expect_json = '''[{
-      "typeName": "IncompleteType *",
-      "staticSize": 8,
+      "typeName": "IncompleteType",
+      "staticSize": 0,
       "dynamicSize": 0,
-      "pointer": 0,
       "NOT": "members"
     }]'''
 
@@ -87,10 +86,10 @@ definitions = '''
     expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.containing_struct]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const IncompleteTypeContainer&"]
     setup = "return IncompleteTypeContainer{};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 88,
       "dynamicSize": 0,
diff --git a/test/integration/runner_common.h b/test/integration/runner_common.h
index d2b5e57..d1f2bae 100644
--- a/test/integration/runner_common.h
+++ b/test/integration/runner_common.h
@@ -51,6 +51,8 @@ class IntegrationBase : public ::testing::Test {
   std::string stdout_;
   std::string stderr_;
 
+  std::vector<std::string> features;
+
   /*
    * compare_json
    *
diff --git a/test/integration/std_smart_ptr.toml b/test/integration/std_smart_ptr.toml
index dfb86ae..c9ab383 100644
--- a/test/integration/std_smart_ptr.toml
+++ b/test/integration/std_smart_ptr.toml
@@ -373,7 +373,7 @@ definitions = '''
     expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_present_chase]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     setup = '''
     static std::shared_ptr<std::uint64_t> shd = std::make_shared<std::uint64_t>(0xDEADBEEF);
     std::weak_ptr<std::uint64_t> weak = shd;
@@ -392,7 +392,7 @@ definitions = '''
     expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_expired_chase]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     setup = '''
     return std::make_shared<std::uint64_t>(0xDEADBEEF);
     '''
diff --git a/test/integration/thrift_isset.toml b/test/integration/thrift_isset.toml
index 064cb13..98e19fd 100644
--- a/test/integration/thrift_isset.toml
+++ b/test/integration/thrift_isset.toml
@@ -85,7 +85,7 @@ namespace cpp2 {
       ret.c_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
@@ -109,7 +109,7 @@ namespace cpp2 {
       ret.j_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":44,
       "dynamicSize":0,
@@ -136,7 +136,7 @@ namespace cpp2 {
       ret.c_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":20,
       "dynamicSize":0,
@@ -156,7 +156,7 @@ namespace cpp2 {
       ret.b_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
@@ -176,7 +176,7 @@ namespace cpp2 {
       ret.f_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":28,
       "dynamicSize":0,
@@ -199,7 +199,7 @@ namespace cpp2 {
       ret.e_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":32,
       "dynamicSize":0,
diff --git a/test/integration/thrift_isset_missing.toml b/test/integration/thrift_isset_missing.toml
index 63b578f..3e7ad94 100644
--- a/test/integration/thrift_isset_missing.toml
+++ b/test/integration/thrift_isset_missing.toml
@@ -68,7 +68,7 @@ raw_definitions = '''
       ret.__isset.at(2) = true;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
@@ -89,7 +89,7 @@ raw_definitions = '''
       ret.__isset.at(2) = true;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
@@ -117,7 +117,7 @@ raw_definitions = '''
 
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":32,
       "dynamicSize":0,
diff --git a/test/integration/thrift_namespaces.toml b/test/integration/thrift_namespaces.toml
index f2ce9f9..2218ba3 100644
--- a/test/integration/thrift_namespaces.toml
+++ b/test/integration/thrift_namespaces.toml
@@ -16,7 +16,7 @@ thrift_definitions = '''
       namespaceA::namespaceB::TTTTT ret;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,

From d1ff72553e08034903b736abe94c18a93538a4e9 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 12 Jan 2024 21:43:30 +0000
Subject: [PATCH 132/188] test: add features field to integration tests

Previously we tested different feature flags by using the `cli_options` field
in the test `.toml`. This works for OID but doesn't work for JIT OIL and won't
work for AoT OIL when those tests get added.

This change adds a new higher level `features` field to the test `.toml` which
adds the features to the config file as a prefix. This works with all methods
of generation.

Change the existing `cli_options` features to `features` except for where
they're testing something specific. Enable tests that were previously disabled
for OIL but only didn't work because of not being able to enable features.
Change pointer tests that are currently broken for OIL from `oil_disable` to
`oil_skip` - they can work, but codegen is broken for them at the minute.

Test plan:
- CI
- `make test` is no worse
---
 test/integration/README.md                    | 12 +++++++-
 test/integration/anonymous.toml               | 19 ++++++------
 test/integration/arrays.toml                  |  2 +-
 test/integration/bitfields.toml               | 14 ++++-----
 test/integration/cycles.toml                  |  8 ++---
 test/integration/gen_tests.py                 | 19 +++++++++---
 test/integration/inheritance_polymorphic.toml | 12 ++++----
 .../inheritance_polymorphic_diamond.toml      | 20 ++++++-------
 ...eritance_polymorphic_non_dynamic_base.toml | 12 ++++----
 test/integration/pointers.toml                | 26 +++++++++--------
 test/integration/pointers_function.toml       | 10 +++----
 test/integration/pointers_incomplete.toml     | 29 +++++++++----------
 test/integration/runner_common.h              |  2 ++
 test/integration/std_smart_ptr.toml           |  4 +--
 test/integration/thrift_isset.toml            | 12 ++++----
 test/integration/thrift_isset_missing.toml    |  6 ++--
 test/integration/thrift_namespaces.toml       |  2 +-
 17 files changed, 115 insertions(+), 94 deletions(-)

diff --git a/test/integration/README.md b/test/integration/README.md
index 4d7d734..08759aa 100644
--- a/test/integration/README.md
+++ b/test/integration/README.md
@@ -201,13 +201,23 @@ definitions = '''
 
     Implies `oil_disable`.
 
+  - `features`
+
+    Append this list of features to the configuration. This works for all types
+    of test.
+
+    Example:
+    ```
+    features = ["chase-raw-pointers"]
+    ```
+
   - `cli_options`
 
     Additional command line arguments passed to oid.
 
     Example:
     ```
-    cli_options = ["-fchase-raw-pointers"]
+    cli_options = ["-Ftype-graph"]
     ```
 
   - `skip`, `oid_skip`, `oil_skip`
diff --git a/test/integration/anonymous.toml b/test/integration/anonymous.toml
index 15b9f10..70600d1 100644
--- a/test/integration/anonymous.toml
+++ b/test/integration/anonymous.toml
@@ -84,7 +84,7 @@ definitions = '''
   [cases.regular_struct]
     param_types = ["const Node&"]
     setup = "return Node{1, 2, 3};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":12,
       "dynamicSize":0,
@@ -97,7 +97,7 @@ definitions = '''
       ]}]'''
 
   [cases.anon_struct]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const AnonStructContainer&"]
     setup = '''
       return AnonStructContainer{
@@ -106,7 +106,7 @@ definitions = '''
         }
       };
     '''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 8,
       "dynamicSize": 12,
@@ -143,10 +143,10 @@ definitions = '''
           new Node{1, 2, 3}
         }
       };'''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
 
   [cases.anon_typedef]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const AnonTypedefContainer&"]
     setup = '''
       return AnonTypedefContainer{
@@ -154,7 +154,7 @@ definitions = '''
           .node = new Node{1, 2, 3}
         }
       };'''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 8,
       "dynamicSize": 12,
@@ -196,7 +196,6 @@ definitions = '''
   [cases.anon_union]
     param_types = ["const AnonUnionContainer&"]
     setup = 'return AnonUnionContainer{ .a = 3 };'
-    cli_options = ["-fchase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 24,
       "dynamicSize": 0,
@@ -218,10 +217,10 @@ definitions = '''
     }]'''
 
   [cases.nested_anon_struct]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const NestedAnonContainer&"]
+    features = ["chase-raw-pointers"]
     setup = 'return NestedAnonContainer{.m = { .v = {.as = {new Node{1, 2, 3}}}}};'
-    cli_options = ["-fchase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 80,
       "dynamicSize": 12,
@@ -290,7 +289,7 @@ definitions = '''
   #     result.ns[3].nodes.resize(22);
   #     return result;
   #   '''
-  #   cli_options = ["-fchase-raw-pointers"]
+  #   features = ["chase-raw-pointers"]
   #   expect_json = '''[{
   #     "staticSize": 104,
   #     "dynamicSize": 556,
diff --git a/test/integration/arrays.toml b/test/integration/arrays.toml
index 25e542a..9178559 100644
--- a/test/integration/arrays.toml
+++ b/test/integration/arrays.toml
@@ -76,7 +76,7 @@ definitions = '''
         "elementStaticSize":4
       }]}]'''
   [cases.multidim]
-    cli_options = ["-ftype-graph"]
+    
     param_types = ["const MultiDim&"]
     setup = "return {};"
     expect_json = '''[
diff --git a/test/integration/bitfields.toml b/test/integration/bitfields.toml
index bd1e052..6f66647 100644
--- a/test/integration/bitfields.toml
+++ b/test/integration/bitfields.toml
@@ -53,7 +53,7 @@ definitions = '''
 # if they were regular primitives.
 [cases]
   [cases.single]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["Single&"]
     setup = "return {};"
@@ -62,7 +62,7 @@ definitions = '''
         {"staticSize":4, "dynamicSize":0, "exclusiveSize":4}
     ]}]'''
   [cases.within_bytes]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["WithinBytes&"]
     setup = "return {};"
@@ -73,7 +73,7 @@ definitions = '''
         {"staticSize":1, "dynamicSize":0, "exclusiveSize":1}
     ]}]'''
   [cases.straddle_bytes]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["StraddleBytes&"]
     setup = "return {};"
@@ -84,7 +84,7 @@ definitions = '''
         {"staticSize":1, "dynamicSize":0, "exclusiveSize":1}
     ]}]'''
   [cases.mixed]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["Mixed&"]
     setup = "return {};"
@@ -98,13 +98,13 @@ definitions = '''
     ]}]'''
   [cases.more_bits_than_type] # TODO member sizes are wrong
     skip = "drgn errors out"
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["MoreBitsThanType&"]
     setup = "return {};"
     expect_json = '"TODO"'
   [cases.zero_bits]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["ZeroBits&"]
     setup = "return {};"
@@ -114,7 +114,7 @@ definitions = '''
         {"staticSize":1, "dynamicSize":0, "exclusiveSize":1}
     ]}]'''
   [cases.enum]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["Enum&"]
     setup = "return {};"
diff --git a/test/integration/cycles.toml b/test/integration/cycles.toml
index d6f6daf..77b0d0b 100644
--- a/test/integration/cycles.toml
+++ b/test/integration/cycles.toml
@@ -23,7 +23,7 @@ definitions = '''
 '''
 [cases]
   [cases.raw_ptr]
-    oil_disable = "oil can't chase pointers safely"
+    oil_skip = "cycles are broken" # https://github.com/facebookexperimental/object-introspection/issues/293
     param_types = ["RawNode*"]
     setup = '''
       RawNode *first  = new RawNode{1, nullptr};
@@ -34,7 +34,7 @@ definitions = '''
       third->next = first;
       return first;
     '''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''
     [
       {
@@ -96,7 +96,7 @@ definitions = '''
     '''
 
   [cases.raw_ptr_wrapped]
-    oil_disable = "oil can't chase pointers safely"
+    oil_skip = "cycles are broken" # https://github.com/facebookexperimental/object-introspection/issues/293
     param_types = ["Wrapper<RawNode*>&"]
     setup = '''
       RawNode *first  = new RawNode{1, nullptr};
@@ -107,7 +107,7 @@ definitions = '''
       third->next = first;
       return Wrapper<RawNode*>(first);
     '''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 8,
       "dynamicSize": 48,
diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py
index b72e371..d10ddd7 100644
--- a/test/integration/gen_tests.py
+++ b/test/integration/gen_tests.py
@@ -268,6 +268,19 @@ def add_tests(f, config):
         add_oil_integration_test(f, config, case_name, case)
 
 
+def get_config_strings(case):
+    config_prefix = case.get("config_prefix", "")
+    if "features" in case:
+        feature_config = "features = ["
+        for f in case.get("features", []):
+            feature_config += '"' + f + '"'
+        feature_config += "]\n"
+        config_prefix = feature_config + config_prefix
+    config_suffix = case.get("config_suffix", "")
+
+    return (config_prefix, config_suffix)
+
+
 def add_oid_integration_test(f, config, case_name, case):
     probe_type = case.get("type", "entry")
     args = case.get("args", "arg0")
@@ -283,8 +296,7 @@ def add_oid_integration_test(f, config, case_name, case):
         "{" + ", ".join(f'"{option}"' for option in case.get("cli_options", ())) + "}"
     )
 
-    config_prefix = case.get("config_prefix", "")
-    config_suffix = case.get("config_suffix", "")
+    config_prefix, config_suffix = get_config_strings(case)
 
     f.write(
         f"\n"
@@ -356,8 +368,7 @@ def add_oil_integration_test(f, config, case_name, case):
     if "oil_disable" in case or "target_function" in case:
         return
 
-    config_prefix = case.get("config_prefix", "")
-    config_suffix = case.get("config_suffix", "")
+    config_prefix, config_suffix = get_config_strings(case)
 
     f.write(
         f"\n"
diff --git a/test/integration/inheritance_polymorphic.toml b/test/integration/inheritance_polymorphic.toml
index fe50462..0b9e577 100644
--- a/test/integration/inheritance_polymorphic.toml
+++ b/test/integration/inheritance_polymorphic.toml
@@ -24,7 +24,7 @@ definitions = '''
 [cases]
   [cases.a_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["A"]
     setup = "return {};"
@@ -39,7 +39,7 @@ definitions = '''
 
   [cases.b_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["B"]
     setup = '''
@@ -58,7 +58,7 @@ definitions = '''
       ]}]'''
   [cases.b_as_b]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const B&"]
     arg_types = ["B"]
     setup = '''
@@ -78,7 +78,7 @@ definitions = '''
 
   [cases.c_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["C"]
     setup = '''
@@ -98,7 +98,7 @@ definitions = '''
       ]}]'''
   [cases.c_as_b]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const B&"]
     arg_types = ["C"]
     setup = '''
@@ -118,7 +118,7 @@ definitions = '''
       ]}]'''
   [cases.c_as_c]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const C&"]
     arg_types = ["C"]
     setup = '''
diff --git a/test/integration/inheritance_polymorphic_diamond.toml b/test/integration/inheritance_polymorphic_diamond.toml
index ec9a0e1..cff0638 100644
--- a/test/integration/inheritance_polymorphic_diamond.toml
+++ b/test/integration/inheritance_polymorphic_diamond.toml
@@ -31,7 +31,7 @@ definitions = '''
 [cases]
   [cases.root_as_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Root&"]
     arg_types = ["Root"]
     setup = "return {};"
@@ -46,7 +46,7 @@ definitions = '''
 
   [cases.middle1_as_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Root&"]
     arg_types = ["Middle1"]
     setup = '''
@@ -65,7 +65,7 @@ definitions = '''
       ]}]'''
   [cases.middle1_as_middle1]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Middle1&"]
     arg_types = ["ns_inheritance_polymorphic_diamond::Middle1"]
     setup = '''
@@ -85,7 +85,7 @@ definitions = '''
 
   [cases.middle2_as_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Root&"]
     arg_types = ["Middle2"]
     setup = '''
@@ -104,7 +104,7 @@ definitions = '''
       ]}]'''
   [cases.middle2_as_middle2]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Middle2&"]
     arg_types = ["Middle2"]
     setup = '''
@@ -124,7 +124,7 @@ definitions = '''
 
   [cases.child_as_middle1_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     # We need to explicitly cast from Child to Middle1 before going to root to
     # resolve the diamond problem
     param_types = ["const Root&"]
@@ -150,7 +150,7 @@ definitions = '''
       ]}]'''
   [cases.child_as_middle2_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     # We need to explicitly cast from Child to Middle2 before going to root to
     # resolve the diamond problem
     param_types = ["const Root&"]
@@ -176,7 +176,7 @@ definitions = '''
       ]}]'''
   [cases.child_as_middle1]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Middle1&"]
     arg_types = ["Child"]
     setup = '''
@@ -200,7 +200,7 @@ definitions = '''
       ]}]'''
   [cases.child_as_middle2]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Middle2&"]
     arg_types = ["Child"]
     setup = '''
@@ -224,7 +224,7 @@ definitions = '''
       ]}]'''
   [cases.child_as_child]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Child&"]
     arg_types = ["Child"]
     setup = '''
diff --git a/test/integration/inheritance_polymorphic_non_dynamic_base.toml b/test/integration/inheritance_polymorphic_non_dynamic_base.toml
index fff0199..e482e92 100644
--- a/test/integration/inheritance_polymorphic_non_dynamic_base.toml
+++ b/test/integration/inheritance_polymorphic_non_dynamic_base.toml
@@ -22,7 +22,7 @@ definitions = '''
 [cases]
   [cases.a_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["A"]
     setup = "return {};"
@@ -54,7 +54,7 @@ definitions = '''
 
   [cases.b_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["B"]
     setup = '''
@@ -71,7 +71,7 @@ definitions = '''
       ]}]'''
   [cases.b_as_b]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const B&"]
     arg_types = ["B"]
     setup = '''
@@ -109,7 +109,7 @@ definitions = '''
 
   [cases.c_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["C"]
     setup = '''
@@ -126,7 +126,7 @@ definitions = '''
       ]}]'''
   [cases.c_as_b]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const B&"]
     arg_types = ["C"]
     setup = '''
@@ -146,7 +146,7 @@ definitions = '''
       ]}]'''
   [cases.c_as_c]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const C&"]
     arg_types = ["C"]
     setup = '''
diff --git a/test/integration/pointers.toml b/test/integration/pointers.toml
index 71bf2ad..cfd3627 100644
--- a/test/integration/pointers.toml
+++ b/test/integration/pointers.toml
@@ -17,7 +17,7 @@ definitions = '''
     skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19
     param_types = ["int*"]
     setup = "return new int(1);"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "typeName": "int *",
       "staticSize": 8,
@@ -59,7 +59,7 @@ definitions = '''
     skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19
     param_types = ["void*"]
     setup = "return new int(1);"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "typeName": "void *",
       "staticSize": 8,
@@ -95,7 +95,7 @@ definitions = '''
     skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19
     param_types = ["std::vector<int>*"]
     setup = "return new std::vector<int>{1,2,3};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "typeName": "std::vector<int> *",
       "staticSize": 8,
@@ -134,10 +134,10 @@ definitions = '''
 
 
   [cases.struct_primitive_ptrs]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, new int(0), new int(0)};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":4,
@@ -161,9 +161,10 @@ definitions = '''
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_primitive_ptrs_null]
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, nullptr, nullptr};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
@@ -177,10 +178,10 @@ definitions = '''
 
 
   [cases.struct_vector_ptr]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{new std::vector<int>{1,2,3}};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":36,
@@ -199,9 +200,10 @@ definitions = '''
         {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_vector_ptr_null]
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{nullptr};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
@@ -213,10 +215,10 @@ definitions = '''
 
 
   [cases.vector_of_pointers]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const std::vector<int*>&"]
     setup = "return {{new int(1), nullptr, new int(3)}};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":32,
@@ -258,7 +260,7 @@ definitions = '''
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.feature_config]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const std::vector<int*>&"]
     setup = "return {{new int(1), nullptr, new int(3)}};"
     config_prefix = 'features = ["chase-raw-pointers"]'
diff --git a/test/integration/pointers_function.toml b/test/integration/pointers_function.toml
index a62ef9a..1c7934f 100644
--- a/test/integration/pointers_function.toml
+++ b/test/integration/pointers_function.toml
@@ -29,11 +29,10 @@ definitions = '''
       }]
     }]'''
   [cases.raw_chase] # We should never chase function pointers
-    oid_skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22
-    oil_disable = "oil can't chase raw pointers safely"
+    skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22
     param_types = ["const FuncPtrStruct&"]
     setup = "return {{myFunction}};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 8,
       "dynamicSize": 0,
@@ -73,11 +72,10 @@ definitions = '''
       "NOT": "members"
     }]'''
   [cases.std_function_chase] # We should never chase function pointers
-    oid_skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22
-    oil_disable = "oil can't chase raw pointers safely"
+    skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22
     param_types = ["std::function<void(int)> &"]
     setup = "return myFunction;"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "typeName": "function<void (int)>",
       "staticSize": 32,
diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml
index 2d34865..87af9ad 100644
--- a/test/integration/pointers_incomplete.toml
+++ b/test/integration/pointers_incomplete.toml
@@ -19,38 +19,37 @@ definitions = '''
 '''
 [cases]
   [cases.raw]
-    oil_disable = "oil can't chase raw pointers safely"
     oid_skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oil_skip = "oil codegen fails on top level incomplete objects" # https://github.com/facebookexperimental/object-introspection/issues/459
     param_types = ["IncompleteType*"]
     setup = "return static_cast<IncompleteType*>(::operator new(5));"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
-      "typeName": "IncompleteType *",
-      "staticSize": 8,
+      "typeName": "IncompleteType",
+      "staticSize": 0,
       "dynamicSize": 0,
-      "NOT": {"pointer": 0},
       "NOT": "members"
     }]'''
   [cases.raw_no_follow]
-    skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oid_skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oil_skip = "oil codegen fails on top level incomplete objects" # https://github.com/facebookexperimental/object-introspection/issues/459
     param_types = ["IncompleteType*"]
     setup = "return static_cast<IncompleteType*>(::operator new(5));"
     expect_json = '''[{
-      "typeName": "IncompleteType *",
-      "staticSize": 8,
+      "typeName": "IncompleteType",
+      "staticSize": 0,
       "dynamicSize": 0,
-      "NOT": {"pointer": 0},
       "NOT": "members"
     }]'''
   [cases.raw_null]
-    skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oid_skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oil_skip = "oil codegen fails on top level incomplete objects" # https://github.com/facebookexperimental/object-introspection/issues/459
     param_types = ["IncompleteType*"]
     setup = "return nullptr;"
     expect_json = '''[{
-      "typeName": "IncompleteType *",
-      "staticSize": 8,
+      "typeName": "IncompleteType",
+      "staticSize": 0,
       "dynamicSize": 0,
-      "pointer": 0,
       "NOT": "members"
     }]'''
 
@@ -87,10 +86,10 @@ definitions = '''
     expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.containing_struct]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const IncompleteTypeContainer&"]
     setup = "return IncompleteTypeContainer{};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 88,
       "dynamicSize": 0,
diff --git a/test/integration/runner_common.h b/test/integration/runner_common.h
index d2b5e57..d1f2bae 100644
--- a/test/integration/runner_common.h
+++ b/test/integration/runner_common.h
@@ -51,6 +51,8 @@ class IntegrationBase : public ::testing::Test {
   std::string stdout_;
   std::string stderr_;
 
+  std::vector<std::string> features;
+
   /*
    * compare_json
    *
diff --git a/test/integration/std_smart_ptr.toml b/test/integration/std_smart_ptr.toml
index dfb86ae..c9ab383 100644
--- a/test/integration/std_smart_ptr.toml
+++ b/test/integration/std_smart_ptr.toml
@@ -373,7 +373,7 @@ definitions = '''
     expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_present_chase]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     setup = '''
     static std::shared_ptr<std::uint64_t> shd = std::make_shared<std::uint64_t>(0xDEADBEEF);
     std::weak_ptr<std::uint64_t> weak = shd;
@@ -392,7 +392,7 @@ definitions = '''
     expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_expired_chase]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     setup = '''
     return std::make_shared<std::uint64_t>(0xDEADBEEF);
     '''
diff --git a/test/integration/thrift_isset.toml b/test/integration/thrift_isset.toml
index 064cb13..98e19fd 100644
--- a/test/integration/thrift_isset.toml
+++ b/test/integration/thrift_isset.toml
@@ -85,7 +85,7 @@ namespace cpp2 {
       ret.c_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
@@ -109,7 +109,7 @@ namespace cpp2 {
       ret.j_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":44,
       "dynamicSize":0,
@@ -136,7 +136,7 @@ namespace cpp2 {
       ret.c_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":20,
       "dynamicSize":0,
@@ -156,7 +156,7 @@ namespace cpp2 {
       ret.b_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
@@ -176,7 +176,7 @@ namespace cpp2 {
       ret.f_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":28,
       "dynamicSize":0,
@@ -199,7 +199,7 @@ namespace cpp2 {
       ret.e_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":32,
       "dynamicSize":0,
diff --git a/test/integration/thrift_isset_missing.toml b/test/integration/thrift_isset_missing.toml
index 63b578f..3e7ad94 100644
--- a/test/integration/thrift_isset_missing.toml
+++ b/test/integration/thrift_isset_missing.toml
@@ -68,7 +68,7 @@ raw_definitions = '''
       ret.__isset.at(2) = true;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
@@ -89,7 +89,7 @@ raw_definitions = '''
       ret.__isset.at(2) = true;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
@@ -117,7 +117,7 @@ raw_definitions = '''
 
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":32,
       "dynamicSize":0,
diff --git a/test/integration/thrift_namespaces.toml b/test/integration/thrift_namespaces.toml
index f2ce9f9..2218ba3 100644
--- a/test/integration/thrift_namespaces.toml
+++ b/test/integration/thrift_namespaces.toml
@@ -16,7 +16,7 @@ thrift_definitions = '''
       namespaceA::namespaceB::TTTTT ret;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,

From 2a1d2d4973a0db0da578f2516743689c7ce0d8c6 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Mon, 15 Jan 2024 15:39:09 +0000
Subject: [PATCH 133/188] tbv2: fix pointer codegen

A previous change enabled running OIL tests with specific features enabled.
This highlighted that pointer code generation under TreeBuilder-v2 was very
broken. This change updates pointer code generation to work and enables the
skipped tests. All enabled tests need `expected_json_v2` added to them due to
formatting differences.

Reformatted and rewrote the basic type handler that handles primitives and
pointers. Removed the reliance on `features` to decide whether to generate for
TreeBuilder-v2 as the intermediate features have been removed.

There were a couple of other changes needed to enable these tests on TBv2 that
aren't worth their own issues and PRs, I sneaked them in here.

Extra changes:
- Added `Pointer` and `Reference` to TopoSorter so they generate
  `NameProvider` instances. It might be worth visiting the graph differently
  for `NameProvider` as it requires so many instances that others generators do
  not. Will consider that in the future.
- Follow typedefs when calculating exclusive size for a type.

Closes #458.

Test plan:
- CI
- Enabled previously disabled tests.
---
 oi/CodeGen.cpp                            |  34 ++--
 oi/FuncGen.cpp                            | 216 ++++++++++++----------
 oi/FuncGen.h                              |   2 +-
 oi/type_graph/TopoSorter.cpp              |  23 ++-
 oi/type_graph/TopoSorter.h                |   1 +
 test/integration/anonymous.toml           | 123 +++++++++++-
 test/integration/pointers.toml            |  82 +++++++-
 test/integration/pointers_incomplete.toml |  27 ++-
 test/test_topo_sorter.cpp                 |  18 ++
 9 files changed, 387 insertions(+), 139 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 84fb9b4..95b23a4 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -195,7 +195,12 @@ void genDecls(const TypeGraph& typeGraph, std::string& code) {
 namespace {
 
 size_t calculateExclusiveSize(const Type& t) {
-  if (const auto* c = dynamic_cast<const Class*>(&t)) {
+  const Type* finalType = &t;
+  while (const auto* td = dynamic_cast<const Typedef*>(finalType)) {
+    finalType = &td->underlyingType();
+  }
+
+  if (const auto* c = dynamic_cast<const Class*>(finalType)) {
     return std::accumulate(
         c->members.cbegin(), c->members.cend(), 0, [](size_t a, const auto& m) {
           if (m.name.starts_with(AddPadding::MemberPrefix))
@@ -203,7 +208,7 @@ size_t calculateExclusiveSize(const Type& t) {
           return a;
         });
   }
-  return t.size();
+  return finalType->size();
 }
 
 }  // namespace
@@ -211,7 +216,7 @@ size_t calculateExclusiveSize(const Type& t) {
 void genNames(const TypeGraph& typeGraph, std::string& code) {
   code += R"(
 template <typename T>
-struct NameProvider {};
+struct NameProvider;
 )";
 
   // TODO: stop types being duplicated at this point and remove this check
@@ -239,6 +244,9 @@ struct ExclusiveSizeProvider {
 )";
 
   for (const Type& t : typeGraph.finalTypes) {
+    if (dynamic_cast<const Typedef*>(&t))
+      continue;
+
     size_t exclusiveSize = calculateExclusiveSize(t);
     if (exclusiveSize != t.size()) {
       code += "template <> struct ExclusiveSizeProvider<";
@@ -1063,22 +1071,6 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
     }
 )";
 
-  if (features[Feature::TreeBuilderV2]) {
-    code += R"(
-template <typename Ctx, typename T>
-constexpr inst::Field make_field(std::string_view name) {
-  return inst::Field{
-    sizeof(T),
-    ExclusiveSizeProvider<T>::size,
-    name,
-    NameProvider<T>::names,
-    TypeHandler<Ctx, T>::fields,
-    TypeHandler<Ctx, T>::processors,
-  };
-}
-)";
-  }
-
   // TODO: bit of a hack - making ContainerInfo a node in the type graph and
   // traversing for it would remove the need for this set altogether.
   std::unordered_set<const ContainerInfo*> used{};
@@ -1260,9 +1252,6 @@ void CodeGen::generate(TypeGraph& typeGraph,
     code += "using namespace oi::detail;\n";
     code += "using oi::exporters::ParsedData;\n";
     code += "using namespace oi::exporters;\n";
-    code += "namespace OIInternal {\nnamespace {\n";
-    FuncGen::DefineBasicTypeHandlers(code, config_.features);
-    code += "} // namespace\n} // namespace OIInternal\n";
   }
 
   if (config_.features[Feature::CaptureThriftIsset]) {
@@ -1302,6 +1291,7 @@ void CodeGen::generate(TypeGraph& typeGraph,
   }
 
   if (config_.features[Feature::TreeBuilderV2]) {
+    FuncGen::DefineBasicTypeHandlers(code);
     addStandardTypeHandlers(typeGraph, config_.features, code);
     addTypeHandlers(typeGraph, code);
   } else {
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 0b1e293..56beb87 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -613,116 +613,128 @@ class BackInserter {
  * pointer's value always, then the value of the pointer if it is unique. void
  * is of type Unit and always stores nothing.
  */
-void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
+void FuncGen::DefineBasicTypeHandlers(std::string& code) {
   code += R"(
-    template <typename Ctx, typename T>
-    struct TypeHandler {
-        using DB = typename Ctx::DataBuffer;
-      private:
-        static auto choose_type() {
-            if constexpr(std::is_pointer_v<T>) {
-                return std::type_identity<types::st::Pair<DB,
-                  types::st::VarInt<DB>,
-                  types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>
-                >>();
-            } else {
-                return std::type_identity<types::st::Unit<DB>>();
-            }
-        }
-
-      public:
-        using type = typename decltype(choose_type())::type;
+template <typename Ctx, typename T>
+struct TypeHandler;
 )";
-  if (features[Feature::TreeBuilderV2]) {
-    code += R"(private:
-        static void process_pointer(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
-          el.pointer = std::get<ParsedData::VarInt>(d.val).value;
-        }
-        static void process_pointer_content(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
-          static constexpr std::array<std::string_view, 1> names{"TODO"};
-          static constexpr auto childField = inst::Field{
-            sizeof(T),
-            "*",
-            names,
-            TypeHandler<Ctx, T>::fields,
-            TypeHandler<Ctx, T>::processors,
-          };
 
-          const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
-
-          el.container_stats.emplace(result::Element::ContainerStats{ .capacity = 1 });
-
-          if (sum.index == 0)
-            return;
-
-          el.container_stats->length = 1;
-          stack_ins(childField);
-        }
-
-        static constexpr auto choose_fields() {
-          if constexpr(std::is_pointer_v<T>) {
-            return std::array<exporters::inst::Field, 0>{};
-          } else {
-            return std::array<exporters::inst::Field, 0>{};
-          }
-        }
-        static constexpr auto choose_processors() {
-          if constexpr(std::is_pointer_v<T>) {
-            return std::array<inst::ProcessorInst, 2>{
-              {types::st::VarInt<DB>::describe, &process_pointer},
-              {types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>::describe, &process_pointer_content},
-            };
-          } else {
-            return std::array<inst::ProcessorInst, 0>{};
-          }
-        }
-      public:
-        static constexpr auto fields = choose_fields();
-        static constexpr auto processors = choose_processors();
+  code += R"(
+template <typename Ctx, typename T>
+constexpr inst::Field make_field(std::string_view name) {
+  return inst::Field{
+      sizeof(T),
+      ExclusiveSizeProvider<T>::size,
+      name,
+      NameProvider<T>::names,
+      TypeHandler<Ctx, T>::fields,
+      TypeHandler<Ctx, T>::processors,
+  };
+}
 )";
+  code += R"(
+template <typename Ctx, typename T>
+struct TypeHandler {
+  using DB = typename Ctx::DataBuffer;
+
+ private:
+  static void process_pointer(result::Element& el,
+                              std::function<void(inst::Inst)> stack_ins,
+                              ParsedData d) {
+    el.pointer = std::get<ParsedData::VarInt>(d.val).value;
   }
-  code += R"(
-        static types::st::Unit<DB> getSizeType(
-          Ctx& ctx,
-          const T& t,
-          typename TypeHandler<Ctx, T>::type returnArg) {
-              if constexpr(std::is_pointer_v<T>) {
-                JLOG("ptr val @");
-                JLOGPTR(t);
-                auto r0 = returnArg.write((uintptr_t)t);
-                if (t && ctx.pointers.add((uintptr_t)t)) {
-                  return r0.template delegate<1>([&t](auto ret) {
-                    if constexpr (!std::is_void<std::remove_pointer_t<T>>::value) {
-                      return TypeHandler<Ctx, std::remove_pointer_t<T>>::getSizeType(*t, ret);
-                    } else {
-                      return ret;
-                    }
-                  });
-                } else {
-                  return r0.template delegate<0>(std::identity());
-                }
-              } else {
-                return returnArg;
-              }
-        }
-    };
-  )";
+  static void process_pointer_content(result::Element& el,
+                                      std::function<void(inst::Inst)> stack_ins,
+                                      ParsedData d) {
+    using U = std::decay_t<std::remove_pointer_t<T>>;
+    const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
+
+    if constexpr (oi_is_complete<U>) {
+      static constexpr auto childField = make_field<Ctx, U>("*");
+
+      el.container_stats.emplace(result::Element::ContainerStats{ .capacity = 1, .length = 0 });
+
+      if (sum.index == 0)
+        return;
+
+      el.container_stats->length = 1;
+      stack_ins(childField);
+    }
+  }
+
+  static auto choose_type() {
+    if constexpr (std::is_pointer_v<T>) {
+      return std::type_identity<types::st::Pair<
+          DB,
+          types::st::VarInt<DB>,
+          types::st::Sum<
+              DB,
+              types::st::Unit<DB>,
+              typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>>>();
+    } else {
+      return std::type_identity<types::st::Unit<DB>>();
+    }
+  }
+  static constexpr auto choose_processors() {
+    if constexpr (std::is_pointer_v<T>) {
+      return std::array<inst::ProcessorInst, 2>{
+          exporters::inst::ProcessorInst{types::st::VarInt<DB>::describe,
+                                         &process_pointer},
+          exporters::inst::ProcessorInst{
+              types::st::Sum<
+                  DB,
+                  types::st::Unit<DB>,
+                  typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>::
+                  describe,
+              &process_pointer_content},
+      };
+    } else {
+      return std::array<inst::ProcessorInst, 0>{};
+    }
+  }
+
+ public:
+  using type = typename decltype(choose_type())::type;
+
+  static constexpr std::array<exporters::inst::Field, 0> fields{};
+  static constexpr auto processors = choose_processors();
+
+  static types::st::Unit<DB> getSizeType(
+      Ctx& ctx, const T& t, typename TypeHandler<Ctx, T>::type returnArg) {
+    if constexpr (std::is_pointer_v<T>) {
+      JLOG("ptr val @");
+      JLOGPTR(t);
+      auto r0 = returnArg.write((uintptr_t)t);
+      if (t && ctx.pointers.add((uintptr_t)t)) {
+        return r0.template delegate<1>([&ctx, &t](auto ret) {
+          using U = std::decay_t<std::remove_pointer_t<T>>;
+          if constexpr (oi_is_complete<U>) {
+            return TypeHandler<Ctx, U>::getSizeType(ctx, *t, ret);
+          } else {
+            return ret;
+          }
+        });
+      } else {
+        return r0.template delegate<0>(std::identity());
+      }
+    } else {
+      return returnArg;
+    }
+  }
+};
+)";
 
   code += R"(
-    template <typename Ctx>
-    class TypeHandler<Ctx, void> {
-        using DB = typename Ctx::DataBuffer;
-      public:
-        using type = types::st::Unit<DB>;
+template <typename Ctx>
+class TypeHandler<Ctx, void> {
+  using DB = typename Ctx::DataBuffer;
+
+ public:
+  using type = types::st::Unit<DB>;
+  static constexpr std::array<exporters::inst::Field, 0> fields{};
+  static constexpr std::array<exporters::inst::ProcessorInst, 0> processors{};
+};
 )";
-  if (features[Feature::TreeBuilderV2]) {
-    code +=
-        "static constexpr std::array<exporters::inst::Field, 0> fields{};\n";
-    code +=
-        "static constexpr std::array<exporters::inst::ProcessorInst, 0> "
-        "processors{};\n";
-  }
-  code += "};\n";
 }
 
 ContainerInfo FuncGen::GetOiArrayContainerInfo() {
diff --git a/oi/FuncGen.h b/oi/FuncGen.h
index 33aaf9c..e2b2fba 100644
--- a/oi/FuncGen.h
+++ b/oi/FuncGen.h
@@ -76,7 +76,7 @@ class FuncGen {
 
   static void DefineDataSegmentDataBuffer(std::string& testCode);
   static void DefineBackInserterDataBuffer(std::string& code);
-  static void DefineBasicTypeHandlers(std::string& code, FeatureSet features);
+  static void DefineBasicTypeHandlers(std::string& code);
 
   static ContainerInfo GetOiArrayContainerInfo();
 };
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index ff2715a..ed892c0 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -126,12 +126,27 @@ void TopoSorter::visit(Pointer& p) {
     // Typedefs can not be forward declared, so we must sort them before
     // pointers which reference them
     accept(p.pointeeType());
-    return;
+  } else {
+    // Pointers do not create a dependency, but we do still care about the types
+    // they point to, so delay them until the end.
+    acceptAfter(p.pointeeType());
   }
 
-  // Pointers do not create a dependency, but we do still care about the types
-  // they point to, so delay them until the end.
-  acceptAfter(p.pointeeType());
+  sortedTypes_.push_back(p);
+}
+
+void TopoSorter::visit(Reference& r) {
+  if (dynamic_cast<Typedef*>(&r.pointeeType())) {
+    // Typedefs can not be forward declared, so we must sort them before
+    // pointers which reference them
+    accept(r.pointeeType());
+  } else {
+    // Pointers do not create a dependency, but we do still care about the types
+    // they point to, so delay them until the end.
+    acceptAfter(r.pointeeType());
+  }
+
+  sortedTypes_.push_back(r);
 }
 
 void TopoSorter::visit(CaptureKeys& c) {
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 44f2364..22e9b15 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -46,6 +46,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Enum& e) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& r) override;
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
   void visit(Incomplete& i) override;
diff --git a/test/integration/anonymous.toml b/test/integration/anonymous.toml
index 70600d1..d1e6286 100644
--- a/test/integration/anonymous.toml
+++ b/test/integration/anonymous.toml
@@ -97,7 +97,6 @@ definitions = '''
       ]}]'''
 
   [cases.anon_struct]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const AnonStructContainer&"]
     setup = '''
       return AnonStructContainer{
@@ -133,6 +132,40 @@ definitions = '''
         }]
       }]
     }]'''
+    expect_json_v2 = '''[{
+      "name":"a0",
+      "typeNames":["ns_anonymous::AnonStructContainer"],
+      "staticSize":8,
+      "exclusiveSize":0,
+      "size":20,
+      "members":[{
+        "name":"anon",
+        "typeNames":["__oi_anon_1"],
+        "staticSize":8,
+        "exclusiveSize":0,
+        "size":20,
+        "members":[{
+          "name":"node",
+          "typeNames":["ns_anonymous::Node*"],
+          "staticSize":8,
+          "exclusiveSize":8,
+          "size":20,
+          "length":1,
+          "capacity":1,
+          "members":[{
+            "name":"*",
+            "typeNames":["ns_anonymous::Node"],
+            "staticSize":12,
+            "exclusiveSize":0,
+            "size":12,
+            "members":[
+              { "name": "a", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "b", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "c", "staticSize": 4, "exclusiveSize": 4, "size": 4 }
+            ]
+        }]
+      }]}
+    ]}]'''
 
   [cases.anon_struct_ptr]
     skip = "We don't support pointer to anon-structs yet" # https://github.com/facebookexperimental/object-introspection/issues/20
@@ -146,7 +179,6 @@ definitions = '''
     features = ["chase-raw-pointers"]
 
   [cases.anon_typedef]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const AnonTypedefContainer&"]
     setup = '''
       return AnonTypedefContainer{
@@ -192,6 +224,34 @@ definitions = '''
         }]
       }]
     }]'''
+    expect_json_v2 = '''[{
+      "staticSize": 8,
+      "exclusiveSize": 0,
+      "size": 20,
+      "members": [{
+        "typeNames":["AnonStruct", "__oi_anon_2"],
+        "staticSize": 8,
+        "exclusiveSize": 0,
+        "size": 20,
+        "members": [{
+          "typeNames": ["ns_anonymous::Node*"],
+          "staticSize": 8,
+          "exclusiveSize": 8,
+          "size": 20,
+          "members": [{
+            "typeNames": ["ns_anonymous::Node"],
+            "staticSize": 12,
+            "exclusiveSize": 0,
+            "size": 12,
+            "members": [
+              { "name": "a", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "b", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "c", "staticSize": 4, "exclusiveSize": 4, "size": 4 }
+            ]
+          }]
+        }]
+      }]
+    }]'''
 
   [cases.anon_union]
     param_types = ["const AnonUnionContainer&"]
@@ -217,7 +277,6 @@ definitions = '''
     }]'''
 
   [cases.nested_anon_struct]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const NestedAnonContainer&"]
     features = ["chase-raw-pointers"]
     setup = 'return NestedAnonContainer{.m = { .v = {.as = {new Node{1, 2, 3}}}}};'
@@ -272,6 +331,64 @@ definitions = '''
         "dynamicSize": 0
       }]
     }]'''
+    expect_json_v2 = '''[{
+      "staticSize":80,
+      "exclusiveSize":0,
+      "size":92,
+      "members":[
+        {
+          "name":"m",
+          "typeNames": ["__oi_anon_1"],
+          "staticSize": 48,
+          "exclusiveSize":0,
+          "size": 60,
+          "members":[
+            {"name":"__oi_anon_0", "typeNames":["__oi_anon_2"], "staticSize":16, "exclusiveSize":16, "size":16},
+            {
+              "name":"v",
+              "typeNames":["__oi_anon_3"],
+              "staticSize":32,
+              "exclusiveSize":4,
+              "size":44,
+              "members":[
+                {"name":"a", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+                {"name":"b", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+                {"name":"c", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+                {"name":"__oi_anon_4", "typeNames":["__oi_anon_4"], "staticSize":8, "exclusiveSize":8, "size":8},
+                {
+                  "name":"as",
+                  "typeNames":["AnonStruct", "__oi_anon_6"],
+                  "staticSize":8,
+                  "exclusiveSize":0,
+                  "size":20,
+                  "members":[{
+                    "name":"node",
+                    "typeNames":["ns_anonymous::Node*"],
+                    "staticSize":8,
+                    "exclusiveSize":8,
+                    "size":20,
+                    "members":[{
+                      "name":"*",
+                      "typeNames":["ns_anonymous::Node"],
+                      "staticSize":12,
+                      "exclusiveSize":0,
+                      "size":12,
+                      "members":[
+                        { "name": "a", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+                        { "name": "b", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+                        { "name": "c", "staticSize": 4, "exclusiveSize": 4, "size": 4 }
+                      ]
+                    }]
+                  }]
+                }
+              ]
+            }
+          ]
+        },
+        {"name":"__oi_anon_1", "typeNames": ["__oi_anon_8"], "staticSize":24, "exclusiveSize":24, "size":24},
+        {"name":"__oi_anon_2", "typeNames": ["__oi_anon_9"], "staticSize":8, "exclusiveSize":8, "size":8}
+      ]
+    }]'''
 
   # This test is disabled due to GCC not supporting it
   # [cases.anon_array]
diff --git a/test/integration/pointers.toml b/test/integration/pointers.toml
index cfd3627..6868679 100644
--- a/test/integration/pointers.toml
+++ b/test/integration/pointers.toml
@@ -134,7 +134,6 @@ definitions = '''
 
 
   [cases.struct_primitive_ptrs]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, new int(0), new int(0)};"
     features = ["chase-raw-pointers"]
@@ -147,6 +146,16 @@ definitions = '''
         {"name":"b", "staticSize":8, "exclusiveSize":8, "dynamicSize":4},
         {"name":"c", "staticSize":8, "exclusiveSize":8, "dynamicSize":0}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":4,
+      "size":28,
+      "members":[
+        {"name":"a", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"b", "typeNames":["int32_t*"], "staticSize":8, "exclusiveSize":8, "size":12},
+        {"name":"c", "typeNames":["void*"], "staticSize":8, "exclusiveSize":8, "size":8}
+      ]
+    }]'''
   [cases.struct_primitive_ptrs_no_follow]
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, new int(0), new int(0)};"
@@ -161,7 +170,6 @@ definitions = '''
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_primitive_ptrs_null]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, nullptr, nullptr};"
     features = ["chase-raw-pointers"]
@@ -175,10 +183,19 @@ definitions = '''
         {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":4,
+      "size":24,
+      "members":[
+        {"name":"a", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"b", "typeNames":["int32_t*"], "staticSize":8, "exclusiveSize":8, "size":8},
+        {"name":"c", "typeNames":["void*"], "staticSize":8, "exclusiveSize":8, "size":8}
+      ]
+    }]'''
 
 
   [cases.struct_vector_ptr]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{new std::vector<int>{1,2,3}};"
     features = ["chase-raw-pointers"]
@@ -188,6 +205,22 @@ definitions = '''
       "members":[
         {"name":"vec", "staticSize":8, "dynamicSize":36}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":8,
+      "exclusiveSize":0,
+      "size":44,
+      "members": [
+        {
+          "typeNames":["std::vector<int32_t, std::allocator<int32_t>>*"],
+          "staticSize":8,
+          "exclusiveSize":8,
+          "size":44,
+          "length":1,
+          "capacity":1,
+          "members":[{ "staticSize":24, "exclusiveSize":24, "size":36 }]
+        }
+      ]
+    }]'''
   [cases.struct_vector_ptr_no_follow]
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{new std::vector<int>{1,2,3}};"
@@ -200,7 +233,6 @@ definitions = '''
         {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_vector_ptr_null]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{nullptr};"
     features = ["chase-raw-pointers"]
@@ -212,10 +244,25 @@ definitions = '''
       "members":[
         {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
+      "members": [
+        {
+          "typeNames":["std::vector<int32_t, std::allocator<int32_t>>*"],
+          "staticSize":8,
+          "exclusiveSize":8,
+          "size":8,
+          "length":0,
+          "capacity":1,
+          "members":[]
+        }
+      ]
+    }]'''
 
 
   [cases.vector_of_pointers]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const std::vector<int*>&"]
     setup = "return {{new int(1), nullptr, new int(3)}};"
     features = ["chase-raw-pointers"]
@@ -230,6 +277,18 @@ definitions = '''
         {"staticSize":8, "dynamicSize":0, "pointer":0},
         {"staticSize":8, "dynamicSize":4, "NOT": {"pointer":0}}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":24,
+      "size":56,
+      "length":3,
+      "capacity":3,
+      "members":[
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}},
+        {"staticSize":8, "exclusiveSize":8, "size":8, "pointer":0},
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}}
+      ]
+    }]'''
   [cases.vector_of_pointers_no_follow]
     skip = "pointer field is missing from results" # https://github.com/facebookexperimental/object-introspection/issues/21
     param_types = ["const std::vector<int*>&"]
@@ -260,7 +319,6 @@ definitions = '''
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.feature_config]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const std::vector<int*>&"]
     setup = "return {{new int(1), nullptr, new int(3)}};"
     config_prefix = 'features = ["chase-raw-pointers"]'
@@ -275,3 +333,15 @@ definitions = '''
         {"staticSize":8, "dynamicSize":0, "pointer":0},
         {"staticSize":8, "dynamicSize":4, "NOT": {"pointer":0}}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":24,
+      "size":56,
+      "length":3,
+      "capacity":3,
+      "members":[
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}},
+        {"staticSize":8, "exclusiveSize":8, "size":8, "pointer":0},
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}}
+      ]
+    }]'''
diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml
index 87af9ad..b062043 100644
--- a/test/integration/pointers_incomplete.toml
+++ b/test/integration/pointers_incomplete.toml
@@ -86,7 +86,6 @@ definitions = '''
     expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.containing_struct]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const IncompleteTypeContainer&"]
     setup = "return IncompleteTypeContainer{};"
     features = ["chase-raw-pointers"]
@@ -115,6 +114,32 @@ definitions = '''
         }
       ]
     }]'''
+    expect_json_v2 = '''[{
+      "staticSize": 88,
+      "exclusiveSize": 21,
+      "size": 88,
+      "members": [
+        { "name": "ptrundef", "staticSize": 8, "exclusiveSize": 8, "size": 8 },
+        { "name": "__makePad1", "staticSize": 1, "exclusiveSize": 1, "size": 1 },
+        { "name": "shundef", "staticSize": 16, "exclusiveSize": 16, "size": 16 },
+        { "name": "__makePad2", "staticSize": 1, "exclusiveSize": 1, "size": 1 },
+        { "name": "shoptundef",
+          "staticSize": 24,
+          "exclusiveSize": 24,
+          "size": 24,
+          "length": 0,
+          "capacity": 1
+        },
+        { "name": "__makePad3", "staticSize": 1, "exclusiveSize": 1, "size": 1 },
+        { "name": "optundef",
+          "staticSize": 16,
+          "exclusiveSize": 16,
+          "size": 16,
+          "length": 0,
+          "capacity": 1
+        }
+      ]
+    }]'''
 
   [cases.containing_struct_no_follow]
     param_types = ["const IncompleteTypeContainer&"]
diff --git a/test/test_topo_sorter.cpp b/test/test_topo_sorter.cpp
index 969c6aa..7c18806 100644
--- a/test/test_topo_sorter.cpp
+++ b/test/test_topo_sorter.cpp
@@ -233,6 +233,22 @@ TEST(TopoSorterTest, Pointers) {
   myclass.members.push_back(Member{mypointer, "ptr", 0});
 
   test({myclass}, R"(
+ClassA*
+MyClass
+ClassA
+)");
+}
+
+TEST(TopoSorterTest, References) {
+  // References do not require pointee types to be defined first
+  auto classA = Class{0, Class::Kind::Class, "ClassA", 69};
+  auto myreference = Reference{1, classA};
+
+  auto myclass = Class{2, Class::Kind::Class, "MyClass", 69};
+  myclass.members.push_back(Member{myreference, "ref", 0});
+
+  test({myclass}, R"(
+ClassA*
 MyClass
 ClassA
 )");
@@ -255,6 +271,7 @@ TEST(TopoSorterTest, PointerCycle) {
   // the same sorted order for ClassA and ClassB.
   for (const auto& input : inputs) {
     test(input, R"(
+ClassA*
 ClassB
 ClassA
 )");
@@ -273,6 +290,7 @@ TEST(TopoSorterTest, PointerToTypedef) {
   test({myclass}, R"(
 ClassA
 aliasA
+aliasA*
 MyClass
 )");
 }

From e276f0a185cb7dc5561e1a52b01d47cf6526860d Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Mon, 15 Jan 2024 16:23:10 +0000
Subject: [PATCH 134/188] tbv2: add is_primitive to output

C++ has a concept of Primitive which holds in the type graph. However we don't
currently expose this information to the end user. Expose this from the OIL
iterator to allow future features like primitive rollups.

This affects containers like maps which have a fake `[]` element with no type.
They use this to group together the key/value in a map and to account for any
per element storage overhead. Currently the decision is to make the fake `[]`
element a primitive if all of its children are primitives. This allows for more
effective primitive rollups if that is implemented. This implementation detail
may be changed in future.

Test Plan:
- CI
- Updated simple tests.
---
 include/oi/exporters/Json.h            |  5 +++--
 include/oi/exporters/inst.h            | 16 ++++++++-----
 include/oi/result/Element.h            |  1 +
 oi/CodeGen.cpp                         |  6 ++++-
 oi/FuncGen.cpp                         | 11 +++------
 oi/IntrospectionResult.cpp             |  1 +
 test/integration/simple.toml           | 31 ++++++++++++++++++++++----
 types/f14_fast_map.toml                |  1 +
 types/f14_node_map.toml                |  1 +
 types/f14_value_map.toml               |  1 +
 types/f14_vector_map.toml              |  1 +
 types/map_seq_type.toml                |  1 +
 types/multi_map_type.toml              |  1 +
 types/std_map_type.toml                |  1 +
 types/std_unordered_map_type.toml      |  1 +
 types/std_unordered_multimap_type.toml |  1 +
 16 files changed, 60 insertions(+), 20 deletions(-)

diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h
index dd88ade..3dd37f4 100644
--- a/include/oi/exporters/Json.h
+++ b/include/oi/exporters/Json.h
@@ -98,8 +98,8 @@ inline void Json::printStringField(std::string_view name,
 inline void Json::printBoolField(std::string_view name,
                                  bool value,
                                  std::string_view indent) {
-  out_ << tab() << '"' << name << "\":" << space() << value << ',' << endl()
-       << indent;
+  out_ << tab() << '"' << name << "\":" << space() << (value ? "true" : "false")
+       << ',' << endl() << indent;
 }
 inline void Json::printUnsignedField(std::string_view name,
                                      uint64_t value,
@@ -159,6 +159,7 @@ inline void Json::printFields(const result::Element& el,
   }
   if (el.is_set_stats.has_value())
     printUnsignedField("is_set", el.is_set_stats->is_set, indent);
+  printBoolField("is_primitive", el.is_primitive, indent);
 }
 
 template <typename It>
diff --git a/include/oi/exporters/inst.h b/include/oi/exporters/inst.h
index 8397cd6..5484376 100644
--- a/include/oi/exporters/inst.h
+++ b/include/oi/exporters/inst.h
@@ -58,19 +58,22 @@ struct Field {
                   std::string_view name_,
                   const std::array<std::string_view, N0>& type_names_,
                   const std::array<Field, N1>& fields_,
-                  const std::array<ProcessorInst, N2>& processors_);
+                  const std::array<ProcessorInst, N2>& processors_,
+                  bool is_primitive_);
   template <size_t N0, size_t N1, size_t N2>
   constexpr Field(size_t static_size_,
                   std::string_view name_,
                   const std::array<std::string_view, N0>& type_names_,
                   const std::array<Field, N1>& fields_,
-                  const std::array<ProcessorInst, N2>& processors_)
+                  const std::array<ProcessorInst, N2>& processors_,
+                  bool is_primitive_)
       : Field(static_size_,
               static_size_,
               name_,
               type_names_,
               fields_,
-              processors_) {
+              processors_,
+              is_primitive_) {
   }
   constexpr Field(const Field&) = default;  // no idea why this is needed
 
@@ -80,6 +83,7 @@ struct Field {
   std::span<const std::string_view> type_names;
   std::span<const Field> fields;
   std::span<const ProcessorInst> processors;
+  bool is_primitive;
 };
 
 template <size_t N0, size_t N1, size_t N2>
@@ -88,13 +92,15 @@ constexpr Field::Field(size_t static_size_,
                        std::string_view name_,
                        const std::array<std::string_view, N0>& type_names_,
                        const std::array<Field, N1>& fields_,
-                       const std::array<ProcessorInst, N2>& processors_)
+                       const std::array<ProcessorInst, N2>& processors_,
+                       bool is_primitive_)
     : static_size(static_size_),
       exclusive_size(exclusive_size_),
       name(name_),
       type_names(type_names_),
       fields(fields_),
-      processors(processors_) {
+      processors(processors_),
+      is_primitive(is_primitive_) {
 }
 
 }  // namespace oi::exporters::inst
diff --git a/include/oi/result/Element.h b/include/oi/result/Element.h
index 1eea86b..c971236 100644
--- a/include/oi/result/Element.h
+++ b/include/oi/result/Element.h
@@ -52,6 +52,7 @@ struct Element {
       std::nullopt};
   std::optional<ContainerStats> container_stats;
   std::optional<IsSetStats> is_set_stats;
+  bool is_primitive;
 };
 
 }  // namespace oi::result
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 84fb9b4..cb5bf54 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -798,12 +798,15 @@ void CodeGen::genClassTreeBuilderInstructions(const Class& c,
     if (m.name.starts_with(AddPadding::MemberPrefix))
       continue;
     std::string fullName = c.name() + "::" + m.name;
+    bool isPrimitive = dynamic_cast<const Primitive*>(&m.type());
     code += "      inst::Field{sizeof(" + fullName + "), " +
             std::to_string(calculateExclusiveSize(m.type())) + ",\"" +
             m.inputName + "\", member_" + std::to_string(index) +
             "_type_names, TypeHandler<Ctx, decltype(" + fullName +
             ")>::fields, TypeHandler<Ctx, decltype(" + fullName +
-            ")>::processors},\n";
+            ")>::processors, ";
+    code += isPrimitive ? "true" : "false";
+    code += "},\n";
   }
   code += "  };\n";
   code +=
@@ -1074,6 +1077,7 @@ constexpr inst::Field make_field(std::string_view name) {
     NameProvider<T>::names,
     TypeHandler<Ctx, T>::fields,
     TypeHandler<Ctx, T>::processors,
+    std::is_fundamental_v<T>
   };
 }
 )";
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 97d2707..5ac3abf 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -399,7 +399,8 @@ const std::array<std::string_view, )";
       ", OIInternal::TypeHandler<FakeContext, "
       "OIInternal::__ROOT_TYPE__>::fields, "
       "OIInternal::TypeHandler<FakeContext, "
-      "OIInternal::__ROOT_TYPE__>::processors};\n";
+      "OIInternal::__ROOT_TYPE__>::processors, "
+      "std::is_fundamental_v<OIInternal::__ROOT_TYPE__>};\n";
   code += "} // namespace\n";
   code +=
       "extern const exporters::inst::Inst __attribute__((used, retain)) "
@@ -640,13 +641,7 @@ void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
         }
         static void process_pointer_content(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
           static constexpr std::array<std::string_view, 1> names{"TODO"};
-          static constexpr auto childField = inst::Field{
-            sizeof(T),
-            "*",
-            names,
-            TypeHandler<Ctx, T>::fields,
-            TypeHandler<Ctx, T>::processors,
-          };
+          static constexpr auto childField = make_field<Ctx, T>("*");
 
           const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
 
diff --git a/oi/IntrospectionResult.cpp b/oi/IntrospectionResult.cpp
index bc59f5e..5310001 100644
--- a/oi/IntrospectionResult.cpp
+++ b/oi/IntrospectionResult.cpp
@@ -78,6 +78,7 @@ IntrospectionResult::const_iterator::operator++() {
                 .exclusive_size = ty.exclusive_size,
                 .container_stats = std::nullopt,
                 .is_set_stats = std::nullopt,
+                .is_primitive = ty.is_primitive,
             };
 
             for (const auto& [dy, handler] : ty.processors) {
diff --git a/test/integration/simple.toml b/test/integration/simple.toml
index 0945f70..4557319 100644
--- a/test/integration/simple.toml
+++ b/test/integration/simple.toml
@@ -24,12 +24,21 @@ definitions = '''
       "staticSize":16,
       "dynamicSize":0,
       "exclusiveSize": 3,
-      "size": 16,
       "members":[
         {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
         {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize": 3,
+      "size": 16,
+      "is_primitive": false,
+      "members":[
+        {"name":"a", "staticSize":4, "exclusiveSize": 4, "size": 4, "is_primitive": true},
+        {"name":"b", "staticSize":1, "exclusiveSize": 1, "size": 1, "is_primitive": true},
+        {"name":"c", "staticSize":8, "exclusiveSize": 8, "size": 8, "is_primitive": true}
+      ]}]'''
   [cases.class]
     param_types = ["const SimpleClass&"]
     setup = "return {};"
@@ -37,18 +46,32 @@ definitions = '''
       "staticSize":16,
       "dynamicSize":0,
       "exclusiveSize": 3,
-      "size": 16,
       "members":[
         {"name":"a", "staticSize":4, "dynamicSize":0, "exclusiveSize": 4, "size": 4},
         {"name":"b", "staticSize":1, "dynamicSize":0, "exclusiveSize": 1, "size": 1},
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize": 8, "size": 8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize": 3,
+      "size": 16,
+      "is_primitive": false,
+      "members":[
+        {"name":"a", "staticSize":4, "exclusiveSize": 4, "size": 4, "is_primitive": true},
+        {"name":"b", "staticSize":1, "exclusiveSize": 1, "size": 1, "is_primitive": true},
+        {"name":"c", "staticSize":8, "exclusiveSize": 8, "size": 8, "is_primitive": true}
+      ]}]'''
   [cases.union]
     param_types = ["const SimpleUnion&"]
     setup = "return {};"
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
-      "exclusiveSize":8,
-      "size":8
+      "exclusiveSize":8
+    }]'''
+    expect_json_v2 = '''[{
+      "staticSize":8,
+      "exclusiveSize":8,
+      "size":8,
+      "is_primitive":false
     }]'''
diff --git a/types/f14_fast_map.toml b/types/f14_fast_map.toml
index 06c6f72..69a5173 100644
--- a/types/f14_fast_map.toml
+++ b/types/f14_fast_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/f14_node_map.toml b/types/f14_node_map.toml
index de1dcda..cc89f06 100644
--- a/types/f14_node_map.toml
+++ b/types/f14_node_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/f14_value_map.toml b/types/f14_value_map.toml
index c09bbf0..aa6aa59 100644
--- a/types/f14_value_map.toml
+++ b/types/f14_value_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/f14_vector_map.toml b/types/f14_vector_map.toml
index a673bbd..f640dcc 100644
--- a/types/f14_vector_map.toml
+++ b/types/f14_vector_map.toml
@@ -95,6 +95,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml
index a09efff..f4281eb 100644
--- a/types/map_seq_type.toml
+++ b/types/map_seq_type.toml
@@ -91,6 +91,7 @@ static constexpr auto entry = inst::Field {
   std::array<std::string_view, 0>{},
   entryFields,
   processors,
+  entryFields[0].is_primitive && entryFields[1].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/multi_map_type.toml b/types/multi_map_type.toml
index a5d628f..e313417 100644
--- a/types/multi_map_type.toml
+++ b/types/multi_map_type.toml
@@ -118,6 +118,7 @@ static constexpr auto element = inst::Field {
   std::array<std::string_view, 0>{},
   elementFields,
   std::array<inst::ProcessorInst, 0>{},
+  elementFields[0].is_primitive && elementFields[1].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/std_map_type.toml b/types/std_map_type.toml
index ee652ec..b972b70 100644
--- a/types/std_map_type.toml
+++ b/types/std_map_type.toml
@@ -135,6 +135,7 @@ static constexpr inst::Field element{
   std::array<std::string_view, 0>{},
   element_fields,
   processors,
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 auto list = std::get<ParsedData::List>(d.val);
diff --git a/types/std_unordered_map_type.toml b/types/std_unordered_map_type.toml
index 5b2402f..108731e 100644
--- a/types/std_unordered_map_type.toml
+++ b/types/std_unordered_map_type.toml
@@ -135,6 +135,7 @@ static constexpr auto element = inst::Field{
   std::array<std::string_view, 0>{},
   element_fields,
   processors,
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)
diff --git a/types/std_unordered_multimap_type.toml b/types/std_unordered_multimap_type.toml
index a6f7671..8452154 100644
--- a/types/std_unordered_multimap_type.toml
+++ b/types/std_unordered_multimap_type.toml
@@ -121,6 +121,7 @@ static constexpr auto element = inst::Field{
   std::array<std::string_view, 0>{},
   element_fields,
   std::array<inst::ProcessorInst, 0>{},
+  element_fields[0].is_primitive && element_fields[1].is_primitive,
 };
 
 for (size_t i = 0; i < list.length; i++)

From b50b3a3c9a4474a886da4b52ed09bc557e6d81e8 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 16 Jan 2024 11:15:05 +0000
Subject: [PATCH 135/188] test: add features field to integration tests

Previously we tested different feature flags by using the `cli_options` field
in the test `.toml`. This works for OID but doesn't work for JIT OIL and won't
work for AoT OIL when those tests get added.

This change adds a new higher level `features` field to the test `.toml` which
adds the features to the config file as a prefix. This works with all methods
of generation.

Change the existing `cli_options` features to `features` except for where
they're testing something specific. Enable tests that were previously disabled
for OIL but only didn't work because of not being able to enable features.
Change pointer tests that are currently broken for OIL from `oil_disable` to
`oil_skip` - they can work, but codegen is broken for them at the minute.

Test plan:
- CI
- `make test` is no worse
---
 test/integration/README.md                    | 12 +++++++-
 test/integration/anonymous.toml               | 19 ++++++------
 test/integration/arrays.toml                  |  2 +-
 test/integration/bitfields.toml               | 14 ++++-----
 test/integration/cycles.toml                  |  8 ++---
 test/integration/gen_tests.py                 | 19 +++++++++---
 test/integration/inheritance_polymorphic.toml | 12 ++++----
 .../inheritance_polymorphic_diamond.toml      | 20 ++++++-------
 ...eritance_polymorphic_non_dynamic_base.toml | 12 ++++----
 test/integration/pointers.toml                | 26 +++++++++--------
 test/integration/pointers_function.toml       | 10 +++----
 test/integration/pointers_incomplete.toml     | 29 +++++++++----------
 test/integration/runner_common.h              |  2 ++
 test/integration/std_smart_ptr.toml           |  4 +--
 test/integration/thrift_isset.toml            | 12 ++++----
 test/integration/thrift_isset_missing.toml    |  6 ++--
 test/integration/thrift_namespaces.toml       |  2 +-
 17 files changed, 115 insertions(+), 94 deletions(-)

diff --git a/test/integration/README.md b/test/integration/README.md
index 4d7d734..08759aa 100644
--- a/test/integration/README.md
+++ b/test/integration/README.md
@@ -201,13 +201,23 @@ definitions = '''
 
     Implies `oil_disable`.
 
+  - `features`
+
+    Append this list of features to the configuration. This works for all types
+    of test.
+
+    Example:
+    ```
+    features = ["chase-raw-pointers"]
+    ```
+
   - `cli_options`
 
     Additional command line arguments passed to oid.
 
     Example:
     ```
-    cli_options = ["-fchase-raw-pointers"]
+    cli_options = ["-Ftype-graph"]
     ```
 
   - `skip`, `oid_skip`, `oil_skip`
diff --git a/test/integration/anonymous.toml b/test/integration/anonymous.toml
index 15b9f10..70600d1 100644
--- a/test/integration/anonymous.toml
+++ b/test/integration/anonymous.toml
@@ -84,7 +84,7 @@ definitions = '''
   [cases.regular_struct]
     param_types = ["const Node&"]
     setup = "return Node{1, 2, 3};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":12,
       "dynamicSize":0,
@@ -97,7 +97,7 @@ definitions = '''
       ]}]'''
 
   [cases.anon_struct]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const AnonStructContainer&"]
     setup = '''
       return AnonStructContainer{
@@ -106,7 +106,7 @@ definitions = '''
         }
       };
     '''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 8,
       "dynamicSize": 12,
@@ -143,10 +143,10 @@ definitions = '''
           new Node{1, 2, 3}
         }
       };'''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
 
   [cases.anon_typedef]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const AnonTypedefContainer&"]
     setup = '''
       return AnonTypedefContainer{
@@ -154,7 +154,7 @@ definitions = '''
           .node = new Node{1, 2, 3}
         }
       };'''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 8,
       "dynamicSize": 12,
@@ -196,7 +196,6 @@ definitions = '''
   [cases.anon_union]
     param_types = ["const AnonUnionContainer&"]
     setup = 'return AnonUnionContainer{ .a = 3 };'
-    cli_options = ["-fchase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 24,
       "dynamicSize": 0,
@@ -218,10 +217,10 @@ definitions = '''
     }]'''
 
   [cases.nested_anon_struct]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const NestedAnonContainer&"]
+    features = ["chase-raw-pointers"]
     setup = 'return NestedAnonContainer{.m = { .v = {.as = {new Node{1, 2, 3}}}}};'
-    cli_options = ["-fchase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 80,
       "dynamicSize": 12,
@@ -290,7 +289,7 @@ definitions = '''
   #     result.ns[3].nodes.resize(22);
   #     return result;
   #   '''
-  #   cli_options = ["-fchase-raw-pointers"]
+  #   features = ["chase-raw-pointers"]
   #   expect_json = '''[{
   #     "staticSize": 104,
   #     "dynamicSize": 556,
diff --git a/test/integration/arrays.toml b/test/integration/arrays.toml
index a4fd3b6..82fe0ed 100644
--- a/test/integration/arrays.toml
+++ b/test/integration/arrays.toml
@@ -76,7 +76,7 @@ definitions = '''
         "elementStaticSize":4
       }]}]'''
   [cases.multidim]
-    cli_options = ["-ftype-graph"]
+    
     param_types = ["const MultiDim&"]
     setup = "return {};"
     expect_json = '''[
diff --git a/test/integration/bitfields.toml b/test/integration/bitfields.toml
index bd1e052..6f66647 100644
--- a/test/integration/bitfields.toml
+++ b/test/integration/bitfields.toml
@@ -53,7 +53,7 @@ definitions = '''
 # if they were regular primitives.
 [cases]
   [cases.single]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["Single&"]
     setup = "return {};"
@@ -62,7 +62,7 @@ definitions = '''
         {"staticSize":4, "dynamicSize":0, "exclusiveSize":4}
     ]}]'''
   [cases.within_bytes]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["WithinBytes&"]
     setup = "return {};"
@@ -73,7 +73,7 @@ definitions = '''
         {"staticSize":1, "dynamicSize":0, "exclusiveSize":1}
     ]}]'''
   [cases.straddle_bytes]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["StraddleBytes&"]
     setup = "return {};"
@@ -84,7 +84,7 @@ definitions = '''
         {"staticSize":1, "dynamicSize":0, "exclusiveSize":1}
     ]}]'''
   [cases.mixed]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["Mixed&"]
     setup = "return {};"
@@ -98,13 +98,13 @@ definitions = '''
     ]}]'''
   [cases.more_bits_than_type] # TODO member sizes are wrong
     skip = "drgn errors out"
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["MoreBitsThanType&"]
     setup = "return {};"
     expect_json = '"TODO"'
   [cases.zero_bits]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["ZeroBits&"]
     setup = "return {};"
@@ -114,7 +114,7 @@ definitions = '''
         {"staticSize":1, "dynamicSize":0, "exclusiveSize":1}
     ]}]'''
   [cases.enum]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["Enum&"]
     setup = "return {};"
diff --git a/test/integration/cycles.toml b/test/integration/cycles.toml
index d6f6daf..77b0d0b 100644
--- a/test/integration/cycles.toml
+++ b/test/integration/cycles.toml
@@ -23,7 +23,7 @@ definitions = '''
 '''
 [cases]
   [cases.raw_ptr]
-    oil_disable = "oil can't chase pointers safely"
+    oil_skip = "cycles are broken" # https://github.com/facebookexperimental/object-introspection/issues/293
     param_types = ["RawNode*"]
     setup = '''
       RawNode *first  = new RawNode{1, nullptr};
@@ -34,7 +34,7 @@ definitions = '''
       third->next = first;
       return first;
     '''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''
     [
       {
@@ -96,7 +96,7 @@ definitions = '''
     '''
 
   [cases.raw_ptr_wrapped]
-    oil_disable = "oil can't chase pointers safely"
+    oil_skip = "cycles are broken" # https://github.com/facebookexperimental/object-introspection/issues/293
     param_types = ["Wrapper<RawNode*>&"]
     setup = '''
       RawNode *first  = new RawNode{1, nullptr};
@@ -107,7 +107,7 @@ definitions = '''
       third->next = first;
       return Wrapper<RawNode*>(first);
     '''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 8,
       "dynamicSize": 48,
diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py
index b72e371..d10ddd7 100644
--- a/test/integration/gen_tests.py
+++ b/test/integration/gen_tests.py
@@ -268,6 +268,19 @@ def add_tests(f, config):
         add_oil_integration_test(f, config, case_name, case)
 
 
+def get_config_strings(case):
+    config_prefix = case.get("config_prefix", "")
+    if "features" in case:
+        feature_config = "features = ["
+        for f in case.get("features", []):
+            feature_config += '"' + f + '"'
+        feature_config += "]\n"
+        config_prefix = feature_config + config_prefix
+    config_suffix = case.get("config_suffix", "")
+
+    return (config_prefix, config_suffix)
+
+
 def add_oid_integration_test(f, config, case_name, case):
     probe_type = case.get("type", "entry")
     args = case.get("args", "arg0")
@@ -283,8 +296,7 @@ def add_oid_integration_test(f, config, case_name, case):
         "{" + ", ".join(f'"{option}"' for option in case.get("cli_options", ())) + "}"
     )
 
-    config_prefix = case.get("config_prefix", "")
-    config_suffix = case.get("config_suffix", "")
+    config_prefix, config_suffix = get_config_strings(case)
 
     f.write(
         f"\n"
@@ -356,8 +368,7 @@ def add_oil_integration_test(f, config, case_name, case):
     if "oil_disable" in case or "target_function" in case:
         return
 
-    config_prefix = case.get("config_prefix", "")
-    config_suffix = case.get("config_suffix", "")
+    config_prefix, config_suffix = get_config_strings(case)
 
     f.write(
         f"\n"
diff --git a/test/integration/inheritance_polymorphic.toml b/test/integration/inheritance_polymorphic.toml
index fe50462..0b9e577 100644
--- a/test/integration/inheritance_polymorphic.toml
+++ b/test/integration/inheritance_polymorphic.toml
@@ -24,7 +24,7 @@ definitions = '''
 [cases]
   [cases.a_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["A"]
     setup = "return {};"
@@ -39,7 +39,7 @@ definitions = '''
 
   [cases.b_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["B"]
     setup = '''
@@ -58,7 +58,7 @@ definitions = '''
       ]}]'''
   [cases.b_as_b]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const B&"]
     arg_types = ["B"]
     setup = '''
@@ -78,7 +78,7 @@ definitions = '''
 
   [cases.c_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["C"]
     setup = '''
@@ -98,7 +98,7 @@ definitions = '''
       ]}]'''
   [cases.c_as_b]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const B&"]
     arg_types = ["C"]
     setup = '''
@@ -118,7 +118,7 @@ definitions = '''
       ]}]'''
   [cases.c_as_c]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const C&"]
     arg_types = ["C"]
     setup = '''
diff --git a/test/integration/inheritance_polymorphic_diamond.toml b/test/integration/inheritance_polymorphic_diamond.toml
index ec9a0e1..cff0638 100644
--- a/test/integration/inheritance_polymorphic_diamond.toml
+++ b/test/integration/inheritance_polymorphic_diamond.toml
@@ -31,7 +31,7 @@ definitions = '''
 [cases]
   [cases.root_as_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Root&"]
     arg_types = ["Root"]
     setup = "return {};"
@@ -46,7 +46,7 @@ definitions = '''
 
   [cases.middle1_as_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Root&"]
     arg_types = ["Middle1"]
     setup = '''
@@ -65,7 +65,7 @@ definitions = '''
       ]}]'''
   [cases.middle1_as_middle1]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Middle1&"]
     arg_types = ["ns_inheritance_polymorphic_diamond::Middle1"]
     setup = '''
@@ -85,7 +85,7 @@ definitions = '''
 
   [cases.middle2_as_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Root&"]
     arg_types = ["Middle2"]
     setup = '''
@@ -104,7 +104,7 @@ definitions = '''
       ]}]'''
   [cases.middle2_as_middle2]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Middle2&"]
     arg_types = ["Middle2"]
     setup = '''
@@ -124,7 +124,7 @@ definitions = '''
 
   [cases.child_as_middle1_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     # We need to explicitly cast from Child to Middle1 before going to root to
     # resolve the diamond problem
     param_types = ["const Root&"]
@@ -150,7 +150,7 @@ definitions = '''
       ]}]'''
   [cases.child_as_middle2_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     # We need to explicitly cast from Child to Middle2 before going to root to
     # resolve the diamond problem
     param_types = ["const Root&"]
@@ -176,7 +176,7 @@ definitions = '''
       ]}]'''
   [cases.child_as_middle1]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Middle1&"]
     arg_types = ["Child"]
     setup = '''
@@ -200,7 +200,7 @@ definitions = '''
       ]}]'''
   [cases.child_as_middle2]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Middle2&"]
     arg_types = ["Child"]
     setup = '''
@@ -224,7 +224,7 @@ definitions = '''
       ]}]'''
   [cases.child_as_child]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Child&"]
     arg_types = ["Child"]
     setup = '''
diff --git a/test/integration/inheritance_polymorphic_non_dynamic_base.toml b/test/integration/inheritance_polymorphic_non_dynamic_base.toml
index fff0199..e482e92 100644
--- a/test/integration/inheritance_polymorphic_non_dynamic_base.toml
+++ b/test/integration/inheritance_polymorphic_non_dynamic_base.toml
@@ -22,7 +22,7 @@ definitions = '''
 [cases]
   [cases.a_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["A"]
     setup = "return {};"
@@ -54,7 +54,7 @@ definitions = '''
 
   [cases.b_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["B"]
     setup = '''
@@ -71,7 +71,7 @@ definitions = '''
       ]}]'''
   [cases.b_as_b]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const B&"]
     arg_types = ["B"]
     setup = '''
@@ -109,7 +109,7 @@ definitions = '''
 
   [cases.c_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["C"]
     setup = '''
@@ -126,7 +126,7 @@ definitions = '''
       ]}]'''
   [cases.c_as_b]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const B&"]
     arg_types = ["C"]
     setup = '''
@@ -146,7 +146,7 @@ definitions = '''
       ]}]'''
   [cases.c_as_c]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const C&"]
     arg_types = ["C"]
     setup = '''
diff --git a/test/integration/pointers.toml b/test/integration/pointers.toml
index 71bf2ad..cfd3627 100644
--- a/test/integration/pointers.toml
+++ b/test/integration/pointers.toml
@@ -17,7 +17,7 @@ definitions = '''
     skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19
     param_types = ["int*"]
     setup = "return new int(1);"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "typeName": "int *",
       "staticSize": 8,
@@ -59,7 +59,7 @@ definitions = '''
     skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19
     param_types = ["void*"]
     setup = "return new int(1);"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "typeName": "void *",
       "staticSize": 8,
@@ -95,7 +95,7 @@ definitions = '''
     skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19
     param_types = ["std::vector<int>*"]
     setup = "return new std::vector<int>{1,2,3};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "typeName": "std::vector<int> *",
       "staticSize": 8,
@@ -134,10 +134,10 @@ definitions = '''
 
 
   [cases.struct_primitive_ptrs]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, new int(0), new int(0)};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":4,
@@ -161,9 +161,10 @@ definitions = '''
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_primitive_ptrs_null]
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, nullptr, nullptr};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
@@ -177,10 +178,10 @@ definitions = '''
 
 
   [cases.struct_vector_ptr]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{new std::vector<int>{1,2,3}};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":36,
@@ -199,9 +200,10 @@ definitions = '''
         {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_vector_ptr_null]
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{nullptr};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
@@ -213,10 +215,10 @@ definitions = '''
 
 
   [cases.vector_of_pointers]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const std::vector<int*>&"]
     setup = "return {{new int(1), nullptr, new int(3)}};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":32,
@@ -258,7 +260,7 @@ definitions = '''
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.feature_config]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const std::vector<int*>&"]
     setup = "return {{new int(1), nullptr, new int(3)}};"
     config_prefix = 'features = ["chase-raw-pointers"]'
diff --git a/test/integration/pointers_function.toml b/test/integration/pointers_function.toml
index a62ef9a..1c7934f 100644
--- a/test/integration/pointers_function.toml
+++ b/test/integration/pointers_function.toml
@@ -29,11 +29,10 @@ definitions = '''
       }]
     }]'''
   [cases.raw_chase] # We should never chase function pointers
-    oid_skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22
-    oil_disable = "oil can't chase raw pointers safely"
+    skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22
     param_types = ["const FuncPtrStruct&"]
     setup = "return {{myFunction}};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 8,
       "dynamicSize": 0,
@@ -73,11 +72,10 @@ definitions = '''
       "NOT": "members"
     }]'''
   [cases.std_function_chase] # We should never chase function pointers
-    oid_skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22
-    oil_disable = "oil can't chase raw pointers safely"
+    skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22
     param_types = ["std::function<void(int)> &"]
     setup = "return myFunction;"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "typeName": "function<void (int)>",
       "staticSize": 32,
diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml
index 2d34865..87af9ad 100644
--- a/test/integration/pointers_incomplete.toml
+++ b/test/integration/pointers_incomplete.toml
@@ -19,38 +19,37 @@ definitions = '''
 '''
 [cases]
   [cases.raw]
-    oil_disable = "oil can't chase raw pointers safely"
     oid_skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oil_skip = "oil codegen fails on top level incomplete objects" # https://github.com/facebookexperimental/object-introspection/issues/459
     param_types = ["IncompleteType*"]
     setup = "return static_cast<IncompleteType*>(::operator new(5));"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
-      "typeName": "IncompleteType *",
-      "staticSize": 8,
+      "typeName": "IncompleteType",
+      "staticSize": 0,
       "dynamicSize": 0,
-      "NOT": {"pointer": 0},
       "NOT": "members"
     }]'''
   [cases.raw_no_follow]
-    skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oid_skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oil_skip = "oil codegen fails on top level incomplete objects" # https://github.com/facebookexperimental/object-introspection/issues/459
     param_types = ["IncompleteType*"]
     setup = "return static_cast<IncompleteType*>(::operator new(5));"
     expect_json = '''[{
-      "typeName": "IncompleteType *",
-      "staticSize": 8,
+      "typeName": "IncompleteType",
+      "staticSize": 0,
       "dynamicSize": 0,
-      "NOT": {"pointer": 0},
       "NOT": "members"
     }]'''
   [cases.raw_null]
-    skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oid_skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oil_skip = "oil codegen fails on top level incomplete objects" # https://github.com/facebookexperimental/object-introspection/issues/459
     param_types = ["IncompleteType*"]
     setup = "return nullptr;"
     expect_json = '''[{
-      "typeName": "IncompleteType *",
-      "staticSize": 8,
+      "typeName": "IncompleteType",
+      "staticSize": 0,
       "dynamicSize": 0,
-      "pointer": 0,
       "NOT": "members"
     }]'''
 
@@ -87,10 +86,10 @@ definitions = '''
     expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.containing_struct]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const IncompleteTypeContainer&"]
     setup = "return IncompleteTypeContainer{};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 88,
       "dynamicSize": 0,
diff --git a/test/integration/runner_common.h b/test/integration/runner_common.h
index d2b5e57..d1f2bae 100644
--- a/test/integration/runner_common.h
+++ b/test/integration/runner_common.h
@@ -51,6 +51,8 @@ class IntegrationBase : public ::testing::Test {
   std::string stdout_;
   std::string stderr_;
 
+  std::vector<std::string> features;
+
   /*
    * compare_json
    *
diff --git a/test/integration/std_smart_ptr.toml b/test/integration/std_smart_ptr.toml
index dfb86ae..c9ab383 100644
--- a/test/integration/std_smart_ptr.toml
+++ b/test/integration/std_smart_ptr.toml
@@ -373,7 +373,7 @@ definitions = '''
     expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_present_chase]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     setup = '''
     static std::shared_ptr<std::uint64_t> shd = std::make_shared<std::uint64_t>(0xDEADBEEF);
     std::weak_ptr<std::uint64_t> weak = shd;
@@ -392,7 +392,7 @@ definitions = '''
     expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_expired_chase]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     setup = '''
     return std::make_shared<std::uint64_t>(0xDEADBEEF);
     '''
diff --git a/test/integration/thrift_isset.toml b/test/integration/thrift_isset.toml
index 064cb13..98e19fd 100644
--- a/test/integration/thrift_isset.toml
+++ b/test/integration/thrift_isset.toml
@@ -85,7 +85,7 @@ namespace cpp2 {
       ret.c_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
@@ -109,7 +109,7 @@ namespace cpp2 {
       ret.j_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":44,
       "dynamicSize":0,
@@ -136,7 +136,7 @@ namespace cpp2 {
       ret.c_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":20,
       "dynamicSize":0,
@@ -156,7 +156,7 @@ namespace cpp2 {
       ret.b_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
@@ -176,7 +176,7 @@ namespace cpp2 {
       ret.f_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":28,
       "dynamicSize":0,
@@ -199,7 +199,7 @@ namespace cpp2 {
       ret.e_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":32,
       "dynamicSize":0,
diff --git a/test/integration/thrift_isset_missing.toml b/test/integration/thrift_isset_missing.toml
index 63b578f..3e7ad94 100644
--- a/test/integration/thrift_isset_missing.toml
+++ b/test/integration/thrift_isset_missing.toml
@@ -68,7 +68,7 @@ raw_definitions = '''
       ret.__isset.at(2) = true;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
@@ -89,7 +89,7 @@ raw_definitions = '''
       ret.__isset.at(2) = true;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
@@ -117,7 +117,7 @@ raw_definitions = '''
 
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":32,
       "dynamicSize":0,
diff --git a/test/integration/thrift_namespaces.toml b/test/integration/thrift_namespaces.toml
index f2ce9f9..2218ba3 100644
--- a/test/integration/thrift_namespaces.toml
+++ b/test/integration/thrift_namespaces.toml
@@ -16,7 +16,7 @@ thrift_definitions = '''
       namespaceA::namespaceB::TTTTT ret;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,

From 03c81cbdb45ce6c20e240269beb6890c5e925a48 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 16 Jan 2024 11:15:05 +0000
Subject: [PATCH 136/188] tbv2: fix pointer codegen

A previous change enabled running OIL tests with specific features enabled.
This highlighted that pointer code generation under TreeBuilder-v2 was very
broken. This change updates pointer code generation to work and enables the
skipped tests. All enabled tests need `expected_json_v2` added to them due to
formatting differences.

Reformatted and rewrote the basic type handler that handles primitives and
pointers. Removed the reliance on `features` to decide whether to generate for
TreeBuilder-v2 as the intermediate features have been removed.

There were a couple of other changes needed to enable these tests on TBv2 that
aren't worth their own issues and PRs, I sneaked them in here.

Extra changes:
- Added `Pointer` and `Reference` to TopoSorter so they generate
  `NameProvider` instances. It might be worth visiting the graph differently
  for `NameProvider` as it requires so many instances that others generators do
  not. Will consider that in the future.
- Follow typedefs when calculating exclusive size for a type.

Closes #458.

Test plan:
- CI
- Enabled previously disabled tests.
---
 oi/CodeGen.cpp                            |  35 ++--
 oi/FuncGen.cpp                            | 211 ++++++++++++----------
 oi/FuncGen.h                              |   2 +-
 oi/type_graph/TopoSorter.cpp              |  23 ++-
 oi/type_graph/TopoSorter.h                |   1 +
 test/integration/anonymous.toml           | 123 ++++++++++++-
 test/integration/pointers.toml            |  82 ++++++++-
 test/integration/pointers_incomplete.toml |  27 ++-
 test/test_topo_sorter.cpp                 |  18 ++
 9 files changed, 388 insertions(+), 134 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index cb5bf54..963410e 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -195,7 +195,12 @@ void genDecls(const TypeGraph& typeGraph, std::string& code) {
 namespace {
 
 size_t calculateExclusiveSize(const Type& t) {
-  if (const auto* c = dynamic_cast<const Class*>(&t)) {
+  const Type* finalType = &t;
+  while (const auto* td = dynamic_cast<const Typedef*>(finalType)) {
+    finalType = &td->underlyingType();
+  }
+
+  if (const auto* c = dynamic_cast<const Class*>(finalType)) {
     return std::accumulate(
         c->members.cbegin(), c->members.cend(), 0, [](size_t a, const auto& m) {
           if (m.name.starts_with(AddPadding::MemberPrefix))
@@ -203,7 +208,7 @@ size_t calculateExclusiveSize(const Type& t) {
           return a;
         });
   }
-  return t.size();
+  return finalType->size();
 }
 
 }  // namespace
@@ -211,7 +216,7 @@ size_t calculateExclusiveSize(const Type& t) {
 void genNames(const TypeGraph& typeGraph, std::string& code) {
   code += R"(
 template <typename T>
-struct NameProvider {};
+struct NameProvider;
 )";
 
   // TODO: stop types being duplicated at this point and remove this check
@@ -239,6 +244,9 @@ struct ExclusiveSizeProvider {
 )";
 
   for (const Type& t : typeGraph.finalTypes) {
+    if (dynamic_cast<const Typedef*>(&t))
+      continue;
+
     size_t exclusiveSize = calculateExclusiveSize(t);
     if (exclusiveSize != t.size()) {
       code += "template <> struct ExclusiveSizeProvider<";
@@ -1066,23 +1074,6 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
     }
 )";
 
-  if (features[Feature::TreeBuilderV2]) {
-    code += R"(
-template <typename Ctx, typename T>
-constexpr inst::Field make_field(std::string_view name) {
-  return inst::Field{
-    sizeof(T),
-    ExclusiveSizeProvider<T>::size,
-    name,
-    NameProvider<T>::names,
-    TypeHandler<Ctx, T>::fields,
-    TypeHandler<Ctx, T>::processors,
-    std::is_fundamental_v<T>
-  };
-}
-)";
-  }
-
   // TODO: bit of a hack - making ContainerInfo a node in the type graph and
   // traversing for it would remove the need for this set altogether.
   std::unordered_set<const ContainerInfo*> used{};
@@ -1264,9 +1255,6 @@ void CodeGen::generate(TypeGraph& typeGraph,
     code += "using namespace oi::detail;\n";
     code += "using oi::exporters::ParsedData;\n";
     code += "using namespace oi::exporters;\n";
-    code += "namespace OIInternal {\nnamespace {\n";
-    FuncGen::DefineBasicTypeHandlers(code, config_.features);
-    code += "} // namespace\n} // namespace OIInternal\n";
   }
 
   if (config_.features[Feature::CaptureThriftIsset]) {
@@ -1306,6 +1294,7 @@ void CodeGen::generate(TypeGraph& typeGraph,
   }
 
   if (config_.features[Feature::TreeBuilderV2]) {
+    FuncGen::DefineBasicTypeHandlers(code);
     addStandardTypeHandlers(typeGraph, config_.features, code);
     addTypeHandlers(typeGraph, code);
   } else {
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 5ac3abf..809ae14 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -614,110 +614,129 @@ class BackInserter {
  * pointer's value always, then the value of the pointer if it is unique. void
  * is of type Unit and always stores nothing.
  */
-void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
+void FuncGen::DefineBasicTypeHandlers(std::string& code) {
   code += R"(
-    template <typename Ctx, typename T>
-    struct TypeHandler {
-        using DB = typename Ctx::DataBuffer;
-      private:
-        static auto choose_type() {
-            if constexpr(std::is_pointer_v<T>) {
-                return std::type_identity<types::st::Pair<DB,
-                  types::st::VarInt<DB>,
-                  types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>
-                >>();
-            } else {
-                return std::type_identity<types::st::Unit<DB>>();
-            }
-        }
-
-      public:
-        using type = typename decltype(choose_type())::type;
+template <typename Ctx, typename T>
+struct TypeHandler;
 )";
-  if (features[Feature::TreeBuilderV2]) {
-    code += R"(private:
-        static void process_pointer(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
-          el.pointer = std::get<ParsedData::VarInt>(d.val).value;
-        }
-        static void process_pointer_content(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
-          static constexpr std::array<std::string_view, 1> names{"TODO"};
-          static constexpr auto childField = make_field<Ctx, T>("*");
 
-          const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
-
-          el.container_stats.emplace(result::Element::ContainerStats{ .capacity = 1 });
-
-          if (sum.index == 0)
-            return;
-
-          el.container_stats->length = 1;
-          stack_ins(childField);
-        }
-
-        static constexpr auto choose_fields() {
-          if constexpr(std::is_pointer_v<T>) {
-            return std::array<exporters::inst::Field, 0>{};
-          } else {
-            return std::array<exporters::inst::Field, 0>{};
-          }
-        }
-        static constexpr auto choose_processors() {
-          if constexpr(std::is_pointer_v<T>) {
-            return std::array<inst::ProcessorInst, 2>{
-              {types::st::VarInt<DB>::describe, &process_pointer},
-              {types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>::describe, &process_pointer_content},
-            };
-          } else {
-            return std::array<inst::ProcessorInst, 0>{};
-          }
-        }
-      public:
-        static constexpr auto fields = choose_fields();
-        static constexpr auto processors = choose_processors();
+  code += R"(
+template <typename Ctx, typename T>
+constexpr inst::Field make_field(std::string_view name) {
+  return inst::Field{
+      sizeof(T),
+      ExclusiveSizeProvider<T>::size,
+      name,
+      NameProvider<T>::names,
+      TypeHandler<Ctx, T>::fields,
+      TypeHandler<Ctx, T>::processors,
+      std::is_fundamental_v<T>,
+  };
+}
 )";
+  code += R"(
+template <typename Ctx, typename T>
+struct TypeHandler {
+  using DB = typename Ctx::DataBuffer;
+
+ private:
+  static void process_pointer(result::Element& el,
+                              std::function<void(inst::Inst)> stack_ins,
+                              ParsedData d) {
+    el.pointer = std::get<ParsedData::VarInt>(d.val).value;
   }
-  code += R"(
-        static types::st::Unit<DB> getSizeType(
-          Ctx& ctx,
-          const T& t,
-          typename TypeHandler<Ctx, T>::type returnArg) {
-              if constexpr(std::is_pointer_v<T>) {
-                JLOG("ptr val @");
-                JLOGPTR(t);
-                auto r0 = returnArg.write((uintptr_t)t);
-                if (t && ctx.pointers.add((uintptr_t)t)) {
-                  return r0.template delegate<1>([&t](auto ret) {
-                    if constexpr (!std::is_void<std::remove_pointer_t<T>>::value) {
-                      return TypeHandler<Ctx, std::remove_pointer_t<T>>::getSizeType(*t, ret);
-                    } else {
-                      return ret;
-                    }
-                  });
-                } else {
-                  return r0.template delegate<0>(std::identity());
-                }
-              } else {
-                return returnArg;
-              }
-        }
-    };
-  )";
+  static void process_pointer_content(result::Element& el,
+                                      std::function<void(inst::Inst)> stack_ins,
+                                      ParsedData d) {
+    using U = std::decay_t<std::remove_pointer_t<T>>;
+    const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
+
+    if constexpr (oi_is_complete<U>) {
+      static constexpr auto childField = make_field<Ctx, U>("*");
+
+      el.container_stats.emplace(result::Element::ContainerStats{ .capacity = 1, .length = 0 });
+
+      if (sum.index == 0)
+        return;
+
+      el.container_stats->length = 1;
+      stack_ins(childField);
+    }
+  }
+
+  static auto choose_type() {
+    if constexpr (std::is_pointer_v<T>) {
+      return std::type_identity<types::st::Pair<
+          DB,
+          types::st::VarInt<DB>,
+          types::st::Sum<
+              DB,
+              types::st::Unit<DB>,
+              typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>>>();
+    } else {
+      return std::type_identity<types::st::Unit<DB>>();
+    }
+  }
+  static constexpr auto choose_processors() {
+    if constexpr (std::is_pointer_v<T>) {
+      return std::array<inst::ProcessorInst, 2>{
+          exporters::inst::ProcessorInst{types::st::VarInt<DB>::describe,
+                                         &process_pointer},
+          exporters::inst::ProcessorInst{
+              types::st::Sum<
+                  DB,
+                  types::st::Unit<DB>,
+                  typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>::
+                  describe,
+              &process_pointer_content},
+      };
+    } else {
+      return std::array<inst::ProcessorInst, 0>{};
+    }
+  }
+
+ public:
+  using type = typename decltype(choose_type())::type;
+
+  static constexpr std::array<exporters::inst::Field, 0> fields{};
+  static constexpr auto processors = choose_processors();
+
+  static types::st::Unit<DB> getSizeType(
+      Ctx& ctx, const T& t, typename TypeHandler<Ctx, T>::type returnArg) {
+    if constexpr (std::is_pointer_v<T>) {
+      JLOG("ptr val @");
+      JLOGPTR(t);
+      auto r0 = returnArg.write((uintptr_t)t);
+      if (t && ctx.pointers.add((uintptr_t)t)) {
+        return r0.template delegate<1>([&ctx, &t](auto ret) {
+          using U = std::decay_t<std::remove_pointer_t<T>>;
+          if constexpr (oi_is_complete<U>) {
+            return TypeHandler<Ctx, U>::getSizeType(ctx, *t, ret);
+          } else {
+            return ret;
+          }
+        });
+      } else {
+        return r0.template delegate<0>(std::identity());
+      }
+    } else {
+      return returnArg;
+    }
+  }
+};
+)";
 
   code += R"(
-    template <typename Ctx>
-    class TypeHandler<Ctx, void> {
-        using DB = typename Ctx::DataBuffer;
-      public:
-        using type = types::st::Unit<DB>;
+template <typename Ctx>
+class TypeHandler<Ctx, void> {
+  using DB = typename Ctx::DataBuffer;
+
+ public:
+  using type = types::st::Unit<DB>;
+  static constexpr std::array<exporters::inst::Field, 0> fields{};
+  static constexpr std::array<exporters::inst::ProcessorInst, 0> processors{};
+};
 )";
-  if (features[Feature::TreeBuilderV2]) {
-    code +=
-        "static constexpr std::array<exporters::inst::Field, 0> fields{};\n";
-    code +=
-        "static constexpr std::array<exporters::inst::ProcessorInst, 0> "
-        "processors{};\n";
-  }
-  code += "};\n";
 }
 
 ContainerInfo FuncGen::GetOiArrayContainerInfo() {
diff --git a/oi/FuncGen.h b/oi/FuncGen.h
index 33aaf9c..e2b2fba 100644
--- a/oi/FuncGen.h
+++ b/oi/FuncGen.h
@@ -76,7 +76,7 @@ class FuncGen {
 
   static void DefineDataSegmentDataBuffer(std::string& testCode);
   static void DefineBackInserterDataBuffer(std::string& code);
-  static void DefineBasicTypeHandlers(std::string& code, FeatureSet features);
+  static void DefineBasicTypeHandlers(std::string& code);
 
   static ContainerInfo GetOiArrayContainerInfo();
 };
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index 1be10f5..ccce3fa 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -126,12 +126,27 @@ void TopoSorter::visit(Pointer& p) {
     // Typedefs can not be forward declared, so we must sort them before
     // pointers which reference them
     accept(p.pointeeType());
-    return;
+  } else {
+    // Pointers do not create a dependency, but we do still care about the types
+    // they point to, so delay them until the end.
+    acceptAfter(p.pointeeType());
   }
 
-  // Pointers do not create a dependency, but we do still care about the types
-  // they point to, so delay them until the end.
-  acceptAfter(p.pointeeType());
+  sortedTypes_.push_back(p);
+}
+
+void TopoSorter::visit(Reference& r) {
+  if (dynamic_cast<Typedef*>(&r.pointeeType())) {
+    // Typedefs can not be forward declared, so we must sort them before
+    // pointers which reference them
+    accept(r.pointeeType());
+  } else {
+    // Pointers do not create a dependency, but we do still care about the types
+    // they point to, so delay them until the end.
+    acceptAfter(r.pointeeType());
+  }
+
+  sortedTypes_.push_back(r);
 }
 
 void TopoSorter::visit(CaptureKeys& c) {
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 5c78ad1..6eb7083 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -46,6 +46,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Enum& e) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& r) override;
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
   void visit(Incomplete& i) override;
diff --git a/test/integration/anonymous.toml b/test/integration/anonymous.toml
index 70600d1..d1e6286 100644
--- a/test/integration/anonymous.toml
+++ b/test/integration/anonymous.toml
@@ -97,7 +97,6 @@ definitions = '''
       ]}]'''
 
   [cases.anon_struct]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const AnonStructContainer&"]
     setup = '''
       return AnonStructContainer{
@@ -133,6 +132,40 @@ definitions = '''
         }]
       }]
     }]'''
+    expect_json_v2 = '''[{
+      "name":"a0",
+      "typeNames":["ns_anonymous::AnonStructContainer"],
+      "staticSize":8,
+      "exclusiveSize":0,
+      "size":20,
+      "members":[{
+        "name":"anon",
+        "typeNames":["__oi_anon_1"],
+        "staticSize":8,
+        "exclusiveSize":0,
+        "size":20,
+        "members":[{
+          "name":"node",
+          "typeNames":["ns_anonymous::Node*"],
+          "staticSize":8,
+          "exclusiveSize":8,
+          "size":20,
+          "length":1,
+          "capacity":1,
+          "members":[{
+            "name":"*",
+            "typeNames":["ns_anonymous::Node"],
+            "staticSize":12,
+            "exclusiveSize":0,
+            "size":12,
+            "members":[
+              { "name": "a", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "b", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "c", "staticSize": 4, "exclusiveSize": 4, "size": 4 }
+            ]
+        }]
+      }]}
+    ]}]'''
 
   [cases.anon_struct_ptr]
     skip = "We don't support pointer to anon-structs yet" # https://github.com/facebookexperimental/object-introspection/issues/20
@@ -146,7 +179,6 @@ definitions = '''
     features = ["chase-raw-pointers"]
 
   [cases.anon_typedef]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const AnonTypedefContainer&"]
     setup = '''
       return AnonTypedefContainer{
@@ -192,6 +224,34 @@ definitions = '''
         }]
       }]
     }]'''
+    expect_json_v2 = '''[{
+      "staticSize": 8,
+      "exclusiveSize": 0,
+      "size": 20,
+      "members": [{
+        "typeNames":["AnonStruct", "__oi_anon_2"],
+        "staticSize": 8,
+        "exclusiveSize": 0,
+        "size": 20,
+        "members": [{
+          "typeNames": ["ns_anonymous::Node*"],
+          "staticSize": 8,
+          "exclusiveSize": 8,
+          "size": 20,
+          "members": [{
+            "typeNames": ["ns_anonymous::Node"],
+            "staticSize": 12,
+            "exclusiveSize": 0,
+            "size": 12,
+            "members": [
+              { "name": "a", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "b", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "c", "staticSize": 4, "exclusiveSize": 4, "size": 4 }
+            ]
+          }]
+        }]
+      }]
+    }]'''
 
   [cases.anon_union]
     param_types = ["const AnonUnionContainer&"]
@@ -217,7 +277,6 @@ definitions = '''
     }]'''
 
   [cases.nested_anon_struct]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const NestedAnonContainer&"]
     features = ["chase-raw-pointers"]
     setup = 'return NestedAnonContainer{.m = { .v = {.as = {new Node{1, 2, 3}}}}};'
@@ -272,6 +331,64 @@ definitions = '''
         "dynamicSize": 0
       }]
     }]'''
+    expect_json_v2 = '''[{
+      "staticSize":80,
+      "exclusiveSize":0,
+      "size":92,
+      "members":[
+        {
+          "name":"m",
+          "typeNames": ["__oi_anon_1"],
+          "staticSize": 48,
+          "exclusiveSize":0,
+          "size": 60,
+          "members":[
+            {"name":"__oi_anon_0", "typeNames":["__oi_anon_2"], "staticSize":16, "exclusiveSize":16, "size":16},
+            {
+              "name":"v",
+              "typeNames":["__oi_anon_3"],
+              "staticSize":32,
+              "exclusiveSize":4,
+              "size":44,
+              "members":[
+                {"name":"a", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+                {"name":"b", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+                {"name":"c", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+                {"name":"__oi_anon_4", "typeNames":["__oi_anon_4"], "staticSize":8, "exclusiveSize":8, "size":8},
+                {
+                  "name":"as",
+                  "typeNames":["AnonStruct", "__oi_anon_6"],
+                  "staticSize":8,
+                  "exclusiveSize":0,
+                  "size":20,
+                  "members":[{
+                    "name":"node",
+                    "typeNames":["ns_anonymous::Node*"],
+                    "staticSize":8,
+                    "exclusiveSize":8,
+                    "size":20,
+                    "members":[{
+                      "name":"*",
+                      "typeNames":["ns_anonymous::Node"],
+                      "staticSize":12,
+                      "exclusiveSize":0,
+                      "size":12,
+                      "members":[
+                        { "name": "a", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+                        { "name": "b", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+                        { "name": "c", "staticSize": 4, "exclusiveSize": 4, "size": 4 }
+                      ]
+                    }]
+                  }]
+                }
+              ]
+            }
+          ]
+        },
+        {"name":"__oi_anon_1", "typeNames": ["__oi_anon_8"], "staticSize":24, "exclusiveSize":24, "size":24},
+        {"name":"__oi_anon_2", "typeNames": ["__oi_anon_9"], "staticSize":8, "exclusiveSize":8, "size":8}
+      ]
+    }]'''
 
   # This test is disabled due to GCC not supporting it
   # [cases.anon_array]
diff --git a/test/integration/pointers.toml b/test/integration/pointers.toml
index cfd3627..6868679 100644
--- a/test/integration/pointers.toml
+++ b/test/integration/pointers.toml
@@ -134,7 +134,6 @@ definitions = '''
 
 
   [cases.struct_primitive_ptrs]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, new int(0), new int(0)};"
     features = ["chase-raw-pointers"]
@@ -147,6 +146,16 @@ definitions = '''
         {"name":"b", "staticSize":8, "exclusiveSize":8, "dynamicSize":4},
         {"name":"c", "staticSize":8, "exclusiveSize":8, "dynamicSize":0}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":4,
+      "size":28,
+      "members":[
+        {"name":"a", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"b", "typeNames":["int32_t*"], "staticSize":8, "exclusiveSize":8, "size":12},
+        {"name":"c", "typeNames":["void*"], "staticSize":8, "exclusiveSize":8, "size":8}
+      ]
+    }]'''
   [cases.struct_primitive_ptrs_no_follow]
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, new int(0), new int(0)};"
@@ -161,7 +170,6 @@ definitions = '''
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_primitive_ptrs_null]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, nullptr, nullptr};"
     features = ["chase-raw-pointers"]
@@ -175,10 +183,19 @@ definitions = '''
         {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":4,
+      "size":24,
+      "members":[
+        {"name":"a", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"b", "typeNames":["int32_t*"], "staticSize":8, "exclusiveSize":8, "size":8},
+        {"name":"c", "typeNames":["void*"], "staticSize":8, "exclusiveSize":8, "size":8}
+      ]
+    }]'''
 
 
   [cases.struct_vector_ptr]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{new std::vector<int>{1,2,3}};"
     features = ["chase-raw-pointers"]
@@ -188,6 +205,22 @@ definitions = '''
       "members":[
         {"name":"vec", "staticSize":8, "dynamicSize":36}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":8,
+      "exclusiveSize":0,
+      "size":44,
+      "members": [
+        {
+          "typeNames":["std::vector<int32_t, std::allocator<int32_t>>*"],
+          "staticSize":8,
+          "exclusiveSize":8,
+          "size":44,
+          "length":1,
+          "capacity":1,
+          "members":[{ "staticSize":24, "exclusiveSize":24, "size":36 }]
+        }
+      ]
+    }]'''
   [cases.struct_vector_ptr_no_follow]
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{new std::vector<int>{1,2,3}};"
@@ -200,7 +233,6 @@ definitions = '''
         {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_vector_ptr_null]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{nullptr};"
     features = ["chase-raw-pointers"]
@@ -212,10 +244,25 @@ definitions = '''
       "members":[
         {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
+      "members": [
+        {
+          "typeNames":["std::vector<int32_t, std::allocator<int32_t>>*"],
+          "staticSize":8,
+          "exclusiveSize":8,
+          "size":8,
+          "length":0,
+          "capacity":1,
+          "members":[]
+        }
+      ]
+    }]'''
 
 
   [cases.vector_of_pointers]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const std::vector<int*>&"]
     setup = "return {{new int(1), nullptr, new int(3)}};"
     features = ["chase-raw-pointers"]
@@ -230,6 +277,18 @@ definitions = '''
         {"staticSize":8, "dynamicSize":0, "pointer":0},
         {"staticSize":8, "dynamicSize":4, "NOT": {"pointer":0}}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":24,
+      "size":56,
+      "length":3,
+      "capacity":3,
+      "members":[
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}},
+        {"staticSize":8, "exclusiveSize":8, "size":8, "pointer":0},
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}}
+      ]
+    }]'''
   [cases.vector_of_pointers_no_follow]
     skip = "pointer field is missing from results" # https://github.com/facebookexperimental/object-introspection/issues/21
     param_types = ["const std::vector<int*>&"]
@@ -260,7 +319,6 @@ definitions = '''
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.feature_config]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const std::vector<int*>&"]
     setup = "return {{new int(1), nullptr, new int(3)}};"
     config_prefix = 'features = ["chase-raw-pointers"]'
@@ -275,3 +333,15 @@ definitions = '''
         {"staticSize":8, "dynamicSize":0, "pointer":0},
         {"staticSize":8, "dynamicSize":4, "NOT": {"pointer":0}}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":24,
+      "size":56,
+      "length":3,
+      "capacity":3,
+      "members":[
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}},
+        {"staticSize":8, "exclusiveSize":8, "size":8, "pointer":0},
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}}
+      ]
+    }]'''
diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml
index 87af9ad..b062043 100644
--- a/test/integration/pointers_incomplete.toml
+++ b/test/integration/pointers_incomplete.toml
@@ -86,7 +86,6 @@ definitions = '''
     expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.containing_struct]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const IncompleteTypeContainer&"]
     setup = "return IncompleteTypeContainer{};"
     features = ["chase-raw-pointers"]
@@ -115,6 +114,32 @@ definitions = '''
         }
       ]
     }]'''
+    expect_json_v2 = '''[{
+      "staticSize": 88,
+      "exclusiveSize": 21,
+      "size": 88,
+      "members": [
+        { "name": "ptrundef", "staticSize": 8, "exclusiveSize": 8, "size": 8 },
+        { "name": "__makePad1", "staticSize": 1, "exclusiveSize": 1, "size": 1 },
+        { "name": "shundef", "staticSize": 16, "exclusiveSize": 16, "size": 16 },
+        { "name": "__makePad2", "staticSize": 1, "exclusiveSize": 1, "size": 1 },
+        { "name": "shoptundef",
+          "staticSize": 24,
+          "exclusiveSize": 24,
+          "size": 24,
+          "length": 0,
+          "capacity": 1
+        },
+        { "name": "__makePad3", "staticSize": 1, "exclusiveSize": 1, "size": 1 },
+        { "name": "optundef",
+          "staticSize": 16,
+          "exclusiveSize": 16,
+          "size": 16,
+          "length": 0,
+          "capacity": 1
+        }
+      ]
+    }]'''
 
   [cases.containing_struct_no_follow]
     param_types = ["const IncompleteTypeContainer&"]
diff --git a/test/test_topo_sorter.cpp b/test/test_topo_sorter.cpp
index c97801f..7c00441 100644
--- a/test/test_topo_sorter.cpp
+++ b/test/test_topo_sorter.cpp
@@ -236,6 +236,22 @@ TEST(TopoSorterTest, Pointers) {
   myclass.members.push_back(Member{mypointer, "ptr", 0});
 
   test({myclass}, R"(
+ClassA*
+MyClass
+ClassA
+)");
+}
+
+TEST(TopoSorterTest, References) {
+  // References do not require pointee types to be defined first
+  auto classA = Class{0, Class::Kind::Class, "ClassA", 69};
+  auto myreference = Reference{1, classA};
+
+  auto myclass = Class{2, Class::Kind::Class, "MyClass", 69};
+  myclass.members.push_back(Member{myreference, "ref", 0});
+
+  test({myclass}, R"(
+ClassA*
 MyClass
 ClassA
 )");
@@ -258,6 +274,7 @@ TEST(TopoSorterTest, PointerCycle) {
   // the same sorted order for ClassA and ClassB.
   for (const auto& input : inputs) {
     test(input, R"(
+ClassA*
 ClassB
 ClassA
 )");
@@ -276,6 +293,7 @@ TEST(TopoSorterTest, PointerToTypedef) {
   test({myclass}, R"(
 ClassA
 aliasA
+aliasA*
 MyClass
 )");
 }

From 1e7e8ffee6820fcc6340bccfc9325c3c41ffd1f5 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 16 Jan 2024 11:15:05 +0000
Subject: [PATCH 137/188] test: add features field to integration tests

Previously we tested different feature flags by using the `cli_options` field
in the test `.toml`. This works for OID but doesn't work for JIT OIL and won't
work for AoT OIL when those tests get added.

This change adds a new higher level `features` field to the test `.toml` which
adds the features to the config file as a prefix. This works with all methods
of generation.

Change the existing `cli_options` features to `features` except for where
they're testing something specific. Enable tests that were previously disabled
for OIL but only didn't work because of not being able to enable features.
Change pointer tests that are currently broken for OIL from `oil_disable` to
`oil_skip` - they can work, but codegen is broken for them at the minute.

Test plan:
- CI
- `make test` is no worse
---
 test/integration/README.md                    | 12 +++++++-
 test/integration/anonymous.toml               | 19 ++++++------
 test/integration/arrays.toml                  |  2 +-
 test/integration/bitfields.toml               | 14 ++++-----
 test/integration/cycles.toml                  |  8 ++---
 test/integration/gen_tests.py                 | 19 +++++++++---
 test/integration/inheritance_polymorphic.toml | 12 ++++----
 .../inheritance_polymorphic_diamond.toml      | 20 ++++++-------
 ...eritance_polymorphic_non_dynamic_base.toml | 12 ++++----
 test/integration/pointers.toml                | 26 +++++++++--------
 test/integration/pointers_function.toml       | 10 +++----
 test/integration/pointers_incomplete.toml     | 29 +++++++++----------
 test/integration/std_smart_ptr.toml           |  4 +--
 test/integration/thrift_isset.toml            | 12 ++++----
 test/integration/thrift_isset_missing.toml    |  6 ++--
 test/integration/thrift_namespaces.toml       |  2 +-
 16 files changed, 113 insertions(+), 94 deletions(-)

diff --git a/test/integration/README.md b/test/integration/README.md
index 4d7d734..08759aa 100644
--- a/test/integration/README.md
+++ b/test/integration/README.md
@@ -201,13 +201,23 @@ definitions = '''
 
     Implies `oil_disable`.
 
+  - `features`
+
+    Append this list of features to the configuration. This works for all types
+    of test.
+
+    Example:
+    ```
+    features = ["chase-raw-pointers"]
+    ```
+
   - `cli_options`
 
     Additional command line arguments passed to oid.
 
     Example:
     ```
-    cli_options = ["-fchase-raw-pointers"]
+    cli_options = ["-Ftype-graph"]
     ```
 
   - `skip`, `oid_skip`, `oil_skip`
diff --git a/test/integration/anonymous.toml b/test/integration/anonymous.toml
index 15b9f10..70600d1 100644
--- a/test/integration/anonymous.toml
+++ b/test/integration/anonymous.toml
@@ -84,7 +84,7 @@ definitions = '''
   [cases.regular_struct]
     param_types = ["const Node&"]
     setup = "return Node{1, 2, 3};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":12,
       "dynamicSize":0,
@@ -97,7 +97,7 @@ definitions = '''
       ]}]'''
 
   [cases.anon_struct]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const AnonStructContainer&"]
     setup = '''
       return AnonStructContainer{
@@ -106,7 +106,7 @@ definitions = '''
         }
       };
     '''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 8,
       "dynamicSize": 12,
@@ -143,10 +143,10 @@ definitions = '''
           new Node{1, 2, 3}
         }
       };'''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
 
   [cases.anon_typedef]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const AnonTypedefContainer&"]
     setup = '''
       return AnonTypedefContainer{
@@ -154,7 +154,7 @@ definitions = '''
           .node = new Node{1, 2, 3}
         }
       };'''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 8,
       "dynamicSize": 12,
@@ -196,7 +196,6 @@ definitions = '''
   [cases.anon_union]
     param_types = ["const AnonUnionContainer&"]
     setup = 'return AnonUnionContainer{ .a = 3 };'
-    cli_options = ["-fchase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 24,
       "dynamicSize": 0,
@@ -218,10 +217,10 @@ definitions = '''
     }]'''
 
   [cases.nested_anon_struct]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const NestedAnonContainer&"]
+    features = ["chase-raw-pointers"]
     setup = 'return NestedAnonContainer{.m = { .v = {.as = {new Node{1, 2, 3}}}}};'
-    cli_options = ["-fchase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 80,
       "dynamicSize": 12,
@@ -290,7 +289,7 @@ definitions = '''
   #     result.ns[3].nodes.resize(22);
   #     return result;
   #   '''
-  #   cli_options = ["-fchase-raw-pointers"]
+  #   features = ["chase-raw-pointers"]
   #   expect_json = '''[{
   #     "staticSize": 104,
   #     "dynamicSize": 556,
diff --git a/test/integration/arrays.toml b/test/integration/arrays.toml
index a4fd3b6..82fe0ed 100644
--- a/test/integration/arrays.toml
+++ b/test/integration/arrays.toml
@@ -76,7 +76,7 @@ definitions = '''
         "elementStaticSize":4
       }]}]'''
   [cases.multidim]
-    cli_options = ["-ftype-graph"]
+    
     param_types = ["const MultiDim&"]
     setup = "return {};"
     expect_json = '''[
diff --git a/test/integration/bitfields.toml b/test/integration/bitfields.toml
index bd1e052..6f66647 100644
--- a/test/integration/bitfields.toml
+++ b/test/integration/bitfields.toml
@@ -53,7 +53,7 @@ definitions = '''
 # if they were regular primitives.
 [cases]
   [cases.single]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["Single&"]
     setup = "return {};"
@@ -62,7 +62,7 @@ definitions = '''
         {"staticSize":4, "dynamicSize":0, "exclusiveSize":4}
     ]}]'''
   [cases.within_bytes]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["WithinBytes&"]
     setup = "return {};"
@@ -73,7 +73,7 @@ definitions = '''
         {"staticSize":1, "dynamicSize":0, "exclusiveSize":1}
     ]}]'''
   [cases.straddle_bytes]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["StraddleBytes&"]
     setup = "return {};"
@@ -84,7 +84,7 @@ definitions = '''
         {"staticSize":1, "dynamicSize":0, "exclusiveSize":1}
     ]}]'''
   [cases.mixed]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["Mixed&"]
     setup = "return {};"
@@ -98,13 +98,13 @@ definitions = '''
     ]}]'''
   [cases.more_bits_than_type] # TODO member sizes are wrong
     skip = "drgn errors out"
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["MoreBitsThanType&"]
     setup = "return {};"
     expect_json = '"TODO"'
   [cases.zero_bits]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["ZeroBits&"]
     setup = "return {};"
@@ -114,7 +114,7 @@ definitions = '''
         {"staticSize":1, "dynamicSize":0, "exclusiveSize":1}
     ]}]'''
   [cases.enum]
-    cli_options = ["-ftype-graph"]
+    
     oil_skip = "not implemented"
     param_types = ["Enum&"]
     setup = "return {};"
diff --git a/test/integration/cycles.toml b/test/integration/cycles.toml
index d6f6daf..77b0d0b 100644
--- a/test/integration/cycles.toml
+++ b/test/integration/cycles.toml
@@ -23,7 +23,7 @@ definitions = '''
 '''
 [cases]
   [cases.raw_ptr]
-    oil_disable = "oil can't chase pointers safely"
+    oil_skip = "cycles are broken" # https://github.com/facebookexperimental/object-introspection/issues/293
     param_types = ["RawNode*"]
     setup = '''
       RawNode *first  = new RawNode{1, nullptr};
@@ -34,7 +34,7 @@ definitions = '''
       third->next = first;
       return first;
     '''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''
     [
       {
@@ -96,7 +96,7 @@ definitions = '''
     '''
 
   [cases.raw_ptr_wrapped]
-    oil_disable = "oil can't chase pointers safely"
+    oil_skip = "cycles are broken" # https://github.com/facebookexperimental/object-introspection/issues/293
     param_types = ["Wrapper<RawNode*>&"]
     setup = '''
       RawNode *first  = new RawNode{1, nullptr};
@@ -107,7 +107,7 @@ definitions = '''
       third->next = first;
       return Wrapper<RawNode*>(first);
     '''
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 8,
       "dynamicSize": 48,
diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py
index b72e371..d10ddd7 100644
--- a/test/integration/gen_tests.py
+++ b/test/integration/gen_tests.py
@@ -268,6 +268,19 @@ def add_tests(f, config):
         add_oil_integration_test(f, config, case_name, case)
 
 
+def get_config_strings(case):
+    config_prefix = case.get("config_prefix", "")
+    if "features" in case:
+        feature_config = "features = ["
+        for f in case.get("features", []):
+            feature_config += '"' + f + '"'
+        feature_config += "]\n"
+        config_prefix = feature_config + config_prefix
+    config_suffix = case.get("config_suffix", "")
+
+    return (config_prefix, config_suffix)
+
+
 def add_oid_integration_test(f, config, case_name, case):
     probe_type = case.get("type", "entry")
     args = case.get("args", "arg0")
@@ -283,8 +296,7 @@ def add_oid_integration_test(f, config, case_name, case):
         "{" + ", ".join(f'"{option}"' for option in case.get("cli_options", ())) + "}"
     )
 
-    config_prefix = case.get("config_prefix", "")
-    config_suffix = case.get("config_suffix", "")
+    config_prefix, config_suffix = get_config_strings(case)
 
     f.write(
         f"\n"
@@ -356,8 +368,7 @@ def add_oil_integration_test(f, config, case_name, case):
     if "oil_disable" in case or "target_function" in case:
         return
 
-    config_prefix = case.get("config_prefix", "")
-    config_suffix = case.get("config_suffix", "")
+    config_prefix, config_suffix = get_config_strings(case)
 
     f.write(
         f"\n"
diff --git a/test/integration/inheritance_polymorphic.toml b/test/integration/inheritance_polymorphic.toml
index fe50462..0b9e577 100644
--- a/test/integration/inheritance_polymorphic.toml
+++ b/test/integration/inheritance_polymorphic.toml
@@ -24,7 +24,7 @@ definitions = '''
 [cases]
   [cases.a_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["A"]
     setup = "return {};"
@@ -39,7 +39,7 @@ definitions = '''
 
   [cases.b_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["B"]
     setup = '''
@@ -58,7 +58,7 @@ definitions = '''
       ]}]'''
   [cases.b_as_b]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const B&"]
     arg_types = ["B"]
     setup = '''
@@ -78,7 +78,7 @@ definitions = '''
 
   [cases.c_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["C"]
     setup = '''
@@ -98,7 +98,7 @@ definitions = '''
       ]}]'''
   [cases.c_as_b]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const B&"]
     arg_types = ["C"]
     setup = '''
@@ -118,7 +118,7 @@ definitions = '''
       ]}]'''
   [cases.c_as_c]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const C&"]
     arg_types = ["C"]
     setup = '''
diff --git a/test/integration/inheritance_polymorphic_diamond.toml b/test/integration/inheritance_polymorphic_diamond.toml
index ec9a0e1..cff0638 100644
--- a/test/integration/inheritance_polymorphic_diamond.toml
+++ b/test/integration/inheritance_polymorphic_diamond.toml
@@ -31,7 +31,7 @@ definitions = '''
 [cases]
   [cases.root_as_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Root&"]
     arg_types = ["Root"]
     setup = "return {};"
@@ -46,7 +46,7 @@ definitions = '''
 
   [cases.middle1_as_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Root&"]
     arg_types = ["Middle1"]
     setup = '''
@@ -65,7 +65,7 @@ definitions = '''
       ]}]'''
   [cases.middle1_as_middle1]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Middle1&"]
     arg_types = ["ns_inheritance_polymorphic_diamond::Middle1"]
     setup = '''
@@ -85,7 +85,7 @@ definitions = '''
 
   [cases.middle2_as_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Root&"]
     arg_types = ["Middle2"]
     setup = '''
@@ -104,7 +104,7 @@ definitions = '''
       ]}]'''
   [cases.middle2_as_middle2]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Middle2&"]
     arg_types = ["Middle2"]
     setup = '''
@@ -124,7 +124,7 @@ definitions = '''
 
   [cases.child_as_middle1_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     # We need to explicitly cast from Child to Middle1 before going to root to
     # resolve the diamond problem
     param_types = ["const Root&"]
@@ -150,7 +150,7 @@ definitions = '''
       ]}]'''
   [cases.child_as_middle2_root]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     # We need to explicitly cast from Child to Middle2 before going to root to
     # resolve the diamond problem
     param_types = ["const Root&"]
@@ -176,7 +176,7 @@ definitions = '''
       ]}]'''
   [cases.child_as_middle1]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Middle1&"]
     arg_types = ["Child"]
     setup = '''
@@ -200,7 +200,7 @@ definitions = '''
       ]}]'''
   [cases.child_as_middle2]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Middle2&"]
     arg_types = ["Child"]
     setup = '''
@@ -224,7 +224,7 @@ definitions = '''
       ]}]'''
   [cases.child_as_child]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const Child&"]
     arg_types = ["Child"]
     setup = '''
diff --git a/test/integration/inheritance_polymorphic_non_dynamic_base.toml b/test/integration/inheritance_polymorphic_non_dynamic_base.toml
index fff0199..e482e92 100644
--- a/test/integration/inheritance_polymorphic_non_dynamic_base.toml
+++ b/test/integration/inheritance_polymorphic_non_dynamic_base.toml
@@ -22,7 +22,7 @@ definitions = '''
 [cases]
   [cases.a_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["A"]
     setup = "return {};"
@@ -54,7 +54,7 @@ definitions = '''
 
   [cases.b_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["B"]
     setup = '''
@@ -71,7 +71,7 @@ definitions = '''
       ]}]'''
   [cases.b_as_b]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const B&"]
     arg_types = ["B"]
     setup = '''
@@ -109,7 +109,7 @@ definitions = '''
 
   [cases.c_as_a]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const A&"]
     arg_types = ["C"]
     setup = '''
@@ -126,7 +126,7 @@ definitions = '''
       ]}]'''
   [cases.c_as_b]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const B&"]
     arg_types = ["C"]
     setup = '''
@@ -146,7 +146,7 @@ definitions = '''
       ]}]'''
   [cases.c_as_c]
     oil_skip = "Polymorphic inheritance disabled in OIL"
-    cli_options = ["-fpolymorphic-inheritance"]
+    features = ["polymorphic-inheritance"]
     param_types = ["const C&"]
     arg_types = ["C"]
     setup = '''
diff --git a/test/integration/pointers.toml b/test/integration/pointers.toml
index 71bf2ad..cfd3627 100644
--- a/test/integration/pointers.toml
+++ b/test/integration/pointers.toml
@@ -17,7 +17,7 @@ definitions = '''
     skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19
     param_types = ["int*"]
     setup = "return new int(1);"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "typeName": "int *",
       "staticSize": 8,
@@ -59,7 +59,7 @@ definitions = '''
     skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19
     param_types = ["void*"]
     setup = "return new int(1);"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "typeName": "void *",
       "staticSize": 8,
@@ -95,7 +95,7 @@ definitions = '''
     skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19
     param_types = ["std::vector<int>*"]
     setup = "return new std::vector<int>{1,2,3};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "typeName": "std::vector<int> *",
       "staticSize": 8,
@@ -134,10 +134,10 @@ definitions = '''
 
 
   [cases.struct_primitive_ptrs]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, new int(0), new int(0)};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":4,
@@ -161,9 +161,10 @@ definitions = '''
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_primitive_ptrs_null]
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, nullptr, nullptr};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":0,
@@ -177,10 +178,10 @@ definitions = '''
 
 
   [cases.struct_vector_ptr]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{new std::vector<int>{1,2,3}};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":36,
@@ -199,9 +200,10 @@ definitions = '''
         {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_vector_ptr_null]
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{nullptr};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":8,
       "dynamicSize":0,
@@ -213,10 +215,10 @@ definitions = '''
 
 
   [cases.vector_of_pointers]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const std::vector<int*>&"]
     setup = "return {{new int(1), nullptr, new int(3)}};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize":24,
       "dynamicSize":32,
@@ -258,7 +260,7 @@ definitions = '''
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.feature_config]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const std::vector<int*>&"]
     setup = "return {{new int(1), nullptr, new int(3)}};"
     config_prefix = 'features = ["chase-raw-pointers"]'
diff --git a/test/integration/pointers_function.toml b/test/integration/pointers_function.toml
index a62ef9a..1c7934f 100644
--- a/test/integration/pointers_function.toml
+++ b/test/integration/pointers_function.toml
@@ -29,11 +29,10 @@ definitions = '''
       }]
     }]'''
   [cases.raw_chase] # We should never chase function pointers
-    oid_skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22
-    oil_disable = "oil can't chase raw pointers safely"
+    skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22
     param_types = ["const FuncPtrStruct&"]
     setup = "return {{myFunction}};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 8,
       "dynamicSize": 0,
@@ -73,11 +72,10 @@ definitions = '''
       "NOT": "members"
     }]'''
   [cases.std_function_chase] # We should never chase function pointers
-    oid_skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22
-    oil_disable = "oil can't chase raw pointers safely"
+    skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22
     param_types = ["std::function<void(int)> &"]
     setup = "return myFunction;"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "typeName": "function<void (int)>",
       "staticSize": 32,
diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml
index 2d34865..87af9ad 100644
--- a/test/integration/pointers_incomplete.toml
+++ b/test/integration/pointers_incomplete.toml
@@ -19,38 +19,37 @@ definitions = '''
 '''
 [cases]
   [cases.raw]
-    oil_disable = "oil can't chase raw pointers safely"
     oid_skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oil_skip = "oil codegen fails on top level incomplete objects" # https://github.com/facebookexperimental/object-introspection/issues/459
     param_types = ["IncompleteType*"]
     setup = "return static_cast<IncompleteType*>(::operator new(5));"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
-      "typeName": "IncompleteType *",
-      "staticSize": 8,
+      "typeName": "IncompleteType",
+      "staticSize": 0,
       "dynamicSize": 0,
-      "NOT": {"pointer": 0},
       "NOT": "members"
     }]'''
   [cases.raw_no_follow]
-    skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oid_skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oil_skip = "oil codegen fails on top level incomplete objects" # https://github.com/facebookexperimental/object-introspection/issues/459
     param_types = ["IncompleteType*"]
     setup = "return static_cast<IncompleteType*>(::operator new(5));"
     expect_json = '''[{
-      "typeName": "IncompleteType *",
-      "staticSize": 8,
+      "typeName": "IncompleteType",
+      "staticSize": 0,
       "dynamicSize": 0,
-      "NOT": {"pointer": 0},
       "NOT": "members"
     }]'''
   [cases.raw_null]
-    skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oid_skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17
+    oil_skip = "oil codegen fails on top level incomplete objects" # https://github.com/facebookexperimental/object-introspection/issues/459
     param_types = ["IncompleteType*"]
     setup = "return nullptr;"
     expect_json = '''[{
-      "typeName": "IncompleteType *",
-      "staticSize": 8,
+      "typeName": "IncompleteType",
+      "staticSize": 0,
       "dynamicSize": 0,
-      "pointer": 0,
       "NOT": "members"
     }]'''
 
@@ -87,10 +86,10 @@ definitions = '''
     expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.containing_struct]
-    oil_disable = "oil can't chase raw pointers safely"
+    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const IncompleteTypeContainer&"]
     setup = "return IncompleteTypeContainer{};"
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     expect_json = '''[{
       "staticSize": 88,
       "dynamicSize": 0,
diff --git a/test/integration/std_smart_ptr.toml b/test/integration/std_smart_ptr.toml
index dfb86ae..c9ab383 100644
--- a/test/integration/std_smart_ptr.toml
+++ b/test/integration/std_smart_ptr.toml
@@ -373,7 +373,7 @@ definitions = '''
     expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_present_chase]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     setup = '''
     static std::shared_ptr<std::uint64_t> shd = std::make_shared<std::uint64_t>(0xDEADBEEF);
     std::weak_ptr<std::uint64_t> weak = shd;
@@ -392,7 +392,7 @@ definitions = '''
     expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "size": 16, "members":[]}]'''
   [cases.weak_ptr_int64_expired_chase]
     param_types = ["std::weak_ptr<std::uint64_t>&"]
-    cli_options = ["-fchase-raw-pointers"]
+    features = ["chase-raw-pointers"]
     setup = '''
     return std::make_shared<std::uint64_t>(0xDEADBEEF);
     '''
diff --git a/test/integration/thrift_isset.toml b/test/integration/thrift_isset.toml
index 064cb13..98e19fd 100644
--- a/test/integration/thrift_isset.toml
+++ b/test/integration/thrift_isset.toml
@@ -85,7 +85,7 @@ namespace cpp2 {
       ret.c_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
@@ -109,7 +109,7 @@ namespace cpp2 {
       ret.j_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":44,
       "dynamicSize":0,
@@ -136,7 +136,7 @@ namespace cpp2 {
       ret.c_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":20,
       "dynamicSize":0,
@@ -156,7 +156,7 @@ namespace cpp2 {
       ret.b_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
@@ -176,7 +176,7 @@ namespace cpp2 {
       ret.f_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":28,
       "dynamicSize":0,
@@ -199,7 +199,7 @@ namespace cpp2 {
       ret.e_ref() = 1;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":32,
       "dynamicSize":0,
diff --git a/test/integration/thrift_isset_missing.toml b/test/integration/thrift_isset_missing.toml
index 63b578f..3e7ad94 100644
--- a/test/integration/thrift_isset_missing.toml
+++ b/test/integration/thrift_isset_missing.toml
@@ -68,7 +68,7 @@ raw_definitions = '''
       ret.__isset.at(2) = true;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
@@ -89,7 +89,7 @@ raw_definitions = '''
       ret.__isset.at(2) = true;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,
@@ -117,7 +117,7 @@ raw_definitions = '''
 
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":32,
       "dynamicSize":0,
diff --git a/test/integration/thrift_namespaces.toml b/test/integration/thrift_namespaces.toml
index f2ce9f9..2218ba3 100644
--- a/test/integration/thrift_namespaces.toml
+++ b/test/integration/thrift_namespaces.toml
@@ -16,7 +16,7 @@ thrift_definitions = '''
       namespaceA::namespaceB::TTTTT ret;
       return ret;
     '''
-    cli_options = ["-fcapture-thrift-isset"]
+    features = ["capture-thrift-isset"]
     expect_json = '''[{
       "staticSize":16,
       "dynamicSize":0,

From 4c2e24c9bdc5f6c63c0251cdb72f175170d08755 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 16 Jan 2024 15:14:06 +0000
Subject: [PATCH 138/188] tbv2: support capture-thrift-isset

Support the capture-thrift-isset feature with TreeBuilder-v2. Fairly minor
changes here except the type of the Enum in a template parameter now matters.

We follow the previous behaviour of capturing a value for each field in a
struct that has an `isset_bitset`. This value is a VarInt captured before the
C++ contents of the member. It has 3 values: 0 (not set), 1 (set), and 2
(unavailable). These are handled by the processor and represented in the output
as `false`, `true`, and `std::nullopt_t` respectively.

Changes:
- Add a simple Thrift isset processor before any fields that have Thrift isset.
- Store the fully qualified names of enum types in DrgnParser - it already
  worked out this information anyway for naming the values and this is
  consistent with classes.
- Forward all enum template parameters under their input name under the
  assumption that they will all be policy type things like `IssetBitsetOption`.
  This could turn out to be wrong.

Test plan:
- CI (doesn't test thrift changes but covers other regressions)
- Updated Thrift enum tests for new format.
- `FILTER='OilIntegration.*' make test` - Thrift tests failed before, succeed
  after.
---
 include/oi/exporters/Json.h                |  2 +-
 oi/CodeGen.cpp                             | 81 ++++++++++++++++---
 oi/OITraceCode.cpp                         | 19 +++++
 oi/type_graph/ClangTypeParser.cpp          |  5 +-
 oi/type_graph/DrgnParser.cpp               | 34 +++++---
 oi/type_graph/Types.h                      | 10 ++-
 test/integration/enums.toml                |  6 +-
 test/integration/thrift_isset.toml         | 92 ++++++++++++++++++++--
 test/integration/thrift_isset_missing.toml | 51 +++++++++++-
 9 files changed, 262 insertions(+), 38 deletions(-)

diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h
index 3dd37f4..3416ef8 100644
--- a/include/oi/exporters/Json.h
+++ b/include/oi/exporters/Json.h
@@ -158,7 +158,7 @@ inline void Json::printFields(const result::Element& el,
     printUnsignedField("capacity", el.container_stats->capacity, indent);
   }
   if (el.is_set_stats.has_value())
-    printUnsignedField("is_set", el.is_set_stats->is_set, indent);
+    printBoolField("is_set", el.is_set_stats->is_set, indent);
   printBoolField("is_primitive", el.is_primitive, indent);
 }
 
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index cb5bf54..94fea05 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -687,7 +687,9 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
     }
 
     if (thriftIssetMember != nullptr && thriftIssetMember != &member) {
-      code += "\n  .write(getThriftIsset(t, " + std::to_string(i) + "))";
+      code += "\n      .write(getThriftIsset(t, ";
+      code += std::to_string(i);
+      code += "))";
     }
 
     code += "\n      .";
@@ -765,6 +767,12 @@ void CodeGen::genClassStaticType(const Class& c, std::string& code) {
 
 void CodeGen::genClassTreeBuilderInstructions(const Class& c,
                                               std::string& code) {
+  const Member* thriftIssetMember = nullptr;
+  if (const auto it = thriftIssetMembers_.find(&c);
+      it != thriftIssetMembers_.end()) {
+    thriftIssetMember = it->second;
+  }
+
   code += " private:\n";
   size_t index = 0;
   for (const auto& m : c.members) {
@@ -799,12 +807,32 @@ void CodeGen::genClassTreeBuilderInstructions(const Class& c,
       continue;
     std::string fullName = c.name() + "::" + m.name;
     bool isPrimitive = dynamic_cast<const Primitive*>(&m.type());
-    code += "      inst::Field{sizeof(" + fullName + "), " +
-            std::to_string(calculateExclusiveSize(m.type())) + ",\"" +
-            m.inputName + "\", member_" + std::to_string(index) +
-            "_type_names, TypeHandler<Ctx, decltype(" + fullName +
-            ")>::fields, TypeHandler<Ctx, decltype(" + fullName +
-            ")>::processors, ";
+    code += "      inst::Field{sizeof(";
+    code += fullName;
+    code += "), ";
+    code += std::to_string(calculateExclusiveSize(m.type()));
+    code += ",\"";
+    code += m.inputName;
+    code += "\", member_";
+    code += std::to_string(index);
+    code += "_type_names, TypeHandler<Ctx, decltype(";
+    code += fullName;
+    code += ")>::fields, ";
+
+    if (thriftIssetMember != nullptr && thriftIssetMember != &m) {
+      code += "ThriftIssetHandler<";
+      // code += "member_";
+      // code += std::to_string(index);
+      // code += "_processors";
+    }
+    code += "TypeHandler<Ctx, decltype(";
+    code += fullName;
+    code += ")>";
+    if (thriftIssetMember != nullptr && thriftIssetMember != &m) {
+      code += '>';
+    }
+    code += "::processors, ";
+
     code += isPrimitive ? "true" : "false";
     code += "},\n";
   }
@@ -825,10 +853,10 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) {
   static int getThriftIsset(const %1%& t, size_t i) {
     using thrift_data = apache::thrift::TStructDataStorage<%2%>;
 
-    if (&thrift_data::isset_indexes == nullptr) return -1;
+    if (&thrift_data::isset_indexes == nullptr) return 2;
 
     auto idx = thrift_data::isset_indexes[i];
-    if (idx == -1) return -1;
+    if (idx == -1) return 2;
 
     return t.%3%.get(idx);
   }
@@ -900,7 +928,15 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   for (const auto& p : templateParams) {
     if (p.value) {
       code += ", ";
-      code += p.type().name();
+
+      // HACK: forward all enums directly. this might turn out to be a problem
+      // if there are enums we should be regenerating/use in the body.
+      if (const auto* e = dynamic_cast<const Enum*>(&p.type())) {
+        code += e->inputName();
+      } else {
+        code += p.type().name();
+      }
+
       code += " N" + std::to_string(values++);
     } else {
       code += ", typename T" + std::to_string(types++);
@@ -1048,11 +1084,32 @@ void addCaptureKeySupport(std::string& code) {
   )";
 }
 
+void addThriftIssetSupport(std::string& code) {
+  code += R"(
+void processThriftIsset(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
+  auto v = std::get<ParsedData::VarInt>(d.val).value;
+  if (v <= 1) {
+    el.is_set_stats.emplace(result::Element::IsSetStats { v == 1 });
+  }
+}
+static constexpr exporters::inst::ProcessorInst thriftIssetProcessor{
+  types::st::VarInt<int>::describe,
+  &processThriftIsset,
+};
+
+template <typename Handler>
+struct ThriftIssetHandler {
+  static constexpr auto processors = arrayPrepend(Handler::processors, thriftIssetProcessor);
+};
+)";
+}
+
 void addStandardTypeHandlers(TypeGraph& typeGraph,
                              FeatureSet features,
                              std::string& code) {
-  if (features[Feature::TreeBuilderV2])
-    addCaptureKeySupport(code);
+  addCaptureKeySupport(code);
+  if (features[Feature::CaptureThriftIsset])
+    addThriftIssetSupport(code);
 
   // Provide a wrapper function, getSizeType, to infer T instead of having to
   // explicitly specify it with TypeHandler<Ctx, T>::getSizeType every time.
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index 323c83e..e430603 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -40,6 +40,11 @@ constexpr int oidMagicId = 0x01DE8;
 
 namespace {
 
+template <typename T, size_t N>
+constexpr std::array<T, N + 1> arrayPrepend(std::array<T, N> a, T t);
+template <typename T, size_t N, size_t... I>
+constexpr std::array<T, N+1> arrayPrependHelper(std::array<T, N> a, T t, std::index_sequence<I...>);
+
 template <size_t Size = (1 << 20) / sizeof(uintptr_t)>
 class PointerHashSet {
  private:
@@ -182,6 +187,20 @@ bool isStorageInline(const auto& c) {
          (uintptr_t)std::data(c) >= (uintptr_t)&c;
 }
 
+namespace {
+
+template <typename T, size_t N, size_t... I>
+constexpr std::array<T, N+1> arrayPrependHelper(std::array<T, N> a, T t, std::index_sequence<I...>) {
+  return {t, a[I]...};
+}
+
+template <typename T, size_t N>
+constexpr std::array<T, N + 1> arrayPrepend(std::array<T, N> a, T t) {
+  return arrayPrependHelper(a, t, std::make_index_sequence<N>());
+}
+
+}
+
 namespace OIInternal {
 namespace {
 
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index 24d5835..cb33377 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -137,6 +137,8 @@ Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
 }
 
 Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
   std::string name = ty.getDecl()->getNameAsString();
   auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
 
@@ -148,7 +150,8 @@ Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
     }
   }
 
-  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+  return makeType<Enum>(
+      ty, std::move(name), std::move(fqName), size, std::move(enumeratorMap));
 }
 
 Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
diff --git a/oi/type_graph/DrgnParser.cpp b/oi/type_graph/DrgnParser.cpp
index 0116edf..8a7070d 100644
--- a/oi/type_graph/DrgnParser.cpp
+++ b/oi/type_graph/DrgnParser.cpp
@@ -73,6 +73,8 @@ void warnForDrgnError(struct drgn_type* type,
                       const std::string& msg,
                       struct drgn_error* err);
 
+std::string getDrgnFullyQualifiedName(struct drgn_type* type);
+
 }  // namespace
 
 Type& DrgnParser::parse(struct drgn_type* root) {
@@ -150,6 +152,8 @@ Class& DrgnParser::enumerateClass(struct drgn_type* type) {
   std::string fqName;
   char* nameStr = nullptr;
   size_t length = 0;
+  // HACK: Leak this error. Freeing it causes a SEGV which suggests our
+  // underlying implementation is bad.
   auto* err = drgn_type_fully_qualified_name(type, &nameStr, &length);
   if (err == nullptr && nameStr != nullptr) {
     fqName = nameStr;
@@ -307,14 +311,7 @@ void DrgnParser::enumerateTemplateParam(struct drgn_type* type,
     // This template parameter is a value
     std::string value;
 
-    if (drgn_type_kind(obj->type) == DRGN_TYPE_ENUM) {
-      char* nameStr = nullptr;
-      size_t length = 0;
-      err = drgn_type_fully_qualified_name(obj->type, &nameStr, &length);
-      if (err != nullptr || nameStr == nullptr) {
-        throw DrgnParserError{"Failed to get enum's fully qualified name", err};
-      }
-
+    if (dynamic_cast<const Enum*>(&ttype)) {
       uint64_t enumVal;
       switch (obj->encoding) {
         case DRGN_OBJECT_ENCODING_SIGNED:
@@ -333,7 +330,9 @@ void DrgnParser::enumerateTemplateParam(struct drgn_type* type,
       size_t numEnumerators = drgn_type_num_enumerators(obj->type);
       for (size_t j = 0; j < numEnumerators; j++) {
         if (enumerators[j].uvalue == enumVal) {
-          value = std::string{nameStr} + "::" + enumerators[j].name;
+          value = ttype.inputName();
+          value += "::";
+          value += enumerators[j].name;
           break;
         }
       }
@@ -416,6 +415,8 @@ void DrgnParser::enumerateClassFunctions(struct drgn_type* type,
 }
 
 Enum& DrgnParser::enumerateEnum(struct drgn_type* type) {
+  std::string fqName = getDrgnFullyQualifiedName(type);
+
   const char* typeTag = drgn_type_tag(type);
   std::string name = typeTag ? typeTag : "";
   uint64_t size = get_drgn_type_size(type);
@@ -432,7 +433,8 @@ Enum& DrgnParser::enumerateEnum(struct drgn_type* type) {
     }
   }
 
-  return makeType<Enum>(type, name, size, std::move(enumeratorMap));
+  return makeType<Enum>(
+      type, std::move(name), std::move(fqName), size, std::move(enumeratorMap));
 }
 
 Typedef& DrgnParser::enumerateTypedef(struct drgn_type* type) {
@@ -528,6 +530,18 @@ void warnForDrgnError(struct drgn_type* type,
                << ": " << err->code << " " << err->message;
   drgn_error_destroy(err);
 }
+
+std::string getDrgnFullyQualifiedName(drgn_type* type) {
+  char* nameStr = nullptr;
+  size_t length = 0;
+  auto* err = drgn_type_fully_qualified_name(type, &nameStr, &length);
+  if (err != nullptr)
+    throw DrgnParserError("failed to get fully qualified name!", err);
+  if (nameStr != nullptr)
+    return nameStr;
+  return {};
+}
+
 }  // namespace
 
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f09f926..45b7d06 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -477,14 +477,20 @@ class Container : public Type {
 class Enum : public Type {
  public:
   explicit Enum(std::string name,
+                std::string inputName,
                 size_t size,
                 std::map<int64_t, std::string> enumerators = {})
-      : name_(name),
-        inputName_(std::move(name)),
+      : name_(std::move(name)),
+        inputName_(std::move(inputName)),
         size_(size),
         enumerators_(std::move(enumerators)) {
   }
 
+  explicit Enum(std::string name,
+                size_t size,
+                std::map<int64_t, std::string> enumerators = {})
+      : Enum{name, std::move(name), size, std::move(enumerators)} {};
+
   static inline constexpr bool has_node_id = false;
 
   DECLARE_ACCEPT
diff --git a/test/integration/enums.toml b/test/integration/enums.toml
index fa91cbc..f5274a3 100644
--- a/test/integration/enums.toml
+++ b/test/integration/enums.toml
@@ -36,12 +36,12 @@ definitions = '''
     param_types = ["ScopedEnum"]
     setup = "return {};"
     expect_json = '[{"staticSize":4, "dynamicSize":0}]'
-    expect_json_v2 = '[{"typeNames": ["ScopedEnum"], "staticSize":4, "exclusiveSize":4, "size":4}]'
+    expect_json_v2 = '[{"typeNames": ["ns_enums::ScopedEnum"], "staticSize":4, "exclusiveSize":4, "size":4}]'
   [cases.scoped_uint8]
     param_types = ["ScopedEnumUint8"]
     setup = "return {};"
     expect_json = '[{"staticSize":1, "dynamicSize":0}]'
-    expect_json_v2 = '[{"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1}]'
+    expect_json_v2 = '[{"typeNames": ["ns_enums::ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1}]'
   [cases.unscoped]
     param_types = ["UNSCOPED_ENUM"]
     setup = "return {};"
@@ -60,7 +60,7 @@ definitions = '''
     expect_json = '[{"staticSize":2, "dynamicSize":0}]'
     expect_json_v2 = '''[
       {"staticSize": 2, "exclusiveSize": 0, "size":2, "members": [
-        {"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1},
+        {"typeNames": ["ns_enums::ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1},
         {"typeNames": ["uint8_t"], "staticSize":1, "exclusiveSize":1, "size":1}
       ]}
     ]'''
diff --git a/test/integration/thrift_isset.toml b/test/integration/thrift_isset.toml
index 98e19fd..cdbcdb5 100644
--- a/test/integration/thrift_isset.toml
+++ b/test/integration/thrift_isset.toml
@@ -77,7 +77,6 @@ namespace cpp2 {
 
 [cases]
   [cases.unpacked]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructUnpacked&"]
     setup = '''
       cpp2::MyThriftStructUnpacked ret;
@@ -95,9 +94,19 @@ namespace cpp2 {
         {"name":"__fbthrift_field_c", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize":1,
+      "size":16,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]
+    }]'''
 
   [cases.packed]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructPacked&"]
     setup = '''
       cpp2::MyThriftStructPacked ret;
@@ -126,9 +135,25 @@ namespace cpp2 {
         {"name":"__fbthrift_field_j", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":2}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":44,
+      "exclusiveSize":2,
+      "size":44,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_d", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_e", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_f", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_g", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_h", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_i", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_j", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":2, "NOT":"is_set"}
+      ]}]'''
 
   [cases.packed_non_atomic]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructPackedNonAtomic&"]
     setup = '''
       cpp2::MyThriftStructPackedNonAtomic ret;
@@ -147,9 +172,19 @@ namespace cpp2 {
         {"name":"__fbthrift_field_d", "staticSize":4, "isset":false},
         {"name":"__isset", "staticSize":1}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":20,
+      "exclusiveSize":3,
+      "size":20,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_d", "staticSize":4, "is_set":false},
+        {"name":"__isset", "staticSize":1, "NOT":"is_set"}
+      ]}]'''
 
   [cases.out_of_order]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructOutOfOrder&"]
     setup = '''
       cpp2::MyThriftStructOutOfOrder ret;
@@ -166,9 +201,18 @@ namespace cpp2 {
         {"name":"__fbthrift_field_c", "staticSize":4, "isset":false},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize":1,
+      "size":16,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":false},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
 
   [cases.required]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructRequired&"]
     setup = '''
       cpp2::MyThriftStructRequired ret;
@@ -189,9 +233,21 @@ namespace cpp2 {
         {"name":"__fbthrift_field_f", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":28,
+      "exclusiveSize":1,
+      "size":28,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_d", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_e", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_f", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
 
   [cases.box]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructBoxed&"]
     setup = '''
       cpp2::MyThriftStructBoxed ret;
@@ -211,6 +267,18 @@ namespace cpp2 {
         {"name":"__fbthrift_field_e", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":32,
+      "exclusiveSize":1,
+      "size":32,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":8, "NOT":"is_set"},
+        {"name":"__fbthrift_field_b", "staticSize":8, "NOT":"is_set"},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_d", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_e", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
 
   [cases.no_capture]
     param_types = ["const cpp2::MyThriftStructBoxed&"]
@@ -231,3 +299,15 @@ namespace cpp2 {
         {"name":"__fbthrift_field_e", "staticSize":4, "NOT":"isset"},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":32,
+      "exclusiveSize":1,
+      "size":32,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":8, "NOT":"is_set"},
+        {"name":"__fbthrift_field_b", "staticSize":8, "NOT":"is_set"},
+        {"name":"__fbthrift_field_c", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_d", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_e", "staticSize":4, "NOT":"is_set"},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
diff --git a/test/integration/thrift_isset_missing.toml b/test/integration/thrift_isset_missing.toml
index 3e7ad94..311f736 100644
--- a/test/integration/thrift_isset_missing.toml
+++ b/test/integration/thrift_isset_missing.toml
@@ -58,7 +58,6 @@ raw_definitions = '''
 '''
 [cases]
   [cases.present]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const FakeThriftWithData&"]
     setup = '''
       FakeThriftWithData ret;
@@ -78,8 +77,17 @@ raw_definitions = '''
         {"name":"__fbthrift_field_c", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize":1,
+      "size":16,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
   [cases.missing]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const FakeThriftWithoutData&"]
     setup = '''
       FakeThriftWithoutData ret;
@@ -99,8 +107,17 @@ raw_definitions = '''
         {"name":"__fbthrift_field_c", "staticSize":4, "NOT":"isset"},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize":1,
+      "size":16,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_b", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_c", "staticSize":4, "NOT":"is_set"},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
   [cases.mixed]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const Mixed&"]
     setup = '''
       Mixed ret;
@@ -143,3 +160,31 @@ raw_definitions = '''
           ]
         }
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":32,
+      "exclusiveSize":0,
+      "size":32,
+      "members":[
+        {
+          "staticSize":16,
+          "exclusiveSize":1,
+          "size":16,
+          "members":[
+            {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+            {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+            {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+            {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+          ]
+        },
+        {
+          "staticSize":16,
+          "exclusiveSize":1,
+          "size":16,
+          "members":[
+            {"name":"__fbthrift_field_a", "staticSize":4, "NOT":"is_set"},
+            {"name":"__fbthrift_field_b", "staticSize":4, "NOT":"is_set"},
+            {"name":"__fbthrift_field_c", "staticSize":4, "NOT":"is_set"},
+            {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+          ]
+        }
+      ]}]'''

From 5c8d0cf9d6df0b8a88a7a73a2b1cc81641ff1514 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 16 Jan 2024 15:14:06 +0000
Subject: [PATCH 139/188] tbv2: support capture-thrift-isset

Support the capture-thrift-isset feature with TreeBuilder-v2. Fairly minor
changes here except the type of the Enum in a template parameter now matters.

We follow the previous behaviour of capturing a value for each field in a
struct that has an `isset_bitset`. This value is a VarInt captured before the
C++ contents of the member. It has 3 values: 0 (not set), 1 (set), and 2
(unavailable). These are handled by the processor and represented in the output
as `false`, `true`, and `std::nullopt_t` respectively.

Changes:
- Add a simple Thrift isset processor before any fields that have Thrift isset.
- Store the fully qualified names of enum types in DrgnParser - it already
  worked out this information anyway for naming the values and this is
  consistent with classes.
- Forward all enum template parameters under their input name under the
  assumption that they will all be policy type things like `IssetBitsetOption`.
  This could turn out to be wrong.

Test plan:
- CI (doesn't test thrift changes but covers other regressions)
- Updated Thrift enum tests for new format.
- `FILTER='OilIntegration.*' make test` - Thrift tests failed before, succeed
  after.
---
 include/oi/exporters/Json.h                |  2 +-
 oi/CodeGen.cpp                             | 81 ++++++++++++++++---
 oi/OITraceCode.cpp                         | 23 ++++++
 oi/type_graph/ClangTypeParser.cpp          |  5 +-
 oi/type_graph/DrgnParser.cpp               | 34 +++++---
 oi/type_graph/Types.h                      | 10 ++-
 test/integration/enums.toml                |  6 +-
 test/integration/thrift_isset.toml         | 92 ++++++++++++++++++++--
 test/integration/thrift_isset_missing.toml | 51 +++++++++++-
 9 files changed, 266 insertions(+), 38 deletions(-)

diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h
index 3dd37f4..3416ef8 100644
--- a/include/oi/exporters/Json.h
+++ b/include/oi/exporters/Json.h
@@ -158,7 +158,7 @@ inline void Json::printFields(const result::Element& el,
     printUnsignedField("capacity", el.container_stats->capacity, indent);
   }
   if (el.is_set_stats.has_value())
-    printUnsignedField("is_set", el.is_set_stats->is_set, indent);
+    printBoolField("is_set", el.is_set_stats->is_set, indent);
   printBoolField("is_primitive", el.is_primitive, indent);
 }
 
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index cb5bf54..94fea05 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -687,7 +687,9 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
     }
 
     if (thriftIssetMember != nullptr && thriftIssetMember != &member) {
-      code += "\n  .write(getThriftIsset(t, " + std::to_string(i) + "))";
+      code += "\n      .write(getThriftIsset(t, ";
+      code += std::to_string(i);
+      code += "))";
     }
 
     code += "\n      .";
@@ -765,6 +767,12 @@ void CodeGen::genClassStaticType(const Class& c, std::string& code) {
 
 void CodeGen::genClassTreeBuilderInstructions(const Class& c,
                                               std::string& code) {
+  const Member* thriftIssetMember = nullptr;
+  if (const auto it = thriftIssetMembers_.find(&c);
+      it != thriftIssetMembers_.end()) {
+    thriftIssetMember = it->second;
+  }
+
   code += " private:\n";
   size_t index = 0;
   for (const auto& m : c.members) {
@@ -799,12 +807,32 @@ void CodeGen::genClassTreeBuilderInstructions(const Class& c,
       continue;
     std::string fullName = c.name() + "::" + m.name;
     bool isPrimitive = dynamic_cast<const Primitive*>(&m.type());
-    code += "      inst::Field{sizeof(" + fullName + "), " +
-            std::to_string(calculateExclusiveSize(m.type())) + ",\"" +
-            m.inputName + "\", member_" + std::to_string(index) +
-            "_type_names, TypeHandler<Ctx, decltype(" + fullName +
-            ")>::fields, TypeHandler<Ctx, decltype(" + fullName +
-            ")>::processors, ";
+    code += "      inst::Field{sizeof(";
+    code += fullName;
+    code += "), ";
+    code += std::to_string(calculateExclusiveSize(m.type()));
+    code += ",\"";
+    code += m.inputName;
+    code += "\", member_";
+    code += std::to_string(index);
+    code += "_type_names, TypeHandler<Ctx, decltype(";
+    code += fullName;
+    code += ")>::fields, ";
+
+    if (thriftIssetMember != nullptr && thriftIssetMember != &m) {
+      code += "ThriftIssetHandler<";
+      // code += "member_";
+      // code += std::to_string(index);
+      // code += "_processors";
+    }
+    code += "TypeHandler<Ctx, decltype(";
+    code += fullName;
+    code += ")>";
+    if (thriftIssetMember != nullptr && thriftIssetMember != &m) {
+      code += '>';
+    }
+    code += "::processors, ";
+
     code += isPrimitive ? "true" : "false";
     code += "},\n";
   }
@@ -825,10 +853,10 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) {
   static int getThriftIsset(const %1%& t, size_t i) {
     using thrift_data = apache::thrift::TStructDataStorage<%2%>;
 
-    if (&thrift_data::isset_indexes == nullptr) return -1;
+    if (&thrift_data::isset_indexes == nullptr) return 2;
 
     auto idx = thrift_data::isset_indexes[i];
-    if (idx == -1) return -1;
+    if (idx == -1) return 2;
 
     return t.%3%.get(idx);
   }
@@ -900,7 +928,15 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   for (const auto& p : templateParams) {
     if (p.value) {
       code += ", ";
-      code += p.type().name();
+
+      // HACK: forward all enums directly. this might turn out to be a problem
+      // if there are enums we should be regenerating/use in the body.
+      if (const auto* e = dynamic_cast<const Enum*>(&p.type())) {
+        code += e->inputName();
+      } else {
+        code += p.type().name();
+      }
+
       code += " N" + std::to_string(values++);
     } else {
       code += ", typename T" + std::to_string(types++);
@@ -1048,11 +1084,32 @@ void addCaptureKeySupport(std::string& code) {
   )";
 }
 
+void addThriftIssetSupport(std::string& code) {
+  code += R"(
+void processThriftIsset(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
+  auto v = std::get<ParsedData::VarInt>(d.val).value;
+  if (v <= 1) {
+    el.is_set_stats.emplace(result::Element::IsSetStats { v == 1 });
+  }
+}
+static constexpr exporters::inst::ProcessorInst thriftIssetProcessor{
+  types::st::VarInt<int>::describe,
+  &processThriftIsset,
+};
+
+template <typename Handler>
+struct ThriftIssetHandler {
+  static constexpr auto processors = arrayPrepend(Handler::processors, thriftIssetProcessor);
+};
+)";
+}
+
 void addStandardTypeHandlers(TypeGraph& typeGraph,
                              FeatureSet features,
                              std::string& code) {
-  if (features[Feature::TreeBuilderV2])
-    addCaptureKeySupport(code);
+  addCaptureKeySupport(code);
+  if (features[Feature::CaptureThriftIsset])
+    addThriftIssetSupport(code);
 
   // Provide a wrapper function, getSizeType, to infer T instead of having to
   // explicitly specify it with TypeHandler<Ctx, T>::getSizeType every time.
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index 323c83e..362ce69 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -40,6 +40,13 @@ constexpr int oidMagicId = 0x01DE8;
 
 namespace {
 
+template <typename T, size_t N>
+constexpr std::array<T, N + 1> arrayPrepend(std::array<T, N> a, T t);
+template <typename T, size_t N, size_t... I>
+constexpr std::array<T, N + 1> arrayPrependHelper(std::array<T, N> a,
+                                                  T t,
+                                                  std::index_sequence<I...>);
+
 template <size_t Size = (1 << 20) / sizeof(uintptr_t)>
 class PointerHashSet {
  private:
@@ -182,6 +189,22 @@ bool isStorageInline(const auto& c) {
          (uintptr_t)std::data(c) >= (uintptr_t)&c;
 }
 
+namespace {
+
+template <typename T, size_t N, size_t... I>
+constexpr std::array<T, N + 1> arrayPrependHelper(std::array<T, N> a,
+                                                  T t,
+                                                  std::index_sequence<I...>) {
+  return {t, a[I]...};
+}
+
+template <typename T, size_t N>
+constexpr std::array<T, N + 1> arrayPrepend(std::array<T, N> a, T t) {
+  return arrayPrependHelper(a, t, std::make_index_sequence<N>());
+}
+
+}  // namespace
+
 namespace OIInternal {
 namespace {
 
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index 24d5835..cb33377 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -137,6 +137,8 @@ Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
 }
 
 Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
   std::string name = ty.getDecl()->getNameAsString();
   auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
 
@@ -148,7 +150,8 @@ Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
     }
   }
 
-  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+  return makeType<Enum>(
+      ty, std::move(name), std::move(fqName), size, std::move(enumeratorMap));
 }
 
 Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
diff --git a/oi/type_graph/DrgnParser.cpp b/oi/type_graph/DrgnParser.cpp
index 0116edf..8a7070d 100644
--- a/oi/type_graph/DrgnParser.cpp
+++ b/oi/type_graph/DrgnParser.cpp
@@ -73,6 +73,8 @@ void warnForDrgnError(struct drgn_type* type,
                       const std::string& msg,
                       struct drgn_error* err);
 
+std::string getDrgnFullyQualifiedName(struct drgn_type* type);
+
 }  // namespace
 
 Type& DrgnParser::parse(struct drgn_type* root) {
@@ -150,6 +152,8 @@ Class& DrgnParser::enumerateClass(struct drgn_type* type) {
   std::string fqName;
   char* nameStr = nullptr;
   size_t length = 0;
+  // HACK: Leak this error. Freeing it causes a SEGV which suggests our
+  // underlying implementation is bad.
   auto* err = drgn_type_fully_qualified_name(type, &nameStr, &length);
   if (err == nullptr && nameStr != nullptr) {
     fqName = nameStr;
@@ -307,14 +311,7 @@ void DrgnParser::enumerateTemplateParam(struct drgn_type* type,
     // This template parameter is a value
     std::string value;
 
-    if (drgn_type_kind(obj->type) == DRGN_TYPE_ENUM) {
-      char* nameStr = nullptr;
-      size_t length = 0;
-      err = drgn_type_fully_qualified_name(obj->type, &nameStr, &length);
-      if (err != nullptr || nameStr == nullptr) {
-        throw DrgnParserError{"Failed to get enum's fully qualified name", err};
-      }
-
+    if (dynamic_cast<const Enum*>(&ttype)) {
       uint64_t enumVal;
       switch (obj->encoding) {
         case DRGN_OBJECT_ENCODING_SIGNED:
@@ -333,7 +330,9 @@ void DrgnParser::enumerateTemplateParam(struct drgn_type* type,
       size_t numEnumerators = drgn_type_num_enumerators(obj->type);
       for (size_t j = 0; j < numEnumerators; j++) {
         if (enumerators[j].uvalue == enumVal) {
-          value = std::string{nameStr} + "::" + enumerators[j].name;
+          value = ttype.inputName();
+          value += "::";
+          value += enumerators[j].name;
           break;
         }
       }
@@ -416,6 +415,8 @@ void DrgnParser::enumerateClassFunctions(struct drgn_type* type,
 }
 
 Enum& DrgnParser::enumerateEnum(struct drgn_type* type) {
+  std::string fqName = getDrgnFullyQualifiedName(type);
+
   const char* typeTag = drgn_type_tag(type);
   std::string name = typeTag ? typeTag : "";
   uint64_t size = get_drgn_type_size(type);
@@ -432,7 +433,8 @@ Enum& DrgnParser::enumerateEnum(struct drgn_type* type) {
     }
   }
 
-  return makeType<Enum>(type, name, size, std::move(enumeratorMap));
+  return makeType<Enum>(
+      type, std::move(name), std::move(fqName), size, std::move(enumeratorMap));
 }
 
 Typedef& DrgnParser::enumerateTypedef(struct drgn_type* type) {
@@ -528,6 +530,18 @@ void warnForDrgnError(struct drgn_type* type,
                << ": " << err->code << " " << err->message;
   drgn_error_destroy(err);
 }
+
+std::string getDrgnFullyQualifiedName(drgn_type* type) {
+  char* nameStr = nullptr;
+  size_t length = 0;
+  auto* err = drgn_type_fully_qualified_name(type, &nameStr, &length);
+  if (err != nullptr)
+    throw DrgnParserError("failed to get fully qualified name!", err);
+  if (nameStr != nullptr)
+    return nameStr;
+  return {};
+}
+
 }  // namespace
 
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f09f926..45b7d06 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -477,14 +477,20 @@ class Container : public Type {
 class Enum : public Type {
  public:
   explicit Enum(std::string name,
+                std::string inputName,
                 size_t size,
                 std::map<int64_t, std::string> enumerators = {})
-      : name_(name),
-        inputName_(std::move(name)),
+      : name_(std::move(name)),
+        inputName_(std::move(inputName)),
         size_(size),
         enumerators_(std::move(enumerators)) {
   }
 
+  explicit Enum(std::string name,
+                size_t size,
+                std::map<int64_t, std::string> enumerators = {})
+      : Enum{name, std::move(name), size, std::move(enumerators)} {};
+
   static inline constexpr bool has_node_id = false;
 
   DECLARE_ACCEPT
diff --git a/test/integration/enums.toml b/test/integration/enums.toml
index fa91cbc..f5274a3 100644
--- a/test/integration/enums.toml
+++ b/test/integration/enums.toml
@@ -36,12 +36,12 @@ definitions = '''
     param_types = ["ScopedEnum"]
     setup = "return {};"
     expect_json = '[{"staticSize":4, "dynamicSize":0}]'
-    expect_json_v2 = '[{"typeNames": ["ScopedEnum"], "staticSize":4, "exclusiveSize":4, "size":4}]'
+    expect_json_v2 = '[{"typeNames": ["ns_enums::ScopedEnum"], "staticSize":4, "exclusiveSize":4, "size":4}]'
   [cases.scoped_uint8]
     param_types = ["ScopedEnumUint8"]
     setup = "return {};"
     expect_json = '[{"staticSize":1, "dynamicSize":0}]'
-    expect_json_v2 = '[{"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1}]'
+    expect_json_v2 = '[{"typeNames": ["ns_enums::ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1}]'
   [cases.unscoped]
     param_types = ["UNSCOPED_ENUM"]
     setup = "return {};"
@@ -60,7 +60,7 @@ definitions = '''
     expect_json = '[{"staticSize":2, "dynamicSize":0}]'
     expect_json_v2 = '''[
       {"staticSize": 2, "exclusiveSize": 0, "size":2, "members": [
-        {"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1},
+        {"typeNames": ["ns_enums::ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1},
         {"typeNames": ["uint8_t"], "staticSize":1, "exclusiveSize":1, "size":1}
       ]}
     ]'''
diff --git a/test/integration/thrift_isset.toml b/test/integration/thrift_isset.toml
index 98e19fd..cdbcdb5 100644
--- a/test/integration/thrift_isset.toml
+++ b/test/integration/thrift_isset.toml
@@ -77,7 +77,6 @@ namespace cpp2 {
 
 [cases]
   [cases.unpacked]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructUnpacked&"]
     setup = '''
       cpp2::MyThriftStructUnpacked ret;
@@ -95,9 +94,19 @@ namespace cpp2 {
         {"name":"__fbthrift_field_c", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize":1,
+      "size":16,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]
+    }]'''
 
   [cases.packed]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructPacked&"]
     setup = '''
       cpp2::MyThriftStructPacked ret;
@@ -126,9 +135,25 @@ namespace cpp2 {
         {"name":"__fbthrift_field_j", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":2}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":44,
+      "exclusiveSize":2,
+      "size":44,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_d", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_e", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_f", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_g", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_h", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_i", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_j", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":2, "NOT":"is_set"}
+      ]}]'''
 
   [cases.packed_non_atomic]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructPackedNonAtomic&"]
     setup = '''
       cpp2::MyThriftStructPackedNonAtomic ret;
@@ -147,9 +172,19 @@ namespace cpp2 {
         {"name":"__fbthrift_field_d", "staticSize":4, "isset":false},
         {"name":"__isset", "staticSize":1}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":20,
+      "exclusiveSize":3,
+      "size":20,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_d", "staticSize":4, "is_set":false},
+        {"name":"__isset", "staticSize":1, "NOT":"is_set"}
+      ]}]'''
 
   [cases.out_of_order]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructOutOfOrder&"]
     setup = '''
       cpp2::MyThriftStructOutOfOrder ret;
@@ -166,9 +201,18 @@ namespace cpp2 {
         {"name":"__fbthrift_field_c", "staticSize":4, "isset":false},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize":1,
+      "size":16,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":false},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
 
   [cases.required]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructRequired&"]
     setup = '''
       cpp2::MyThriftStructRequired ret;
@@ -189,9 +233,21 @@ namespace cpp2 {
         {"name":"__fbthrift_field_f", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":28,
+      "exclusiveSize":1,
+      "size":28,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_d", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_e", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_f", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
 
   [cases.box]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructBoxed&"]
     setup = '''
       cpp2::MyThriftStructBoxed ret;
@@ -211,6 +267,18 @@ namespace cpp2 {
         {"name":"__fbthrift_field_e", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":32,
+      "exclusiveSize":1,
+      "size":32,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":8, "NOT":"is_set"},
+        {"name":"__fbthrift_field_b", "staticSize":8, "NOT":"is_set"},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_d", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_e", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
 
   [cases.no_capture]
     param_types = ["const cpp2::MyThriftStructBoxed&"]
@@ -231,3 +299,15 @@ namespace cpp2 {
         {"name":"__fbthrift_field_e", "staticSize":4, "NOT":"isset"},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":32,
+      "exclusiveSize":1,
+      "size":32,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":8, "NOT":"is_set"},
+        {"name":"__fbthrift_field_b", "staticSize":8, "NOT":"is_set"},
+        {"name":"__fbthrift_field_c", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_d", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_e", "staticSize":4, "NOT":"is_set"},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
diff --git a/test/integration/thrift_isset_missing.toml b/test/integration/thrift_isset_missing.toml
index 3e7ad94..311f736 100644
--- a/test/integration/thrift_isset_missing.toml
+++ b/test/integration/thrift_isset_missing.toml
@@ -58,7 +58,6 @@ raw_definitions = '''
 '''
 [cases]
   [cases.present]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const FakeThriftWithData&"]
     setup = '''
       FakeThriftWithData ret;
@@ -78,8 +77,17 @@ raw_definitions = '''
         {"name":"__fbthrift_field_c", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize":1,
+      "size":16,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
   [cases.missing]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const FakeThriftWithoutData&"]
     setup = '''
       FakeThriftWithoutData ret;
@@ -99,8 +107,17 @@ raw_definitions = '''
         {"name":"__fbthrift_field_c", "staticSize":4, "NOT":"isset"},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize":1,
+      "size":16,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_b", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_c", "staticSize":4, "NOT":"is_set"},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
   [cases.mixed]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const Mixed&"]
     setup = '''
       Mixed ret;
@@ -143,3 +160,31 @@ raw_definitions = '''
           ]
         }
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":32,
+      "exclusiveSize":0,
+      "size":32,
+      "members":[
+        {
+          "staticSize":16,
+          "exclusiveSize":1,
+          "size":16,
+          "members":[
+            {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+            {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+            {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+            {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+          ]
+        },
+        {
+          "staticSize":16,
+          "exclusiveSize":1,
+          "size":16,
+          "members":[
+            {"name":"__fbthrift_field_a", "staticSize":4, "NOT":"is_set"},
+            {"name":"__fbthrift_field_b", "staticSize":4, "NOT":"is_set"},
+            {"name":"__fbthrift_field_c", "staticSize":4, "NOT":"is_set"},
+            {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+          ]
+        }
+      ]}]'''

From 3d5bbe17d3f8b1eb5196a65c5d90649d69232713 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 16 Jan 2024 16:23:52 +0000
Subject: [PATCH 140/188] tbv2: support capture-thrift-isset

Support the capture-thrift-isset feature with TreeBuilder-v2. Fairly minor
changes here except the type of the Enum in a template parameter now matters.

We follow the previous behaviour of capturing a value for each field in a
struct that has an `isset_bitset`. This value is a VarInt captured before the
C++ contents of the member. It has 3 values: 0 (not set), 1 (set), and 2
(unavailable). These are handled by the processor and represented in the output
as `false`, `true`, and `std::nullopt_t` respectively.

Changes:
- Add a simple Thrift isset processor before any fields that have Thrift isset.
- Store the fully qualified names of enum types in DrgnParser - it already
  worked out this information anyway for naming the values and this is
  consistent with classes.
- Forward all enum template parameters under their input name under the
  assumption that they will all be policy type things like `IssetBitsetOption`.
  This could turn out to be wrong.

Test plan:
- CI (doesn't test thrift changes but covers other regressions)
- Updated Thrift enum tests for new format.
- `FILTER='OilIntegration.*' make test` - Thrift tests failed before, succeed
  after.
---
 include/oi/exporters/Json.h                |  2 +-
 oi/CodeGen.cpp                             | 81 ++++++++++++++++---
 oi/OITraceCode.cpp                         | 23 ++++++
 oi/type_graph/ClangTypeParser.cpp          |  5 +-
 oi/type_graph/DrgnParser.cpp               | 34 +++++---
 oi/type_graph/Types.h                      | 10 ++-
 test/integration/enums.toml                |  6 +-
 test/integration/thrift_isset.toml         | 92 ++++++++++++++++++++--
 test/integration/thrift_isset_missing.toml | 51 +++++++++++-
 9 files changed, 266 insertions(+), 38 deletions(-)

diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h
index 3dd37f4..3416ef8 100644
--- a/include/oi/exporters/Json.h
+++ b/include/oi/exporters/Json.h
@@ -158,7 +158,7 @@ inline void Json::printFields(const result::Element& el,
     printUnsignedField("capacity", el.container_stats->capacity, indent);
   }
   if (el.is_set_stats.has_value())
-    printUnsignedField("is_set", el.is_set_stats->is_set, indent);
+    printBoolField("is_set", el.is_set_stats->is_set, indent);
   printBoolField("is_primitive", el.is_primitive, indent);
 }
 
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index cb5bf54..94fea05 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -687,7 +687,9 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
     }
 
     if (thriftIssetMember != nullptr && thriftIssetMember != &member) {
-      code += "\n  .write(getThriftIsset(t, " + std::to_string(i) + "))";
+      code += "\n      .write(getThriftIsset(t, ";
+      code += std::to_string(i);
+      code += "))";
     }
 
     code += "\n      .";
@@ -765,6 +767,12 @@ void CodeGen::genClassStaticType(const Class& c, std::string& code) {
 
 void CodeGen::genClassTreeBuilderInstructions(const Class& c,
                                               std::string& code) {
+  const Member* thriftIssetMember = nullptr;
+  if (const auto it = thriftIssetMembers_.find(&c);
+      it != thriftIssetMembers_.end()) {
+    thriftIssetMember = it->second;
+  }
+
   code += " private:\n";
   size_t index = 0;
   for (const auto& m : c.members) {
@@ -799,12 +807,32 @@ void CodeGen::genClassTreeBuilderInstructions(const Class& c,
       continue;
     std::string fullName = c.name() + "::" + m.name;
     bool isPrimitive = dynamic_cast<const Primitive*>(&m.type());
-    code += "      inst::Field{sizeof(" + fullName + "), " +
-            std::to_string(calculateExclusiveSize(m.type())) + ",\"" +
-            m.inputName + "\", member_" + std::to_string(index) +
-            "_type_names, TypeHandler<Ctx, decltype(" + fullName +
-            ")>::fields, TypeHandler<Ctx, decltype(" + fullName +
-            ")>::processors, ";
+    code += "      inst::Field{sizeof(";
+    code += fullName;
+    code += "), ";
+    code += std::to_string(calculateExclusiveSize(m.type()));
+    code += ",\"";
+    code += m.inputName;
+    code += "\", member_";
+    code += std::to_string(index);
+    code += "_type_names, TypeHandler<Ctx, decltype(";
+    code += fullName;
+    code += ")>::fields, ";
+
+    if (thriftIssetMember != nullptr && thriftIssetMember != &m) {
+      code += "ThriftIssetHandler<";
+      // code += "member_";
+      // code += std::to_string(index);
+      // code += "_processors";
+    }
+    code += "TypeHandler<Ctx, decltype(";
+    code += fullName;
+    code += ")>";
+    if (thriftIssetMember != nullptr && thriftIssetMember != &m) {
+      code += '>';
+    }
+    code += "::processors, ";
+
     code += isPrimitive ? "true" : "false";
     code += "},\n";
   }
@@ -825,10 +853,10 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) {
   static int getThriftIsset(const %1%& t, size_t i) {
     using thrift_data = apache::thrift::TStructDataStorage<%2%>;
 
-    if (&thrift_data::isset_indexes == nullptr) return -1;
+    if (&thrift_data::isset_indexes == nullptr) return 2;
 
     auto idx = thrift_data::isset_indexes[i];
-    if (idx == -1) return -1;
+    if (idx == -1) return 2;
 
     return t.%3%.get(idx);
   }
@@ -900,7 +928,15 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   for (const auto& p : templateParams) {
     if (p.value) {
       code += ", ";
-      code += p.type().name();
+
+      // HACK: forward all enums directly. this might turn out to be a problem
+      // if there are enums we should be regenerating/use in the body.
+      if (const auto* e = dynamic_cast<const Enum*>(&p.type())) {
+        code += e->inputName();
+      } else {
+        code += p.type().name();
+      }
+
       code += " N" + std::to_string(values++);
     } else {
       code += ", typename T" + std::to_string(types++);
@@ -1048,11 +1084,32 @@ void addCaptureKeySupport(std::string& code) {
   )";
 }
 
+void addThriftIssetSupport(std::string& code) {
+  code += R"(
+void processThriftIsset(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
+  auto v = std::get<ParsedData::VarInt>(d.val).value;
+  if (v <= 1) {
+    el.is_set_stats.emplace(result::Element::IsSetStats { v == 1 });
+  }
+}
+static constexpr exporters::inst::ProcessorInst thriftIssetProcessor{
+  types::st::VarInt<int>::describe,
+  &processThriftIsset,
+};
+
+template <typename Handler>
+struct ThriftIssetHandler {
+  static constexpr auto processors = arrayPrepend(Handler::processors, thriftIssetProcessor);
+};
+)";
+}
+
 void addStandardTypeHandlers(TypeGraph& typeGraph,
                              FeatureSet features,
                              std::string& code) {
-  if (features[Feature::TreeBuilderV2])
-    addCaptureKeySupport(code);
+  addCaptureKeySupport(code);
+  if (features[Feature::CaptureThriftIsset])
+    addThriftIssetSupport(code);
 
   // Provide a wrapper function, getSizeType, to infer T instead of having to
   // explicitly specify it with TypeHandler<Ctx, T>::getSizeType every time.
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index 323c83e..362ce69 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -40,6 +40,13 @@ constexpr int oidMagicId = 0x01DE8;
 
 namespace {
 
+template <typename T, size_t N>
+constexpr std::array<T, N + 1> arrayPrepend(std::array<T, N> a, T t);
+template <typename T, size_t N, size_t... I>
+constexpr std::array<T, N + 1> arrayPrependHelper(std::array<T, N> a,
+                                                  T t,
+                                                  std::index_sequence<I...>);
+
 template <size_t Size = (1 << 20) / sizeof(uintptr_t)>
 class PointerHashSet {
  private:
@@ -182,6 +189,22 @@ bool isStorageInline(const auto& c) {
          (uintptr_t)std::data(c) >= (uintptr_t)&c;
 }
 
+namespace {
+
+template <typename T, size_t N, size_t... I>
+constexpr std::array<T, N + 1> arrayPrependHelper(std::array<T, N> a,
+                                                  T t,
+                                                  std::index_sequence<I...>) {
+  return {t, a[I]...};
+}
+
+template <typename T, size_t N>
+constexpr std::array<T, N + 1> arrayPrepend(std::array<T, N> a, T t) {
+  return arrayPrependHelper(a, t, std::make_index_sequence<N>());
+}
+
+}  // namespace
+
 namespace OIInternal {
 namespace {
 
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index 24d5835..cb33377 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -137,6 +137,8 @@ Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
 }
 
 Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
   std::string name = ty.getDecl()->getNameAsString();
   auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
 
@@ -148,7 +150,8 @@ Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
     }
   }
 
-  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+  return makeType<Enum>(
+      ty, std::move(name), std::move(fqName), size, std::move(enumeratorMap));
 }
 
 Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
diff --git a/oi/type_graph/DrgnParser.cpp b/oi/type_graph/DrgnParser.cpp
index 0116edf..8a7070d 100644
--- a/oi/type_graph/DrgnParser.cpp
+++ b/oi/type_graph/DrgnParser.cpp
@@ -73,6 +73,8 @@ void warnForDrgnError(struct drgn_type* type,
                       const std::string& msg,
                       struct drgn_error* err);
 
+std::string getDrgnFullyQualifiedName(struct drgn_type* type);
+
 }  // namespace
 
 Type& DrgnParser::parse(struct drgn_type* root) {
@@ -150,6 +152,8 @@ Class& DrgnParser::enumerateClass(struct drgn_type* type) {
   std::string fqName;
   char* nameStr = nullptr;
   size_t length = 0;
+  // HACK: Leak this error. Freeing it causes a SEGV which suggests our
+  // underlying implementation is bad.
   auto* err = drgn_type_fully_qualified_name(type, &nameStr, &length);
   if (err == nullptr && nameStr != nullptr) {
     fqName = nameStr;
@@ -307,14 +311,7 @@ void DrgnParser::enumerateTemplateParam(struct drgn_type* type,
     // This template parameter is a value
     std::string value;
 
-    if (drgn_type_kind(obj->type) == DRGN_TYPE_ENUM) {
-      char* nameStr = nullptr;
-      size_t length = 0;
-      err = drgn_type_fully_qualified_name(obj->type, &nameStr, &length);
-      if (err != nullptr || nameStr == nullptr) {
-        throw DrgnParserError{"Failed to get enum's fully qualified name", err};
-      }
-
+    if (dynamic_cast<const Enum*>(&ttype)) {
       uint64_t enumVal;
       switch (obj->encoding) {
         case DRGN_OBJECT_ENCODING_SIGNED:
@@ -333,7 +330,9 @@ void DrgnParser::enumerateTemplateParam(struct drgn_type* type,
       size_t numEnumerators = drgn_type_num_enumerators(obj->type);
       for (size_t j = 0; j < numEnumerators; j++) {
         if (enumerators[j].uvalue == enumVal) {
-          value = std::string{nameStr} + "::" + enumerators[j].name;
+          value = ttype.inputName();
+          value += "::";
+          value += enumerators[j].name;
           break;
         }
       }
@@ -416,6 +415,8 @@ void DrgnParser::enumerateClassFunctions(struct drgn_type* type,
 }
 
 Enum& DrgnParser::enumerateEnum(struct drgn_type* type) {
+  std::string fqName = getDrgnFullyQualifiedName(type);
+
   const char* typeTag = drgn_type_tag(type);
   std::string name = typeTag ? typeTag : "";
   uint64_t size = get_drgn_type_size(type);
@@ -432,7 +433,8 @@ Enum& DrgnParser::enumerateEnum(struct drgn_type* type) {
     }
   }
 
-  return makeType<Enum>(type, name, size, std::move(enumeratorMap));
+  return makeType<Enum>(
+      type, std::move(name), std::move(fqName), size, std::move(enumeratorMap));
 }
 
 Typedef& DrgnParser::enumerateTypedef(struct drgn_type* type) {
@@ -528,6 +530,18 @@ void warnForDrgnError(struct drgn_type* type,
                << ": " << err->code << " " << err->message;
   drgn_error_destroy(err);
 }
+
+std::string getDrgnFullyQualifiedName(drgn_type* type) {
+  char* nameStr = nullptr;
+  size_t length = 0;
+  auto* err = drgn_type_fully_qualified_name(type, &nameStr, &length);
+  if (err != nullptr)
+    throw DrgnParserError("failed to get fully qualified name!", err);
+  if (nameStr != nullptr)
+    return nameStr;
+  return {};
+}
+
 }  // namespace
 
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f09f926..45b7d06 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -477,14 +477,20 @@ class Container : public Type {
 class Enum : public Type {
  public:
   explicit Enum(std::string name,
+                std::string inputName,
                 size_t size,
                 std::map<int64_t, std::string> enumerators = {})
-      : name_(name),
-        inputName_(std::move(name)),
+      : name_(std::move(name)),
+        inputName_(std::move(inputName)),
         size_(size),
         enumerators_(std::move(enumerators)) {
   }
 
+  explicit Enum(std::string name,
+                size_t size,
+                std::map<int64_t, std::string> enumerators = {})
+      : Enum{name, std::move(name), size, std::move(enumerators)} {};
+
   static inline constexpr bool has_node_id = false;
 
   DECLARE_ACCEPT
diff --git a/test/integration/enums.toml b/test/integration/enums.toml
index fa91cbc..f5274a3 100644
--- a/test/integration/enums.toml
+++ b/test/integration/enums.toml
@@ -36,12 +36,12 @@ definitions = '''
     param_types = ["ScopedEnum"]
     setup = "return {};"
     expect_json = '[{"staticSize":4, "dynamicSize":0}]'
-    expect_json_v2 = '[{"typeNames": ["ScopedEnum"], "staticSize":4, "exclusiveSize":4, "size":4}]'
+    expect_json_v2 = '[{"typeNames": ["ns_enums::ScopedEnum"], "staticSize":4, "exclusiveSize":4, "size":4}]'
   [cases.scoped_uint8]
     param_types = ["ScopedEnumUint8"]
     setup = "return {};"
     expect_json = '[{"staticSize":1, "dynamicSize":0}]'
-    expect_json_v2 = '[{"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1}]'
+    expect_json_v2 = '[{"typeNames": ["ns_enums::ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1}]'
   [cases.unscoped]
     param_types = ["UNSCOPED_ENUM"]
     setup = "return {};"
@@ -60,7 +60,7 @@ definitions = '''
     expect_json = '[{"staticSize":2, "dynamicSize":0}]'
     expect_json_v2 = '''[
       {"staticSize": 2, "exclusiveSize": 0, "size":2, "members": [
-        {"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1},
+        {"typeNames": ["ns_enums::ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1},
         {"typeNames": ["uint8_t"], "staticSize":1, "exclusiveSize":1, "size":1}
       ]}
     ]'''
diff --git a/test/integration/thrift_isset.toml b/test/integration/thrift_isset.toml
index 98e19fd..cdbcdb5 100644
--- a/test/integration/thrift_isset.toml
+++ b/test/integration/thrift_isset.toml
@@ -77,7 +77,6 @@ namespace cpp2 {
 
 [cases]
   [cases.unpacked]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructUnpacked&"]
     setup = '''
       cpp2::MyThriftStructUnpacked ret;
@@ -95,9 +94,19 @@ namespace cpp2 {
         {"name":"__fbthrift_field_c", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize":1,
+      "size":16,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]
+    }]'''
 
   [cases.packed]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructPacked&"]
     setup = '''
       cpp2::MyThriftStructPacked ret;
@@ -126,9 +135,25 @@ namespace cpp2 {
         {"name":"__fbthrift_field_j", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":2}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":44,
+      "exclusiveSize":2,
+      "size":44,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_d", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_e", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_f", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_g", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_h", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_i", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_j", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":2, "NOT":"is_set"}
+      ]}]'''
 
   [cases.packed_non_atomic]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructPackedNonAtomic&"]
     setup = '''
       cpp2::MyThriftStructPackedNonAtomic ret;
@@ -147,9 +172,19 @@ namespace cpp2 {
         {"name":"__fbthrift_field_d", "staticSize":4, "isset":false},
         {"name":"__isset", "staticSize":1}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":20,
+      "exclusiveSize":3,
+      "size":20,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_d", "staticSize":4, "is_set":false},
+        {"name":"__isset", "staticSize":1, "NOT":"is_set"}
+      ]}]'''
 
   [cases.out_of_order]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructOutOfOrder&"]
     setup = '''
       cpp2::MyThriftStructOutOfOrder ret;
@@ -166,9 +201,18 @@ namespace cpp2 {
         {"name":"__fbthrift_field_c", "staticSize":4, "isset":false},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize":1,
+      "size":16,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":false},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
 
   [cases.required]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructRequired&"]
     setup = '''
       cpp2::MyThriftStructRequired ret;
@@ -189,9 +233,21 @@ namespace cpp2 {
         {"name":"__fbthrift_field_f", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":28,
+      "exclusiveSize":1,
+      "size":28,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_d", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_e", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_f", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
 
   [cases.box]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructBoxed&"]
     setup = '''
       cpp2::MyThriftStructBoxed ret;
@@ -211,6 +267,18 @@ namespace cpp2 {
         {"name":"__fbthrift_field_e", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":32,
+      "exclusiveSize":1,
+      "size":32,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":8, "NOT":"is_set"},
+        {"name":"__fbthrift_field_b", "staticSize":8, "NOT":"is_set"},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_d", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_e", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
 
   [cases.no_capture]
     param_types = ["const cpp2::MyThriftStructBoxed&"]
@@ -231,3 +299,15 @@ namespace cpp2 {
         {"name":"__fbthrift_field_e", "staticSize":4, "NOT":"isset"},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":32,
+      "exclusiveSize":1,
+      "size":32,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":8, "NOT":"is_set"},
+        {"name":"__fbthrift_field_b", "staticSize":8, "NOT":"is_set"},
+        {"name":"__fbthrift_field_c", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_d", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_e", "staticSize":4, "NOT":"is_set"},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
diff --git a/test/integration/thrift_isset_missing.toml b/test/integration/thrift_isset_missing.toml
index 3e7ad94..311f736 100644
--- a/test/integration/thrift_isset_missing.toml
+++ b/test/integration/thrift_isset_missing.toml
@@ -58,7 +58,6 @@ raw_definitions = '''
 '''
 [cases]
   [cases.present]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const FakeThriftWithData&"]
     setup = '''
       FakeThriftWithData ret;
@@ -78,8 +77,17 @@ raw_definitions = '''
         {"name":"__fbthrift_field_c", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize":1,
+      "size":16,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
   [cases.missing]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const FakeThriftWithoutData&"]
     setup = '''
       FakeThriftWithoutData ret;
@@ -99,8 +107,17 @@ raw_definitions = '''
         {"name":"__fbthrift_field_c", "staticSize":4, "NOT":"isset"},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize":1,
+      "size":16,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_b", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_c", "staticSize":4, "NOT":"is_set"},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
   [cases.mixed]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const Mixed&"]
     setup = '''
       Mixed ret;
@@ -143,3 +160,31 @@ raw_definitions = '''
           ]
         }
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":32,
+      "exclusiveSize":0,
+      "size":32,
+      "members":[
+        {
+          "staticSize":16,
+          "exclusiveSize":1,
+          "size":16,
+          "members":[
+            {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+            {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+            {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+            {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+          ]
+        },
+        {
+          "staticSize":16,
+          "exclusiveSize":1,
+          "size":16,
+          "members":[
+            {"name":"__fbthrift_field_a", "staticSize":4, "NOT":"is_set"},
+            {"name":"__fbthrift_field_b", "staticSize":4, "NOT":"is_set"},
+            {"name":"__fbthrift_field_c", "staticSize":4, "NOT":"is_set"},
+            {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+          ]
+        }
+      ]}]'''

From 5585eee4b5ef8149882f559cc1b20473431b4128 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 16 Jan 2024 16:23:52 +0000
Subject: [PATCH 141/188] tbv2: fix pointer codegen

A previous change enabled running OIL tests with specific features enabled.
This highlighted that pointer code generation under TreeBuilder-v2 was very
broken. This change updates pointer code generation to work and enables the
skipped tests. All enabled tests need `expected_json_v2` added to them due to
formatting differences.

Reformatted and rewrote the basic type handler that handles primitives and
pointers. Removed the reliance on `features` to decide whether to generate for
TreeBuilder-v2 as the intermediate features have been removed.

There were a couple of other changes needed to enable these tests on TBv2 that
aren't worth their own issues and PRs, I sneaked them in here.

Extra changes:
- Added `Pointer` and `Reference` to TopoSorter so they generate
  `NameProvider` instances. It might be worth visiting the graph differently
  for `NameProvider` as it requires so many instances that others generators do
  not. Will consider that in the future.
- Follow typedefs when calculating exclusive size for a type.

Closes #458.

Test plan:
- CI
- Enabled previously disabled tests.
---
 oi/CodeGen.cpp                            |  35 ++--
 oi/FuncGen.cpp                            | 211 ++++++++++++----------
 oi/FuncGen.h                              |   2 +-
 oi/type_graph/TopoSorter.cpp              |  23 ++-
 oi/type_graph/TopoSorter.h                |   1 +
 test/integration/anonymous.toml           | 123 ++++++++++++-
 test/integration/pointers.toml            |  82 ++++++++-
 test/integration/pointers_incomplete.toml |  27 ++-
 test/test_topo_sorter.cpp                 |  18 ++
 9 files changed, 388 insertions(+), 134 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index cb5bf54..963410e 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -195,7 +195,12 @@ void genDecls(const TypeGraph& typeGraph, std::string& code) {
 namespace {
 
 size_t calculateExclusiveSize(const Type& t) {
-  if (const auto* c = dynamic_cast<const Class*>(&t)) {
+  const Type* finalType = &t;
+  while (const auto* td = dynamic_cast<const Typedef*>(finalType)) {
+    finalType = &td->underlyingType();
+  }
+
+  if (const auto* c = dynamic_cast<const Class*>(finalType)) {
     return std::accumulate(
         c->members.cbegin(), c->members.cend(), 0, [](size_t a, const auto& m) {
           if (m.name.starts_with(AddPadding::MemberPrefix))
@@ -203,7 +208,7 @@ size_t calculateExclusiveSize(const Type& t) {
           return a;
         });
   }
-  return t.size();
+  return finalType->size();
 }
 
 }  // namespace
@@ -211,7 +216,7 @@ size_t calculateExclusiveSize(const Type& t) {
 void genNames(const TypeGraph& typeGraph, std::string& code) {
   code += R"(
 template <typename T>
-struct NameProvider {};
+struct NameProvider;
 )";
 
   // TODO: stop types being duplicated at this point and remove this check
@@ -239,6 +244,9 @@ struct ExclusiveSizeProvider {
 )";
 
   for (const Type& t : typeGraph.finalTypes) {
+    if (dynamic_cast<const Typedef*>(&t))
+      continue;
+
     size_t exclusiveSize = calculateExclusiveSize(t);
     if (exclusiveSize != t.size()) {
       code += "template <> struct ExclusiveSizeProvider<";
@@ -1066,23 +1074,6 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
     }
 )";
 
-  if (features[Feature::TreeBuilderV2]) {
-    code += R"(
-template <typename Ctx, typename T>
-constexpr inst::Field make_field(std::string_view name) {
-  return inst::Field{
-    sizeof(T),
-    ExclusiveSizeProvider<T>::size,
-    name,
-    NameProvider<T>::names,
-    TypeHandler<Ctx, T>::fields,
-    TypeHandler<Ctx, T>::processors,
-    std::is_fundamental_v<T>
-  };
-}
-)";
-  }
-
   // TODO: bit of a hack - making ContainerInfo a node in the type graph and
   // traversing for it would remove the need for this set altogether.
   std::unordered_set<const ContainerInfo*> used{};
@@ -1264,9 +1255,6 @@ void CodeGen::generate(TypeGraph& typeGraph,
     code += "using namespace oi::detail;\n";
     code += "using oi::exporters::ParsedData;\n";
     code += "using namespace oi::exporters;\n";
-    code += "namespace OIInternal {\nnamespace {\n";
-    FuncGen::DefineBasicTypeHandlers(code, config_.features);
-    code += "} // namespace\n} // namespace OIInternal\n";
   }
 
   if (config_.features[Feature::CaptureThriftIsset]) {
@@ -1306,6 +1294,7 @@ void CodeGen::generate(TypeGraph& typeGraph,
   }
 
   if (config_.features[Feature::TreeBuilderV2]) {
+    FuncGen::DefineBasicTypeHandlers(code);
     addStandardTypeHandlers(typeGraph, config_.features, code);
     addTypeHandlers(typeGraph, code);
   } else {
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 5ac3abf..809ae14 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -614,110 +614,129 @@ class BackInserter {
  * pointer's value always, then the value of the pointer if it is unique. void
  * is of type Unit and always stores nothing.
  */
-void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
+void FuncGen::DefineBasicTypeHandlers(std::string& code) {
   code += R"(
-    template <typename Ctx, typename T>
-    struct TypeHandler {
-        using DB = typename Ctx::DataBuffer;
-      private:
-        static auto choose_type() {
-            if constexpr(std::is_pointer_v<T>) {
-                return std::type_identity<types::st::Pair<DB,
-                  types::st::VarInt<DB>,
-                  types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>
-                >>();
-            } else {
-                return std::type_identity<types::st::Unit<DB>>();
-            }
-        }
-
-      public:
-        using type = typename decltype(choose_type())::type;
+template <typename Ctx, typename T>
+struct TypeHandler;
 )";
-  if (features[Feature::TreeBuilderV2]) {
-    code += R"(private:
-        static void process_pointer(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
-          el.pointer = std::get<ParsedData::VarInt>(d.val).value;
-        }
-        static void process_pointer_content(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
-          static constexpr std::array<std::string_view, 1> names{"TODO"};
-          static constexpr auto childField = make_field<Ctx, T>("*");
 
-          const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
-
-          el.container_stats.emplace(result::Element::ContainerStats{ .capacity = 1 });
-
-          if (sum.index == 0)
-            return;
-
-          el.container_stats->length = 1;
-          stack_ins(childField);
-        }
-
-        static constexpr auto choose_fields() {
-          if constexpr(std::is_pointer_v<T>) {
-            return std::array<exporters::inst::Field, 0>{};
-          } else {
-            return std::array<exporters::inst::Field, 0>{};
-          }
-        }
-        static constexpr auto choose_processors() {
-          if constexpr(std::is_pointer_v<T>) {
-            return std::array<inst::ProcessorInst, 2>{
-              {types::st::VarInt<DB>::describe, &process_pointer},
-              {types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>::describe, &process_pointer_content},
-            };
-          } else {
-            return std::array<inst::ProcessorInst, 0>{};
-          }
-        }
-      public:
-        static constexpr auto fields = choose_fields();
-        static constexpr auto processors = choose_processors();
+  code += R"(
+template <typename Ctx, typename T>
+constexpr inst::Field make_field(std::string_view name) {
+  return inst::Field{
+      sizeof(T),
+      ExclusiveSizeProvider<T>::size,
+      name,
+      NameProvider<T>::names,
+      TypeHandler<Ctx, T>::fields,
+      TypeHandler<Ctx, T>::processors,
+      std::is_fundamental_v<T>,
+  };
+}
 )";
+  code += R"(
+template <typename Ctx, typename T>
+struct TypeHandler {
+  using DB = typename Ctx::DataBuffer;
+
+ private:
+  static void process_pointer(result::Element& el,
+                              std::function<void(inst::Inst)> stack_ins,
+                              ParsedData d) {
+    el.pointer = std::get<ParsedData::VarInt>(d.val).value;
   }
-  code += R"(
-        static types::st::Unit<DB> getSizeType(
-          Ctx& ctx,
-          const T& t,
-          typename TypeHandler<Ctx, T>::type returnArg) {
-              if constexpr(std::is_pointer_v<T>) {
-                JLOG("ptr val @");
-                JLOGPTR(t);
-                auto r0 = returnArg.write((uintptr_t)t);
-                if (t && ctx.pointers.add((uintptr_t)t)) {
-                  return r0.template delegate<1>([&t](auto ret) {
-                    if constexpr (!std::is_void<std::remove_pointer_t<T>>::value) {
-                      return TypeHandler<Ctx, std::remove_pointer_t<T>>::getSizeType(*t, ret);
-                    } else {
-                      return ret;
-                    }
-                  });
-                } else {
-                  return r0.template delegate<0>(std::identity());
-                }
-              } else {
-                return returnArg;
-              }
-        }
-    };
-  )";
+  static void process_pointer_content(result::Element& el,
+                                      std::function<void(inst::Inst)> stack_ins,
+                                      ParsedData d) {
+    using U = std::decay_t<std::remove_pointer_t<T>>;
+    const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
+
+    if constexpr (oi_is_complete<U>) {
+      static constexpr auto childField = make_field<Ctx, U>("*");
+
+      el.container_stats.emplace(result::Element::ContainerStats{ .capacity = 1, .length = 0 });
+
+      if (sum.index == 0)
+        return;
+
+      el.container_stats->length = 1;
+      stack_ins(childField);
+    }
+  }
+
+  static auto choose_type() {
+    if constexpr (std::is_pointer_v<T>) {
+      return std::type_identity<types::st::Pair<
+          DB,
+          types::st::VarInt<DB>,
+          types::st::Sum<
+              DB,
+              types::st::Unit<DB>,
+              typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>>>();
+    } else {
+      return std::type_identity<types::st::Unit<DB>>();
+    }
+  }
+  static constexpr auto choose_processors() {
+    if constexpr (std::is_pointer_v<T>) {
+      return std::array<inst::ProcessorInst, 2>{
+          exporters::inst::ProcessorInst{types::st::VarInt<DB>::describe,
+                                         &process_pointer},
+          exporters::inst::ProcessorInst{
+              types::st::Sum<
+                  DB,
+                  types::st::Unit<DB>,
+                  typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>::
+                  describe,
+              &process_pointer_content},
+      };
+    } else {
+      return std::array<inst::ProcessorInst, 0>{};
+    }
+  }
+
+ public:
+  using type = typename decltype(choose_type())::type;
+
+  static constexpr std::array<exporters::inst::Field, 0> fields{};
+  static constexpr auto processors = choose_processors();
+
+  static types::st::Unit<DB> getSizeType(
+      Ctx& ctx, const T& t, typename TypeHandler<Ctx, T>::type returnArg) {
+    if constexpr (std::is_pointer_v<T>) {
+      JLOG("ptr val @");
+      JLOGPTR(t);
+      auto r0 = returnArg.write((uintptr_t)t);
+      if (t && ctx.pointers.add((uintptr_t)t)) {
+        return r0.template delegate<1>([&ctx, &t](auto ret) {
+          using U = std::decay_t<std::remove_pointer_t<T>>;
+          if constexpr (oi_is_complete<U>) {
+            return TypeHandler<Ctx, U>::getSizeType(ctx, *t, ret);
+          } else {
+            return ret;
+          }
+        });
+      } else {
+        return r0.template delegate<0>(std::identity());
+      }
+    } else {
+      return returnArg;
+    }
+  }
+};
+)";
 
   code += R"(
-    template <typename Ctx>
-    class TypeHandler<Ctx, void> {
-        using DB = typename Ctx::DataBuffer;
-      public:
-        using type = types::st::Unit<DB>;
+template <typename Ctx>
+class TypeHandler<Ctx, void> {
+  using DB = typename Ctx::DataBuffer;
+
+ public:
+  using type = types::st::Unit<DB>;
+  static constexpr std::array<exporters::inst::Field, 0> fields{};
+  static constexpr std::array<exporters::inst::ProcessorInst, 0> processors{};
+};
 )";
-  if (features[Feature::TreeBuilderV2]) {
-    code +=
-        "static constexpr std::array<exporters::inst::Field, 0> fields{};\n";
-    code +=
-        "static constexpr std::array<exporters::inst::ProcessorInst, 0> "
-        "processors{};\n";
-  }
-  code += "};\n";
 }
 
 ContainerInfo FuncGen::GetOiArrayContainerInfo() {
diff --git a/oi/FuncGen.h b/oi/FuncGen.h
index 33aaf9c..e2b2fba 100644
--- a/oi/FuncGen.h
+++ b/oi/FuncGen.h
@@ -76,7 +76,7 @@ class FuncGen {
 
   static void DefineDataSegmentDataBuffer(std::string& testCode);
   static void DefineBackInserterDataBuffer(std::string& code);
-  static void DefineBasicTypeHandlers(std::string& code, FeatureSet features);
+  static void DefineBasicTypeHandlers(std::string& code);
 
   static ContainerInfo GetOiArrayContainerInfo();
 };
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index 1be10f5..ccce3fa 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -126,12 +126,27 @@ void TopoSorter::visit(Pointer& p) {
     // Typedefs can not be forward declared, so we must sort them before
     // pointers which reference them
     accept(p.pointeeType());
-    return;
+  } else {
+    // Pointers do not create a dependency, but we do still care about the types
+    // they point to, so delay them until the end.
+    acceptAfter(p.pointeeType());
   }
 
-  // Pointers do not create a dependency, but we do still care about the types
-  // they point to, so delay them until the end.
-  acceptAfter(p.pointeeType());
+  sortedTypes_.push_back(p);
+}
+
+void TopoSorter::visit(Reference& r) {
+  if (dynamic_cast<Typedef*>(&r.pointeeType())) {
+    // Typedefs can not be forward declared, so we must sort them before
+    // pointers which reference them
+    accept(r.pointeeType());
+  } else {
+    // Pointers do not create a dependency, but we do still care about the types
+    // they point to, so delay them until the end.
+    acceptAfter(r.pointeeType());
+  }
+
+  sortedTypes_.push_back(r);
 }
 
 void TopoSorter::visit(CaptureKeys& c) {
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 5c78ad1..6eb7083 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -46,6 +46,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Enum& e) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& r) override;
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
   void visit(Incomplete& i) override;
diff --git a/test/integration/anonymous.toml b/test/integration/anonymous.toml
index 70600d1..d1e6286 100644
--- a/test/integration/anonymous.toml
+++ b/test/integration/anonymous.toml
@@ -97,7 +97,6 @@ definitions = '''
       ]}]'''
 
   [cases.anon_struct]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const AnonStructContainer&"]
     setup = '''
       return AnonStructContainer{
@@ -133,6 +132,40 @@ definitions = '''
         }]
       }]
     }]'''
+    expect_json_v2 = '''[{
+      "name":"a0",
+      "typeNames":["ns_anonymous::AnonStructContainer"],
+      "staticSize":8,
+      "exclusiveSize":0,
+      "size":20,
+      "members":[{
+        "name":"anon",
+        "typeNames":["__oi_anon_1"],
+        "staticSize":8,
+        "exclusiveSize":0,
+        "size":20,
+        "members":[{
+          "name":"node",
+          "typeNames":["ns_anonymous::Node*"],
+          "staticSize":8,
+          "exclusiveSize":8,
+          "size":20,
+          "length":1,
+          "capacity":1,
+          "members":[{
+            "name":"*",
+            "typeNames":["ns_anonymous::Node"],
+            "staticSize":12,
+            "exclusiveSize":0,
+            "size":12,
+            "members":[
+              { "name": "a", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "b", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "c", "staticSize": 4, "exclusiveSize": 4, "size": 4 }
+            ]
+        }]
+      }]}
+    ]}]'''
 
   [cases.anon_struct_ptr]
     skip = "We don't support pointer to anon-structs yet" # https://github.com/facebookexperimental/object-introspection/issues/20
@@ -146,7 +179,6 @@ definitions = '''
     features = ["chase-raw-pointers"]
 
   [cases.anon_typedef]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const AnonTypedefContainer&"]
     setup = '''
       return AnonTypedefContainer{
@@ -192,6 +224,34 @@ definitions = '''
         }]
       }]
     }]'''
+    expect_json_v2 = '''[{
+      "staticSize": 8,
+      "exclusiveSize": 0,
+      "size": 20,
+      "members": [{
+        "typeNames":["AnonStruct", "__oi_anon_2"],
+        "staticSize": 8,
+        "exclusiveSize": 0,
+        "size": 20,
+        "members": [{
+          "typeNames": ["ns_anonymous::Node*"],
+          "staticSize": 8,
+          "exclusiveSize": 8,
+          "size": 20,
+          "members": [{
+            "typeNames": ["ns_anonymous::Node"],
+            "staticSize": 12,
+            "exclusiveSize": 0,
+            "size": 12,
+            "members": [
+              { "name": "a", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "b", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "c", "staticSize": 4, "exclusiveSize": 4, "size": 4 }
+            ]
+          }]
+        }]
+      }]
+    }]'''
 
   [cases.anon_union]
     param_types = ["const AnonUnionContainer&"]
@@ -217,7 +277,6 @@ definitions = '''
     }]'''
 
   [cases.nested_anon_struct]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const NestedAnonContainer&"]
     features = ["chase-raw-pointers"]
     setup = 'return NestedAnonContainer{.m = { .v = {.as = {new Node{1, 2, 3}}}}};'
@@ -272,6 +331,64 @@ definitions = '''
         "dynamicSize": 0
       }]
     }]'''
+    expect_json_v2 = '''[{
+      "staticSize":80,
+      "exclusiveSize":0,
+      "size":92,
+      "members":[
+        {
+          "name":"m",
+          "typeNames": ["__oi_anon_1"],
+          "staticSize": 48,
+          "exclusiveSize":0,
+          "size": 60,
+          "members":[
+            {"name":"__oi_anon_0", "typeNames":["__oi_anon_2"], "staticSize":16, "exclusiveSize":16, "size":16},
+            {
+              "name":"v",
+              "typeNames":["__oi_anon_3"],
+              "staticSize":32,
+              "exclusiveSize":4,
+              "size":44,
+              "members":[
+                {"name":"a", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+                {"name":"b", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+                {"name":"c", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+                {"name":"__oi_anon_4", "typeNames":["__oi_anon_4"], "staticSize":8, "exclusiveSize":8, "size":8},
+                {
+                  "name":"as",
+                  "typeNames":["AnonStruct", "__oi_anon_6"],
+                  "staticSize":8,
+                  "exclusiveSize":0,
+                  "size":20,
+                  "members":[{
+                    "name":"node",
+                    "typeNames":["ns_anonymous::Node*"],
+                    "staticSize":8,
+                    "exclusiveSize":8,
+                    "size":20,
+                    "members":[{
+                      "name":"*",
+                      "typeNames":["ns_anonymous::Node"],
+                      "staticSize":12,
+                      "exclusiveSize":0,
+                      "size":12,
+                      "members":[
+                        { "name": "a", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+                        { "name": "b", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+                        { "name": "c", "staticSize": 4, "exclusiveSize": 4, "size": 4 }
+                      ]
+                    }]
+                  }]
+                }
+              ]
+            }
+          ]
+        },
+        {"name":"__oi_anon_1", "typeNames": ["__oi_anon_8"], "staticSize":24, "exclusiveSize":24, "size":24},
+        {"name":"__oi_anon_2", "typeNames": ["__oi_anon_9"], "staticSize":8, "exclusiveSize":8, "size":8}
+      ]
+    }]'''
 
   # This test is disabled due to GCC not supporting it
   # [cases.anon_array]
diff --git a/test/integration/pointers.toml b/test/integration/pointers.toml
index cfd3627..6868679 100644
--- a/test/integration/pointers.toml
+++ b/test/integration/pointers.toml
@@ -134,7 +134,6 @@ definitions = '''
 
 
   [cases.struct_primitive_ptrs]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, new int(0), new int(0)};"
     features = ["chase-raw-pointers"]
@@ -147,6 +146,16 @@ definitions = '''
         {"name":"b", "staticSize":8, "exclusiveSize":8, "dynamicSize":4},
         {"name":"c", "staticSize":8, "exclusiveSize":8, "dynamicSize":0}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":4,
+      "size":28,
+      "members":[
+        {"name":"a", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"b", "typeNames":["int32_t*"], "staticSize":8, "exclusiveSize":8, "size":12},
+        {"name":"c", "typeNames":["void*"], "staticSize":8, "exclusiveSize":8, "size":8}
+      ]
+    }]'''
   [cases.struct_primitive_ptrs_no_follow]
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, new int(0), new int(0)};"
@@ -161,7 +170,6 @@ definitions = '''
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_primitive_ptrs_null]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, nullptr, nullptr};"
     features = ["chase-raw-pointers"]
@@ -175,10 +183,19 @@ definitions = '''
         {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":4,
+      "size":24,
+      "members":[
+        {"name":"a", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"b", "typeNames":["int32_t*"], "staticSize":8, "exclusiveSize":8, "size":8},
+        {"name":"c", "typeNames":["void*"], "staticSize":8, "exclusiveSize":8, "size":8}
+      ]
+    }]'''
 
 
   [cases.struct_vector_ptr]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{new std::vector<int>{1,2,3}};"
     features = ["chase-raw-pointers"]
@@ -188,6 +205,22 @@ definitions = '''
       "members":[
         {"name":"vec", "staticSize":8, "dynamicSize":36}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":8,
+      "exclusiveSize":0,
+      "size":44,
+      "members": [
+        {
+          "typeNames":["std::vector<int32_t, std::allocator<int32_t>>*"],
+          "staticSize":8,
+          "exclusiveSize":8,
+          "size":44,
+          "length":1,
+          "capacity":1,
+          "members":[{ "staticSize":24, "exclusiveSize":24, "size":36 }]
+        }
+      ]
+    }]'''
   [cases.struct_vector_ptr_no_follow]
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{new std::vector<int>{1,2,3}};"
@@ -200,7 +233,6 @@ definitions = '''
         {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_vector_ptr_null]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{nullptr};"
     features = ["chase-raw-pointers"]
@@ -212,10 +244,25 @@ definitions = '''
       "members":[
         {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
+      "members": [
+        {
+          "typeNames":["std::vector<int32_t, std::allocator<int32_t>>*"],
+          "staticSize":8,
+          "exclusiveSize":8,
+          "size":8,
+          "length":0,
+          "capacity":1,
+          "members":[]
+        }
+      ]
+    }]'''
 
 
   [cases.vector_of_pointers]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const std::vector<int*>&"]
     setup = "return {{new int(1), nullptr, new int(3)}};"
     features = ["chase-raw-pointers"]
@@ -230,6 +277,18 @@ definitions = '''
         {"staticSize":8, "dynamicSize":0, "pointer":0},
         {"staticSize":8, "dynamicSize":4, "NOT": {"pointer":0}}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":24,
+      "size":56,
+      "length":3,
+      "capacity":3,
+      "members":[
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}},
+        {"staticSize":8, "exclusiveSize":8, "size":8, "pointer":0},
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}}
+      ]
+    }]'''
   [cases.vector_of_pointers_no_follow]
     skip = "pointer field is missing from results" # https://github.com/facebookexperimental/object-introspection/issues/21
     param_types = ["const std::vector<int*>&"]
@@ -260,7 +319,6 @@ definitions = '''
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.feature_config]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const std::vector<int*>&"]
     setup = "return {{new int(1), nullptr, new int(3)}};"
     config_prefix = 'features = ["chase-raw-pointers"]'
@@ -275,3 +333,15 @@ definitions = '''
         {"staticSize":8, "dynamicSize":0, "pointer":0},
         {"staticSize":8, "dynamicSize":4, "NOT": {"pointer":0}}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":24,
+      "size":56,
+      "length":3,
+      "capacity":3,
+      "members":[
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}},
+        {"staticSize":8, "exclusiveSize":8, "size":8, "pointer":0},
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}}
+      ]
+    }]'''
diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml
index 87af9ad..b062043 100644
--- a/test/integration/pointers_incomplete.toml
+++ b/test/integration/pointers_incomplete.toml
@@ -86,7 +86,6 @@ definitions = '''
     expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.containing_struct]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const IncompleteTypeContainer&"]
     setup = "return IncompleteTypeContainer{};"
     features = ["chase-raw-pointers"]
@@ -115,6 +114,32 @@ definitions = '''
         }
       ]
     }]'''
+    expect_json_v2 = '''[{
+      "staticSize": 88,
+      "exclusiveSize": 21,
+      "size": 88,
+      "members": [
+        { "name": "ptrundef", "staticSize": 8, "exclusiveSize": 8, "size": 8 },
+        { "name": "__makePad1", "staticSize": 1, "exclusiveSize": 1, "size": 1 },
+        { "name": "shundef", "staticSize": 16, "exclusiveSize": 16, "size": 16 },
+        { "name": "__makePad2", "staticSize": 1, "exclusiveSize": 1, "size": 1 },
+        { "name": "shoptundef",
+          "staticSize": 24,
+          "exclusiveSize": 24,
+          "size": 24,
+          "length": 0,
+          "capacity": 1
+        },
+        { "name": "__makePad3", "staticSize": 1, "exclusiveSize": 1, "size": 1 },
+        { "name": "optundef",
+          "staticSize": 16,
+          "exclusiveSize": 16,
+          "size": 16,
+          "length": 0,
+          "capacity": 1
+        }
+      ]
+    }]'''
 
   [cases.containing_struct_no_follow]
     param_types = ["const IncompleteTypeContainer&"]
diff --git a/test/test_topo_sorter.cpp b/test/test_topo_sorter.cpp
index c97801f..7c00441 100644
--- a/test/test_topo_sorter.cpp
+++ b/test/test_topo_sorter.cpp
@@ -236,6 +236,22 @@ TEST(TopoSorterTest, Pointers) {
   myclass.members.push_back(Member{mypointer, "ptr", 0});
 
   test({myclass}, R"(
+ClassA*
+MyClass
+ClassA
+)");
+}
+
+TEST(TopoSorterTest, References) {
+  // References do not require pointee types to be defined first
+  auto classA = Class{0, Class::Kind::Class, "ClassA", 69};
+  auto myreference = Reference{1, classA};
+
+  auto myclass = Class{2, Class::Kind::Class, "MyClass", 69};
+  myclass.members.push_back(Member{myreference, "ref", 0});
+
+  test({myclass}, R"(
+ClassA*
 MyClass
 ClassA
 )");
@@ -258,6 +274,7 @@ TEST(TopoSorterTest, PointerCycle) {
   // the same sorted order for ClassA and ClassB.
   for (const auto& input : inputs) {
     test(input, R"(
+ClassA*
 ClassB
 ClassA
 )");
@@ -276,6 +293,7 @@ TEST(TopoSorterTest, PointerToTypedef) {
   test({myclass}, R"(
 ClassA
 aliasA
+aliasA*
 MyClass
 )");
 }

From 351f2f84e419bacfae18389082be71613d4fbdc8 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 16 Jan 2024 16:23:52 +0000
Subject: [PATCH 142/188] tbv2: support capture-thrift-isset

Support the capture-thrift-isset feature with TreeBuilder-v2. Fairly minor
changes here except the type of the Enum in a template parameter now matters.

We follow the previous behaviour of capturing a value for each field in a
struct that has an `isset_bitset`. This value is a VarInt captured before the
C++ contents of the member. It has 3 values: 0 (not set), 1 (set), and 2
(unavailable). These are handled by the processor and represented in the output
as `false`, `true`, and `std::nullopt_t` respectively.

Changes:
- Add a simple Thrift isset processor before any fields that have Thrift isset.
- Store the fully qualified names of enum types in DrgnParser - it already
  worked out this information anyway for naming the values and this is
  consistent with classes.
- Forward all enum template parameters under their input name under the
  assumption that they will all be policy type things like `IssetBitsetOption`.
  This could turn out to be wrong.

Test plan:
- CI (doesn't test thrift changes but covers other regressions)
- Updated Thrift enum tests for new format.
- `FILTER='OilIntegration.*' make test` - Thrift tests failed before, succeed
  after.
---
 include/oi/exporters/Json.h                |  2 +-
 oi/CodeGen.cpp                             | 78 +++++++++++++++---
 oi/OITraceCode.cpp                         | 23 ++++++
 oi/type_graph/ClangTypeParser.cpp          |  5 +-
 oi/type_graph/DrgnParser.cpp               | 34 +++++---
 oi/type_graph/Types.h                      | 10 ++-
 test/integration/enums.toml                |  6 +-
 test/integration/thrift_isset.toml         | 92 ++++++++++++++++++++--
 test/integration/thrift_isset_missing.toml | 51 +++++++++++-
 9 files changed, 263 insertions(+), 38 deletions(-)

diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h
index 3dd37f4..3416ef8 100644
--- a/include/oi/exporters/Json.h
+++ b/include/oi/exporters/Json.h
@@ -158,7 +158,7 @@ inline void Json::printFields(const result::Element& el,
     printUnsignedField("capacity", el.container_stats->capacity, indent);
   }
   if (el.is_set_stats.has_value())
-    printUnsignedField("is_set", el.is_set_stats->is_set, indent);
+    printBoolField("is_set", el.is_set_stats->is_set, indent);
   printBoolField("is_primitive", el.is_primitive, indent);
 }
 
diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index cb5bf54..a125508 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -687,7 +687,9 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
     }
 
     if (thriftIssetMember != nullptr && thriftIssetMember != &member) {
-      code += "\n  .write(getThriftIsset(t, " + std::to_string(i) + "))";
+      code += "\n      .write(getThriftIsset(t, ";
+      code += std::to_string(i);
+      code += "))";
     }
 
     code += "\n      .";
@@ -765,6 +767,12 @@ void CodeGen::genClassStaticType(const Class& c, std::string& code) {
 
 void CodeGen::genClassTreeBuilderInstructions(const Class& c,
                                               std::string& code) {
+  const Member* thriftIssetMember = nullptr;
+  if (const auto it = thriftIssetMembers_.find(&c);
+      it != thriftIssetMembers_.end()) {
+    thriftIssetMember = it->second;
+  }
+
   code += " private:\n";
   size_t index = 0;
   for (const auto& m : c.members) {
@@ -799,12 +807,29 @@ void CodeGen::genClassTreeBuilderInstructions(const Class& c,
       continue;
     std::string fullName = c.name() + "::" + m.name;
     bool isPrimitive = dynamic_cast<const Primitive*>(&m.type());
-    code += "      inst::Field{sizeof(" + fullName + "), " +
-            std::to_string(calculateExclusiveSize(m.type())) + ",\"" +
-            m.inputName + "\", member_" + std::to_string(index) +
-            "_type_names, TypeHandler<Ctx, decltype(" + fullName +
-            ")>::fields, TypeHandler<Ctx, decltype(" + fullName +
-            ")>::processors, ";
+    code += "      inst::Field{sizeof(";
+    code += fullName;
+    code += "), ";
+    code += std::to_string(calculateExclusiveSize(m.type()));
+    code += ",\"";
+    code += m.inputName;
+    code += "\", member_";
+    code += std::to_string(index);
+    code += "_type_names, TypeHandler<Ctx, decltype(";
+    code += fullName;
+    code += ")>::fields, ";
+
+    if (thriftIssetMember != nullptr && thriftIssetMember != &m) {
+      code += "ThriftIssetHandler<";
+    }
+    code += "TypeHandler<Ctx, decltype(";
+    code += fullName;
+    code += ")>";
+    if (thriftIssetMember != nullptr && thriftIssetMember != &m) {
+      code += '>';
+    }
+    code += "::processors, ";
+
     code += isPrimitive ? "true" : "false";
     code += "},\n";
   }
@@ -825,10 +850,10 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) {
   static int getThriftIsset(const %1%& t, size_t i) {
     using thrift_data = apache::thrift::TStructDataStorage<%2%>;
 
-    if (&thrift_data::isset_indexes == nullptr) return -1;
+    if (&thrift_data::isset_indexes == nullptr) return 2;
 
     auto idx = thrift_data::isset_indexes[i];
-    if (idx == -1) return -1;
+    if (idx == -1) return 2;
 
     return t.%3%.get(idx);
   }
@@ -900,7 +925,15 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   for (const auto& p : templateParams) {
     if (p.value) {
       code += ", ";
-      code += p.type().name();
+
+      // HACK: forward all enums directly. this might turn out to be a problem
+      // if there are enums we should be regenerating/use in the body.
+      if (const auto* e = dynamic_cast<const Enum*>(&p.type())) {
+        code += e->inputName();
+      } else {
+        code += p.type().name();
+      }
+
       code += " N" + std::to_string(values++);
     } else {
       code += ", typename T" + std::to_string(types++);
@@ -1048,11 +1081,32 @@ void addCaptureKeySupport(std::string& code) {
   )";
 }
 
+void addThriftIssetSupport(std::string& code) {
+  code += R"(
+void processThriftIsset(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
+  auto v = std::get<ParsedData::VarInt>(d.val).value;
+  if (v <= 1) {
+    el.is_set_stats.emplace(result::Element::IsSetStats { v == 1 });
+  }
+}
+static constexpr exporters::inst::ProcessorInst thriftIssetProcessor{
+  types::st::VarInt<int>::describe,
+  &processThriftIsset,
+};
+
+template <typename Handler>
+struct ThriftIssetHandler {
+  static constexpr auto processors = arrayPrepend(Handler::processors, thriftIssetProcessor);
+};
+)";
+}
+
 void addStandardTypeHandlers(TypeGraph& typeGraph,
                              FeatureSet features,
                              std::string& code) {
-  if (features[Feature::TreeBuilderV2])
-    addCaptureKeySupport(code);
+  addCaptureKeySupport(code);
+  if (features[Feature::CaptureThriftIsset])
+    addThriftIssetSupport(code);
 
   // Provide a wrapper function, getSizeType, to infer T instead of having to
   // explicitly specify it with TypeHandler<Ctx, T>::getSizeType every time.
diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp
index 323c83e..362ce69 100644
--- a/oi/OITraceCode.cpp
+++ b/oi/OITraceCode.cpp
@@ -40,6 +40,13 @@ constexpr int oidMagicId = 0x01DE8;
 
 namespace {
 
+template <typename T, size_t N>
+constexpr std::array<T, N + 1> arrayPrepend(std::array<T, N> a, T t);
+template <typename T, size_t N, size_t... I>
+constexpr std::array<T, N + 1> arrayPrependHelper(std::array<T, N> a,
+                                                  T t,
+                                                  std::index_sequence<I...>);
+
 template <size_t Size = (1 << 20) / sizeof(uintptr_t)>
 class PointerHashSet {
  private:
@@ -182,6 +189,22 @@ bool isStorageInline(const auto& c) {
          (uintptr_t)std::data(c) >= (uintptr_t)&c;
 }
 
+namespace {
+
+template <typename T, size_t N, size_t... I>
+constexpr std::array<T, N + 1> arrayPrependHelper(std::array<T, N> a,
+                                                  T t,
+                                                  std::index_sequence<I...>) {
+  return {t, a[I]...};
+}
+
+template <typename T, size_t N>
+constexpr std::array<T, N + 1> arrayPrepend(std::array<T, N> a, T t) {
+  return arrayPrependHelper(a, t, std::make_index_sequence<N>());
+}
+
+}  // namespace
+
 namespace OIInternal {
 namespace {
 
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index 24d5835..cb33377 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -137,6 +137,8 @@ Typedef& ClangTypeParser::enumerateTypedef(const clang::TypedefType& ty) {
 }
 
 Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
+  std::string fqName = clang::TypeName::getFullyQualifiedName(
+      clang::QualType(&ty, 0), *ast, {ast->getLangOpts()});
   std::string name = ty.getDecl()->getNameAsString();
   auto size = ast->getTypeSize(clang::QualType(&ty, 0)) / 8;
 
@@ -148,7 +150,8 @@ Enum& ClangTypeParser::enumerateEnum(const clang::EnumType& ty) {
     }
   }
 
-  return makeType<Enum>(ty, std::move(name), size, std::move(enumeratorMap));
+  return makeType<Enum>(
+      ty, std::move(name), std::move(fqName), size, std::move(enumeratorMap));
 }
 
 Array& ClangTypeParser::enumerateArray(const clang::ConstantArrayType& ty) {
diff --git a/oi/type_graph/DrgnParser.cpp b/oi/type_graph/DrgnParser.cpp
index 0116edf..73eb531 100644
--- a/oi/type_graph/DrgnParser.cpp
+++ b/oi/type_graph/DrgnParser.cpp
@@ -73,6 +73,8 @@ void warnForDrgnError(struct drgn_type* type,
                       const std::string& msg,
                       struct drgn_error* err);
 
+std::string getDrgnFullyQualifiedName(struct drgn_type* type);
+
 }  // namespace
 
 Type& DrgnParser::parse(struct drgn_type* root) {
@@ -150,6 +152,8 @@ Class& DrgnParser::enumerateClass(struct drgn_type* type) {
   std::string fqName;
   char* nameStr = nullptr;
   size_t length = 0;
+  // HACK: Leak this error. Freeing it causes a SEGV which suggests our
+  // underlying implementation is bad.
   auto* err = drgn_type_fully_qualified_name(type, &nameStr, &length);
   if (err == nullptr && nameStr != nullptr) {
     fqName = nameStr;
@@ -307,14 +311,7 @@ void DrgnParser::enumerateTemplateParam(struct drgn_type* type,
     // This template parameter is a value
     std::string value;
 
-    if (drgn_type_kind(obj->type) == DRGN_TYPE_ENUM) {
-      char* nameStr = nullptr;
-      size_t length = 0;
-      err = drgn_type_fully_qualified_name(obj->type, &nameStr, &length);
-      if (err != nullptr || nameStr == nullptr) {
-        throw DrgnParserError{"Failed to get enum's fully qualified name", err};
-      }
-
+    if (const auto* e = dynamic_cast<const Enum*>(&ttype)) {
       uint64_t enumVal;
       switch (obj->encoding) {
         case DRGN_OBJECT_ENCODING_SIGNED:
@@ -333,7 +330,9 @@ void DrgnParser::enumerateTemplateParam(struct drgn_type* type,
       size_t numEnumerators = drgn_type_num_enumerators(obj->type);
       for (size_t j = 0; j < numEnumerators; j++) {
         if (enumerators[j].uvalue == enumVal) {
-          value = std::string{nameStr} + "::" + enumerators[j].name;
+          value = e->inputName();
+          value += "::";
+          value += enumerators[j].name;
           break;
         }
       }
@@ -416,6 +415,8 @@ void DrgnParser::enumerateClassFunctions(struct drgn_type* type,
 }
 
 Enum& DrgnParser::enumerateEnum(struct drgn_type* type) {
+  std::string fqName = getDrgnFullyQualifiedName(type);
+
   const char* typeTag = drgn_type_tag(type);
   std::string name = typeTag ? typeTag : "";
   uint64_t size = get_drgn_type_size(type);
@@ -432,7 +433,8 @@ Enum& DrgnParser::enumerateEnum(struct drgn_type* type) {
     }
   }
 
-  return makeType<Enum>(type, name, size, std::move(enumeratorMap));
+  return makeType<Enum>(
+      type, std::move(name), std::move(fqName), size, std::move(enumeratorMap));
 }
 
 Typedef& DrgnParser::enumerateTypedef(struct drgn_type* type) {
@@ -528,6 +530,18 @@ void warnForDrgnError(struct drgn_type* type,
                << ": " << err->code << " " << err->message;
   drgn_error_destroy(err);
 }
+
+std::string getDrgnFullyQualifiedName(drgn_type* type) {
+  char* nameStr = nullptr;
+  size_t length = 0;
+  auto* err = drgn_type_fully_qualified_name(type, &nameStr, &length);
+  if (err != nullptr)
+    throw DrgnParserError("failed to get fully qualified name!", err);
+  if (nameStr != nullptr)
+    return nameStr;
+  return {};
+}
+
 }  // namespace
 
 }  // namespace oi::detail::type_graph
diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h
index f09f926..45b7d06 100644
--- a/oi/type_graph/Types.h
+++ b/oi/type_graph/Types.h
@@ -477,14 +477,20 @@ class Container : public Type {
 class Enum : public Type {
  public:
   explicit Enum(std::string name,
+                std::string inputName,
                 size_t size,
                 std::map<int64_t, std::string> enumerators = {})
-      : name_(name),
-        inputName_(std::move(name)),
+      : name_(std::move(name)),
+        inputName_(std::move(inputName)),
         size_(size),
         enumerators_(std::move(enumerators)) {
   }
 
+  explicit Enum(std::string name,
+                size_t size,
+                std::map<int64_t, std::string> enumerators = {})
+      : Enum{name, std::move(name), size, std::move(enumerators)} {};
+
   static inline constexpr bool has_node_id = false;
 
   DECLARE_ACCEPT
diff --git a/test/integration/enums.toml b/test/integration/enums.toml
index fa91cbc..f5274a3 100644
--- a/test/integration/enums.toml
+++ b/test/integration/enums.toml
@@ -36,12 +36,12 @@ definitions = '''
     param_types = ["ScopedEnum"]
     setup = "return {};"
     expect_json = '[{"staticSize":4, "dynamicSize":0}]'
-    expect_json_v2 = '[{"typeNames": ["ScopedEnum"], "staticSize":4, "exclusiveSize":4, "size":4}]'
+    expect_json_v2 = '[{"typeNames": ["ns_enums::ScopedEnum"], "staticSize":4, "exclusiveSize":4, "size":4}]'
   [cases.scoped_uint8]
     param_types = ["ScopedEnumUint8"]
     setup = "return {};"
     expect_json = '[{"staticSize":1, "dynamicSize":0}]'
-    expect_json_v2 = '[{"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1}]'
+    expect_json_v2 = '[{"typeNames": ["ns_enums::ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1}]'
   [cases.unscoped]
     param_types = ["UNSCOPED_ENUM"]
     setup = "return {};"
@@ -60,7 +60,7 @@ definitions = '''
     expect_json = '[{"staticSize":2, "dynamicSize":0}]'
     expect_json_v2 = '''[
       {"staticSize": 2, "exclusiveSize": 0, "size":2, "members": [
-        {"typeNames": ["ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1},
+        {"typeNames": ["ns_enums::ScopedEnumUint8"], "staticSize":1, "exclusiveSize":1, "size":1},
         {"typeNames": ["uint8_t"], "staticSize":1, "exclusiveSize":1, "size":1}
       ]}
     ]'''
diff --git a/test/integration/thrift_isset.toml b/test/integration/thrift_isset.toml
index 98e19fd..cdbcdb5 100644
--- a/test/integration/thrift_isset.toml
+++ b/test/integration/thrift_isset.toml
@@ -77,7 +77,6 @@ namespace cpp2 {
 
 [cases]
   [cases.unpacked]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructUnpacked&"]
     setup = '''
       cpp2::MyThriftStructUnpacked ret;
@@ -95,9 +94,19 @@ namespace cpp2 {
         {"name":"__fbthrift_field_c", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize":1,
+      "size":16,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]
+    }]'''
 
   [cases.packed]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructPacked&"]
     setup = '''
       cpp2::MyThriftStructPacked ret;
@@ -126,9 +135,25 @@ namespace cpp2 {
         {"name":"__fbthrift_field_j", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":2}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":44,
+      "exclusiveSize":2,
+      "size":44,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_d", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_e", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_f", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_g", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_h", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_i", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_j", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":2, "NOT":"is_set"}
+      ]}]'''
 
   [cases.packed_non_atomic]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructPackedNonAtomic&"]
     setup = '''
       cpp2::MyThriftStructPackedNonAtomic ret;
@@ -147,9 +172,19 @@ namespace cpp2 {
         {"name":"__fbthrift_field_d", "staticSize":4, "isset":false},
         {"name":"__isset", "staticSize":1}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":20,
+      "exclusiveSize":3,
+      "size":20,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_d", "staticSize":4, "is_set":false},
+        {"name":"__isset", "staticSize":1, "NOT":"is_set"}
+      ]}]'''
 
   [cases.out_of_order]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructOutOfOrder&"]
     setup = '''
       cpp2::MyThriftStructOutOfOrder ret;
@@ -166,9 +201,18 @@ namespace cpp2 {
         {"name":"__fbthrift_field_c", "staticSize":4, "isset":false},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize":1,
+      "size":16,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":false},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
 
   [cases.required]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructRequired&"]
     setup = '''
       cpp2::MyThriftStructRequired ret;
@@ -189,9 +233,21 @@ namespace cpp2 {
         {"name":"__fbthrift_field_f", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":28,
+      "exclusiveSize":1,
+      "size":28,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_d", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_e", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_f", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
 
   [cases.box]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const cpp2::MyThriftStructBoxed&"]
     setup = '''
       cpp2::MyThriftStructBoxed ret;
@@ -211,6 +267,18 @@ namespace cpp2 {
         {"name":"__fbthrift_field_e", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":32,
+      "exclusiveSize":1,
+      "size":32,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":8, "NOT":"is_set"},
+        {"name":"__fbthrift_field_b", "staticSize":8, "NOT":"is_set"},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_d", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_e", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
 
   [cases.no_capture]
     param_types = ["const cpp2::MyThriftStructBoxed&"]
@@ -231,3 +299,15 @@ namespace cpp2 {
         {"name":"__fbthrift_field_e", "staticSize":4, "NOT":"isset"},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":32,
+      "exclusiveSize":1,
+      "size":32,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":8, "NOT":"is_set"},
+        {"name":"__fbthrift_field_b", "staticSize":8, "NOT":"is_set"},
+        {"name":"__fbthrift_field_c", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_d", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_e", "staticSize":4, "NOT":"is_set"},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
diff --git a/test/integration/thrift_isset_missing.toml b/test/integration/thrift_isset_missing.toml
index 3e7ad94..311f736 100644
--- a/test/integration/thrift_isset_missing.toml
+++ b/test/integration/thrift_isset_missing.toml
@@ -58,7 +58,6 @@ raw_definitions = '''
 '''
 [cases]
   [cases.present]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const FakeThriftWithData&"]
     setup = '''
       FakeThriftWithData ret;
@@ -78,8 +77,17 @@ raw_definitions = '''
         {"name":"__fbthrift_field_c", "staticSize":4, "isset":true},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize":1,
+      "size":16,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
   [cases.missing]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const FakeThriftWithoutData&"]
     setup = '''
       FakeThriftWithoutData ret;
@@ -99,8 +107,17 @@ raw_definitions = '''
         {"name":"__fbthrift_field_c", "staticSize":4, "NOT":"isset"},
         {"name":"__isset", "staticSize":3}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":16,
+      "exclusiveSize":1,
+      "size":16,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_b", "staticSize":4, "NOT":"is_set"},
+        {"name":"__fbthrift_field_c", "staticSize":4, "NOT":"is_set"},
+        {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+      ]}]'''
   [cases.mixed]
-    oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296
     param_types = ["const Mixed&"]
     setup = '''
       Mixed ret;
@@ -143,3 +160,31 @@ raw_definitions = '''
           ]
         }
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":32,
+      "exclusiveSize":0,
+      "size":32,
+      "members":[
+        {
+          "staticSize":16,
+          "exclusiveSize":1,
+          "size":16,
+          "members":[
+            {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+            {"name":"__fbthrift_field_b", "staticSize":4, "is_set":false},
+            {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+            {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+          ]
+        },
+        {
+          "staticSize":16,
+          "exclusiveSize":1,
+          "size":16,
+          "members":[
+            {"name":"__fbthrift_field_a", "staticSize":4, "NOT":"is_set"},
+            {"name":"__fbthrift_field_b", "staticSize":4, "NOT":"is_set"},
+            {"name":"__fbthrift_field_c", "staticSize":4, "NOT":"is_set"},
+            {"name":"__isset", "staticSize":3, "NOT":"is_set"}
+          ]
+        }
+      ]}]'''

From df95636795e7f43f88ee3c4b7f28089723b1fcc4 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 16 Jan 2024 17:07:39 +0000
Subject: [PATCH 143/188] tbv2: fix pointer codegen

A previous change enabled running OIL tests with specific features enabled.
This highlighted that pointer code generation under TreeBuilder-v2 was very
broken. This change updates pointer code generation to work and enables the
skipped tests. All enabled tests need `expected_json_v2` added to them due to
formatting differences.

Reformatted and rewrote the basic type handler that handles primitives and
pointers. Removed the reliance on `features` to decide whether to generate for
TreeBuilder-v2 as the intermediate features have been removed. Pointers are
treated as containers with a capacity of 1 and a length of 0 if null/a cycle
and 1 if followed. This holds for void pointers where, although they aren't
followed, the length is still set.

There were a couple of other changes needed to enable these tests on TBv2 that
aren't worth their own issues and PRs, I sneaked them in here.

Extra changes:
- Added `Pointer` and `Reference` to TopoSorter so they generate
  `NameProvider` instances. It might be worth visiting the graph differently
  for `NameProvider` as it requires so many instances that others generators do
  not. Will consider that in the future.
- Follow typedefs when calculating exclusive size for a type.

Closes #458.

Test plan:
- CI
- Enabled previously disabled tests.
---
 oi/CodeGen.cpp                            |  35 ++--
 oi/FuncGen.cpp                            | 209 ++++++++++++----------
 oi/FuncGen.h                              |   2 +-
 oi/type_graph/TopoSorter.cpp              |  23 ++-
 oi/type_graph/TopoSorter.h                |   1 +
 test/integration/anonymous.toml           | 123 ++++++++++++-
 test/integration/pointers.toml            |  82 ++++++++-
 test/integration/pointers_incomplete.toml |  27 ++-
 test/test_topo_sorter.cpp                 |  18 ++
 9 files changed, 386 insertions(+), 134 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index cb5bf54..963410e 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -195,7 +195,12 @@ void genDecls(const TypeGraph& typeGraph, std::string& code) {
 namespace {
 
 size_t calculateExclusiveSize(const Type& t) {
-  if (const auto* c = dynamic_cast<const Class*>(&t)) {
+  const Type* finalType = &t;
+  while (const auto* td = dynamic_cast<const Typedef*>(finalType)) {
+    finalType = &td->underlyingType();
+  }
+
+  if (const auto* c = dynamic_cast<const Class*>(finalType)) {
     return std::accumulate(
         c->members.cbegin(), c->members.cend(), 0, [](size_t a, const auto& m) {
           if (m.name.starts_with(AddPadding::MemberPrefix))
@@ -203,7 +208,7 @@ size_t calculateExclusiveSize(const Type& t) {
           return a;
         });
   }
-  return t.size();
+  return finalType->size();
 }
 
 }  // namespace
@@ -211,7 +216,7 @@ size_t calculateExclusiveSize(const Type& t) {
 void genNames(const TypeGraph& typeGraph, std::string& code) {
   code += R"(
 template <typename T>
-struct NameProvider {};
+struct NameProvider;
 )";
 
   // TODO: stop types being duplicated at this point and remove this check
@@ -239,6 +244,9 @@ struct ExclusiveSizeProvider {
 )";
 
   for (const Type& t : typeGraph.finalTypes) {
+    if (dynamic_cast<const Typedef*>(&t))
+      continue;
+
     size_t exclusiveSize = calculateExclusiveSize(t);
     if (exclusiveSize != t.size()) {
       code += "template <> struct ExclusiveSizeProvider<";
@@ -1066,23 +1074,6 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
     }
 )";
 
-  if (features[Feature::TreeBuilderV2]) {
-    code += R"(
-template <typename Ctx, typename T>
-constexpr inst::Field make_field(std::string_view name) {
-  return inst::Field{
-    sizeof(T),
-    ExclusiveSizeProvider<T>::size,
-    name,
-    NameProvider<T>::names,
-    TypeHandler<Ctx, T>::fields,
-    TypeHandler<Ctx, T>::processors,
-    std::is_fundamental_v<T>
-  };
-}
-)";
-  }
-
   // TODO: bit of a hack - making ContainerInfo a node in the type graph and
   // traversing for it would remove the need for this set altogether.
   std::unordered_set<const ContainerInfo*> used{};
@@ -1264,9 +1255,6 @@ void CodeGen::generate(TypeGraph& typeGraph,
     code += "using namespace oi::detail;\n";
     code += "using oi::exporters::ParsedData;\n";
     code += "using namespace oi::exporters;\n";
-    code += "namespace OIInternal {\nnamespace {\n";
-    FuncGen::DefineBasicTypeHandlers(code, config_.features);
-    code += "} // namespace\n} // namespace OIInternal\n";
   }
 
   if (config_.features[Feature::CaptureThriftIsset]) {
@@ -1306,6 +1294,7 @@ void CodeGen::generate(TypeGraph& typeGraph,
   }
 
   if (config_.features[Feature::TreeBuilderV2]) {
+    FuncGen::DefineBasicTypeHandlers(code);
     addStandardTypeHandlers(typeGraph, config_.features, code);
     addTypeHandlers(typeGraph, code);
   } else {
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 5ac3abf..fbe6658 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -614,110 +614,127 @@ class BackInserter {
  * pointer's value always, then the value of the pointer if it is unique. void
  * is of type Unit and always stores nothing.
  */
-void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
+void FuncGen::DefineBasicTypeHandlers(std::string& code) {
   code += R"(
-    template <typename Ctx, typename T>
-    struct TypeHandler {
-        using DB = typename Ctx::DataBuffer;
-      private:
-        static auto choose_type() {
-            if constexpr(std::is_pointer_v<T>) {
-                return std::type_identity<types::st::Pair<DB,
-                  types::st::VarInt<DB>,
-                  types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>
-                >>();
-            } else {
-                return std::type_identity<types::st::Unit<DB>>();
-            }
-        }
-
-      public:
-        using type = typename decltype(choose_type())::type;
+template <typename Ctx, typename T>
+struct TypeHandler;
 )";
-  if (features[Feature::TreeBuilderV2]) {
-    code += R"(private:
-        static void process_pointer(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
-          el.pointer = std::get<ParsedData::VarInt>(d.val).value;
-        }
-        static void process_pointer_content(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
-          static constexpr std::array<std::string_view, 1> names{"TODO"};
-          static constexpr auto childField = make_field<Ctx, T>("*");
 
-          const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
-
-          el.container_stats.emplace(result::Element::ContainerStats{ .capacity = 1 });
-
-          if (sum.index == 0)
-            return;
-
-          el.container_stats->length = 1;
-          stack_ins(childField);
-        }
-
-        static constexpr auto choose_fields() {
-          if constexpr(std::is_pointer_v<T>) {
-            return std::array<exporters::inst::Field, 0>{};
-          } else {
-            return std::array<exporters::inst::Field, 0>{};
-          }
-        }
-        static constexpr auto choose_processors() {
-          if constexpr(std::is_pointer_v<T>) {
-            return std::array<inst::ProcessorInst, 2>{
-              {types::st::VarInt<DB>::describe, &process_pointer},
-              {types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>::describe, &process_pointer_content},
-            };
-          } else {
-            return std::array<inst::ProcessorInst, 0>{};
-          }
-        }
-      public:
-        static constexpr auto fields = choose_fields();
-        static constexpr auto processors = choose_processors();
+  code += R"(
+template <typename Ctx, typename T>
+constexpr inst::Field make_field(std::string_view name) {
+  return inst::Field{
+      sizeof(T),
+      ExclusiveSizeProvider<T>::size,
+      name,
+      NameProvider<T>::names,
+      TypeHandler<Ctx, T>::fields,
+      TypeHandler<Ctx, T>::processors,
+      std::is_fundamental_v<T>,
+  };
+}
 )";
+  code += R"(
+template <typename Ctx, typename T>
+struct TypeHandler {
+  using DB = typename Ctx::DataBuffer;
+
+ private:
+  static void process_pointer(result::Element& el,
+                              std::function<void(inst::Inst)> stack_ins,
+                              ParsedData d) {
+    el.pointer = std::get<ParsedData::VarInt>(d.val).value;
   }
-  code += R"(
-        static types::st::Unit<DB> getSizeType(
-          Ctx& ctx,
-          const T& t,
-          typename TypeHandler<Ctx, T>::type returnArg) {
-              if constexpr(std::is_pointer_v<T>) {
-                JLOG("ptr val @");
-                JLOGPTR(t);
-                auto r0 = returnArg.write((uintptr_t)t);
-                if (t && ctx.pointers.add((uintptr_t)t)) {
-                  return r0.template delegate<1>([&t](auto ret) {
-                    if constexpr (!std::is_void<std::remove_pointer_t<T>>::value) {
-                      return TypeHandler<Ctx, std::remove_pointer_t<T>>::getSizeType(*t, ret);
-                    } else {
-                      return ret;
-                    }
-                  });
-                } else {
-                  return r0.template delegate<0>(std::identity());
-                }
-              } else {
-                return returnArg;
-              }
-        }
-    };
-  )";
+  static void process_pointer_content(result::Element& el,
+                                      std::function<void(inst::Inst)> stack_ins,
+                                      ParsedData d) {
+    using U = std::decay_t<std::remove_pointer_t<T>>;
+    const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
+
+    el.container_stats.emplace(result::Element::ContainerStats{ .capacity = 1, .length = 0 });
+    if (sum.index == 0)
+      return;
+    el.container_stats->length = 1;
+
+    if constexpr (oi_is_complete<U>) {
+      static constexpr auto childField = make_field<Ctx, U>("*");
+      stack_ins(childField);
+    }
+  }
+
+  static auto choose_type() {
+    if constexpr (std::is_pointer_v<T>) {
+      return std::type_identity<types::st::Pair<
+          DB,
+          types::st::VarInt<DB>,
+          types::st::Sum<
+              DB,
+              types::st::Unit<DB>,
+              typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>>>();
+    } else {
+      return std::type_identity<types::st::Unit<DB>>();
+    }
+  }
+  static constexpr auto choose_processors() {
+    if constexpr (std::is_pointer_v<T>) {
+      return std::array<inst::ProcessorInst, 2>{
+          exporters::inst::ProcessorInst{types::st::VarInt<DB>::describe,
+                                         &process_pointer},
+          exporters::inst::ProcessorInst{
+              types::st::Sum<
+                  DB,
+                  types::st::Unit<DB>,
+                  typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>::
+                  describe,
+              &process_pointer_content},
+      };
+    } else {
+      return std::array<inst::ProcessorInst, 0>{};
+    }
+  }
+
+ public:
+  using type = typename decltype(choose_type())::type;
+
+  static constexpr std::array<exporters::inst::Field, 0> fields{};
+  static constexpr auto processors = choose_processors();
+
+  static types::st::Unit<DB> getSizeType(
+      Ctx& ctx, const T& t, typename TypeHandler<Ctx, T>::type returnArg) {
+    if constexpr (std::is_pointer_v<T>) {
+      JLOG("ptr val @");
+      JLOGPTR(t);
+      auto r0 = returnArg.write((uintptr_t)t);
+      if (t && ctx.pointers.add((uintptr_t)t)) {
+        return r0.template delegate<1>([&ctx, &t](auto ret) {
+          using U = std::decay_t<std::remove_pointer_t<T>>;
+          if constexpr (oi_is_complete<U>) {
+            return TypeHandler<Ctx, U>::getSizeType(ctx, *t, ret);
+          } else {
+            return ret;
+          }
+        });
+      } else {
+        return r0.template delegate<0>(std::identity());
+      }
+    } else {
+      return returnArg;
+    }
+  }
+};
+)";
 
   code += R"(
-    template <typename Ctx>
-    class TypeHandler<Ctx, void> {
-        using DB = typename Ctx::DataBuffer;
-      public:
-        using type = types::st::Unit<DB>;
+template <typename Ctx>
+class TypeHandler<Ctx, void> {
+  using DB = typename Ctx::DataBuffer;
+
+ public:
+  using type = types::st::Unit<DB>;
+  static constexpr std::array<exporters::inst::Field, 0> fields{};
+  static constexpr std::array<exporters::inst::ProcessorInst, 0> processors{};
+};
 )";
-  if (features[Feature::TreeBuilderV2]) {
-    code +=
-        "static constexpr std::array<exporters::inst::Field, 0> fields{};\n";
-    code +=
-        "static constexpr std::array<exporters::inst::ProcessorInst, 0> "
-        "processors{};\n";
-  }
-  code += "};\n";
 }
 
 ContainerInfo FuncGen::GetOiArrayContainerInfo() {
diff --git a/oi/FuncGen.h b/oi/FuncGen.h
index 33aaf9c..e2b2fba 100644
--- a/oi/FuncGen.h
+++ b/oi/FuncGen.h
@@ -76,7 +76,7 @@ class FuncGen {
 
   static void DefineDataSegmentDataBuffer(std::string& testCode);
   static void DefineBackInserterDataBuffer(std::string& code);
-  static void DefineBasicTypeHandlers(std::string& code, FeatureSet features);
+  static void DefineBasicTypeHandlers(std::string& code);
 
   static ContainerInfo GetOiArrayContainerInfo();
 };
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index 1be10f5..ccce3fa 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -126,12 +126,27 @@ void TopoSorter::visit(Pointer& p) {
     // Typedefs can not be forward declared, so we must sort them before
     // pointers which reference them
     accept(p.pointeeType());
-    return;
+  } else {
+    // Pointers do not create a dependency, but we do still care about the types
+    // they point to, so delay them until the end.
+    acceptAfter(p.pointeeType());
   }
 
-  // Pointers do not create a dependency, but we do still care about the types
-  // they point to, so delay them until the end.
-  acceptAfter(p.pointeeType());
+  sortedTypes_.push_back(p);
+}
+
+void TopoSorter::visit(Reference& r) {
+  if (dynamic_cast<Typedef*>(&r.pointeeType())) {
+    // Typedefs can not be forward declared, so we must sort them before
+    // pointers which reference them
+    accept(r.pointeeType());
+  } else {
+    // Pointers do not create a dependency, but we do still care about the types
+    // they point to, so delay them until the end.
+    acceptAfter(r.pointeeType());
+  }
+
+  sortedTypes_.push_back(r);
 }
 
 void TopoSorter::visit(CaptureKeys& c) {
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 5c78ad1..6eb7083 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -46,6 +46,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Enum& e) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& r) override;
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
   void visit(Incomplete& i) override;
diff --git a/test/integration/anonymous.toml b/test/integration/anonymous.toml
index 70600d1..d1e6286 100644
--- a/test/integration/anonymous.toml
+++ b/test/integration/anonymous.toml
@@ -97,7 +97,6 @@ definitions = '''
       ]}]'''
 
   [cases.anon_struct]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const AnonStructContainer&"]
     setup = '''
       return AnonStructContainer{
@@ -133,6 +132,40 @@ definitions = '''
         }]
       }]
     }]'''
+    expect_json_v2 = '''[{
+      "name":"a0",
+      "typeNames":["ns_anonymous::AnonStructContainer"],
+      "staticSize":8,
+      "exclusiveSize":0,
+      "size":20,
+      "members":[{
+        "name":"anon",
+        "typeNames":["__oi_anon_1"],
+        "staticSize":8,
+        "exclusiveSize":0,
+        "size":20,
+        "members":[{
+          "name":"node",
+          "typeNames":["ns_anonymous::Node*"],
+          "staticSize":8,
+          "exclusiveSize":8,
+          "size":20,
+          "length":1,
+          "capacity":1,
+          "members":[{
+            "name":"*",
+            "typeNames":["ns_anonymous::Node"],
+            "staticSize":12,
+            "exclusiveSize":0,
+            "size":12,
+            "members":[
+              { "name": "a", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "b", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "c", "staticSize": 4, "exclusiveSize": 4, "size": 4 }
+            ]
+        }]
+      }]}
+    ]}]'''
 
   [cases.anon_struct_ptr]
     skip = "We don't support pointer to anon-structs yet" # https://github.com/facebookexperimental/object-introspection/issues/20
@@ -146,7 +179,6 @@ definitions = '''
     features = ["chase-raw-pointers"]
 
   [cases.anon_typedef]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const AnonTypedefContainer&"]
     setup = '''
       return AnonTypedefContainer{
@@ -192,6 +224,34 @@ definitions = '''
         }]
       }]
     }]'''
+    expect_json_v2 = '''[{
+      "staticSize": 8,
+      "exclusiveSize": 0,
+      "size": 20,
+      "members": [{
+        "typeNames":["AnonStruct", "__oi_anon_2"],
+        "staticSize": 8,
+        "exclusiveSize": 0,
+        "size": 20,
+        "members": [{
+          "typeNames": ["ns_anonymous::Node*"],
+          "staticSize": 8,
+          "exclusiveSize": 8,
+          "size": 20,
+          "members": [{
+            "typeNames": ["ns_anonymous::Node"],
+            "staticSize": 12,
+            "exclusiveSize": 0,
+            "size": 12,
+            "members": [
+              { "name": "a", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "b", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "c", "staticSize": 4, "exclusiveSize": 4, "size": 4 }
+            ]
+          }]
+        }]
+      }]
+    }]'''
 
   [cases.anon_union]
     param_types = ["const AnonUnionContainer&"]
@@ -217,7 +277,6 @@ definitions = '''
     }]'''
 
   [cases.nested_anon_struct]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const NestedAnonContainer&"]
     features = ["chase-raw-pointers"]
     setup = 'return NestedAnonContainer{.m = { .v = {.as = {new Node{1, 2, 3}}}}};'
@@ -272,6 +331,64 @@ definitions = '''
         "dynamicSize": 0
       }]
     }]'''
+    expect_json_v2 = '''[{
+      "staticSize":80,
+      "exclusiveSize":0,
+      "size":92,
+      "members":[
+        {
+          "name":"m",
+          "typeNames": ["__oi_anon_1"],
+          "staticSize": 48,
+          "exclusiveSize":0,
+          "size": 60,
+          "members":[
+            {"name":"__oi_anon_0", "typeNames":["__oi_anon_2"], "staticSize":16, "exclusiveSize":16, "size":16},
+            {
+              "name":"v",
+              "typeNames":["__oi_anon_3"],
+              "staticSize":32,
+              "exclusiveSize":4,
+              "size":44,
+              "members":[
+                {"name":"a", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+                {"name":"b", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+                {"name":"c", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+                {"name":"__oi_anon_4", "typeNames":["__oi_anon_4"], "staticSize":8, "exclusiveSize":8, "size":8},
+                {
+                  "name":"as",
+                  "typeNames":["AnonStruct", "__oi_anon_6"],
+                  "staticSize":8,
+                  "exclusiveSize":0,
+                  "size":20,
+                  "members":[{
+                    "name":"node",
+                    "typeNames":["ns_anonymous::Node*"],
+                    "staticSize":8,
+                    "exclusiveSize":8,
+                    "size":20,
+                    "members":[{
+                      "name":"*",
+                      "typeNames":["ns_anonymous::Node"],
+                      "staticSize":12,
+                      "exclusiveSize":0,
+                      "size":12,
+                      "members":[
+                        { "name": "a", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+                        { "name": "b", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+                        { "name": "c", "staticSize": 4, "exclusiveSize": 4, "size": 4 }
+                      ]
+                    }]
+                  }]
+                }
+              ]
+            }
+          ]
+        },
+        {"name":"__oi_anon_1", "typeNames": ["__oi_anon_8"], "staticSize":24, "exclusiveSize":24, "size":24},
+        {"name":"__oi_anon_2", "typeNames": ["__oi_anon_9"], "staticSize":8, "exclusiveSize":8, "size":8}
+      ]
+    }]'''
 
   # This test is disabled due to GCC not supporting it
   # [cases.anon_array]
diff --git a/test/integration/pointers.toml b/test/integration/pointers.toml
index cfd3627..3096cb2 100644
--- a/test/integration/pointers.toml
+++ b/test/integration/pointers.toml
@@ -134,7 +134,6 @@ definitions = '''
 
 
   [cases.struct_primitive_ptrs]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, new int(0), new int(0)};"
     features = ["chase-raw-pointers"]
@@ -147,6 +146,16 @@ definitions = '''
         {"name":"b", "staticSize":8, "exclusiveSize":8, "dynamicSize":4},
         {"name":"c", "staticSize":8, "exclusiveSize":8, "dynamicSize":0}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":4,
+      "size":28,
+      "members":[
+        {"name":"a", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"b", "typeNames":["int32_t*"], "staticSize":8, "exclusiveSize":8, "size":12, "capacity":1, "length":1},
+        {"name":"c", "typeNames":["void*"], "staticSize":8, "exclusiveSize":8, "size":8, "capacity":1, "length":1}
+      ]
+    }]'''
   [cases.struct_primitive_ptrs_no_follow]
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, new int(0), new int(0)};"
@@ -161,7 +170,6 @@ definitions = '''
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_primitive_ptrs_null]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, nullptr, nullptr};"
     features = ["chase-raw-pointers"]
@@ -175,10 +183,19 @@ definitions = '''
         {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":4,
+      "size":24,
+      "members":[
+        {"name":"a", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"b", "typeNames":["int32_t*"], "staticSize":8, "exclusiveSize":8, "size":8, "capacity":1, "length":0},
+        {"name":"c", "typeNames":["void*"], "staticSize":8, "exclusiveSize":8, "size":8, "capacity":1, "length":0}
+      ]
+    }]'''
 
 
   [cases.struct_vector_ptr]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{new std::vector<int>{1,2,3}};"
     features = ["chase-raw-pointers"]
@@ -188,6 +205,22 @@ definitions = '''
       "members":[
         {"name":"vec", "staticSize":8, "dynamicSize":36}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":8,
+      "exclusiveSize":0,
+      "size":44,
+      "members": [
+        {
+          "typeNames":["std::vector<int32_t, std::allocator<int32_t>>*"],
+          "staticSize":8,
+          "exclusiveSize":8,
+          "size":44,
+          "length":1,
+          "capacity":1,
+          "members":[{ "staticSize":24, "exclusiveSize":24, "size":36 }]
+        }
+      ]
+    }]'''
   [cases.struct_vector_ptr_no_follow]
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{new std::vector<int>{1,2,3}};"
@@ -200,7 +233,6 @@ definitions = '''
         {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_vector_ptr_null]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{nullptr};"
     features = ["chase-raw-pointers"]
@@ -212,10 +244,25 @@ definitions = '''
       "members":[
         {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
+      "members": [
+        {
+          "typeNames":["std::vector<int32_t, std::allocator<int32_t>>*"],
+          "staticSize":8,
+          "exclusiveSize":8,
+          "size":8,
+          "length":0,
+          "capacity":1,
+          "members":[]
+        }
+      ]
+    }]'''
 
 
   [cases.vector_of_pointers]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const std::vector<int*>&"]
     setup = "return {{new int(1), nullptr, new int(3)}};"
     features = ["chase-raw-pointers"]
@@ -230,6 +277,18 @@ definitions = '''
         {"staticSize":8, "dynamicSize":0, "pointer":0},
         {"staticSize":8, "dynamicSize":4, "NOT": {"pointer":0}}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":24,
+      "size":56,
+      "length":3,
+      "capacity":3,
+      "members":[
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}},
+        {"staticSize":8, "exclusiveSize":8, "size":8, "pointer":0},
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}}
+      ]
+    }]'''
   [cases.vector_of_pointers_no_follow]
     skip = "pointer field is missing from results" # https://github.com/facebookexperimental/object-introspection/issues/21
     param_types = ["const std::vector<int*>&"]
@@ -260,7 +319,6 @@ definitions = '''
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.feature_config]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const std::vector<int*>&"]
     setup = "return {{new int(1), nullptr, new int(3)}};"
     config_prefix = 'features = ["chase-raw-pointers"]'
@@ -275,3 +333,15 @@ definitions = '''
         {"staticSize":8, "dynamicSize":0, "pointer":0},
         {"staticSize":8, "dynamicSize":4, "NOT": {"pointer":0}}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":24,
+      "size":56,
+      "length":3,
+      "capacity":3,
+      "members":[
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}},
+        {"staticSize":8, "exclusiveSize":8, "size":8, "pointer":0},
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}}
+      ]
+    }]'''
diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml
index 87af9ad..b062043 100644
--- a/test/integration/pointers_incomplete.toml
+++ b/test/integration/pointers_incomplete.toml
@@ -86,7 +86,6 @@ definitions = '''
     expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.containing_struct]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const IncompleteTypeContainer&"]
     setup = "return IncompleteTypeContainer{};"
     features = ["chase-raw-pointers"]
@@ -115,6 +114,32 @@ definitions = '''
         }
       ]
     }]'''
+    expect_json_v2 = '''[{
+      "staticSize": 88,
+      "exclusiveSize": 21,
+      "size": 88,
+      "members": [
+        { "name": "ptrundef", "staticSize": 8, "exclusiveSize": 8, "size": 8 },
+        { "name": "__makePad1", "staticSize": 1, "exclusiveSize": 1, "size": 1 },
+        { "name": "shundef", "staticSize": 16, "exclusiveSize": 16, "size": 16 },
+        { "name": "__makePad2", "staticSize": 1, "exclusiveSize": 1, "size": 1 },
+        { "name": "shoptundef",
+          "staticSize": 24,
+          "exclusiveSize": 24,
+          "size": 24,
+          "length": 0,
+          "capacity": 1
+        },
+        { "name": "__makePad3", "staticSize": 1, "exclusiveSize": 1, "size": 1 },
+        { "name": "optundef",
+          "staticSize": 16,
+          "exclusiveSize": 16,
+          "size": 16,
+          "length": 0,
+          "capacity": 1
+        }
+      ]
+    }]'''
 
   [cases.containing_struct_no_follow]
     param_types = ["const IncompleteTypeContainer&"]
diff --git a/test/test_topo_sorter.cpp b/test/test_topo_sorter.cpp
index c97801f..7c00441 100644
--- a/test/test_topo_sorter.cpp
+++ b/test/test_topo_sorter.cpp
@@ -236,6 +236,22 @@ TEST(TopoSorterTest, Pointers) {
   myclass.members.push_back(Member{mypointer, "ptr", 0});
 
   test({myclass}, R"(
+ClassA*
+MyClass
+ClassA
+)");
+}
+
+TEST(TopoSorterTest, References) {
+  // References do not require pointee types to be defined first
+  auto classA = Class{0, Class::Kind::Class, "ClassA", 69};
+  auto myreference = Reference{1, classA};
+
+  auto myclass = Class{2, Class::Kind::Class, "MyClass", 69};
+  myclass.members.push_back(Member{myreference, "ref", 0});
+
+  test({myclass}, R"(
+ClassA*
 MyClass
 ClassA
 )");
@@ -258,6 +274,7 @@ TEST(TopoSorterTest, PointerCycle) {
   // the same sorted order for ClassA and ClassB.
   for (const auto& input : inputs) {
     test(input, R"(
+ClassA*
 ClassB
 ClassA
 )");
@@ -276,6 +293,7 @@ TEST(TopoSorterTest, PointerToTypedef) {
   test({myclass}, R"(
 ClassA
 aliasA
+aliasA*
 MyClass
 )");
 }

From 4d2eecf6fae339895c22bfd951d2b93cbca5ee4e Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 16 Jan 2024 19:10:16 +0000
Subject: [PATCH 144/188] tbv2: fix pointer codegen

A previous change enabled running OIL tests with specific features enabled.
This highlighted that pointer code generation under TreeBuilder-v2 was very
broken. This change updates pointer code generation to work and enables the
skipped tests. All enabled tests need `expected_json_v2` added to them due to
formatting differences.

Reformatted and rewrote the basic type handler that handles primitives and
pointers. Removed the reliance on `features` to decide whether to generate for
TreeBuilder-v2 as the intermediate features have been removed. Pointers are
treated as containers with a capacity of 1 and a length of 0 if null/a cycle
and 1 if followed. This holds for void pointers where, although they aren't
followed, the length is still set.

There were a couple of other changes needed to enable these tests on TBv2 that
aren't worth their own issues and PRs, I sneaked them in here.

Extra changes:
- Added `Pointer` and `Reference` to TopoSorter so they generate
  `NameProvider` instances. It might be worth visiting the graph differently
  for `NameProvider` as it requires so many instances that others generators do
  not. Will consider that in the future.
- Follow typedefs when calculating exclusive size for a type.

Closes #458.

Test plan:
- CI
- Enabled previously disabled tests.
---
 oi/CodeGen.cpp                            |  35 ++--
 oi/FuncGen.cpp                            | 209 ++++++++++++----------
 oi/FuncGen.h                              |   2 +-
 oi/type_graph/TopoSorter.cpp              |  23 ++-
 oi/type_graph/TopoSorter.h                |   1 +
 test/integration/anonymous.toml           | 123 ++++++++++++-
 test/integration/pointers.toml            |  82 ++++++++-
 test/integration/pointers_incomplete.toml |  27 ++-
 test/test_topo_sorter.cpp                 |  18 ++
 9 files changed, 386 insertions(+), 134 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index a125508..47ffba8 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -195,7 +195,12 @@ void genDecls(const TypeGraph& typeGraph, std::string& code) {
 namespace {
 
 size_t calculateExclusiveSize(const Type& t) {
-  if (const auto* c = dynamic_cast<const Class*>(&t)) {
+  const Type* finalType = &t;
+  while (const auto* td = dynamic_cast<const Typedef*>(finalType)) {
+    finalType = &td->underlyingType();
+  }
+
+  if (const auto* c = dynamic_cast<const Class*>(finalType)) {
     return std::accumulate(
         c->members.cbegin(), c->members.cend(), 0, [](size_t a, const auto& m) {
           if (m.name.starts_with(AddPadding::MemberPrefix))
@@ -203,7 +208,7 @@ size_t calculateExclusiveSize(const Type& t) {
           return a;
         });
   }
-  return t.size();
+  return finalType->size();
 }
 
 }  // namespace
@@ -211,7 +216,7 @@ size_t calculateExclusiveSize(const Type& t) {
 void genNames(const TypeGraph& typeGraph, std::string& code) {
   code += R"(
 template <typename T>
-struct NameProvider {};
+struct NameProvider;
 )";
 
   // TODO: stop types being duplicated at this point and remove this check
@@ -239,6 +244,9 @@ struct ExclusiveSizeProvider {
 )";
 
   for (const Type& t : typeGraph.finalTypes) {
+    if (dynamic_cast<const Typedef*>(&t))
+      continue;
+
     size_t exclusiveSize = calculateExclusiveSize(t);
     if (exclusiveSize != t.size()) {
       code += "template <> struct ExclusiveSizeProvider<";
@@ -1120,23 +1128,6 @@ void addStandardTypeHandlers(TypeGraph& typeGraph,
     }
 )";
 
-  if (features[Feature::TreeBuilderV2]) {
-    code += R"(
-template <typename Ctx, typename T>
-constexpr inst::Field make_field(std::string_view name) {
-  return inst::Field{
-    sizeof(T),
-    ExclusiveSizeProvider<T>::size,
-    name,
-    NameProvider<T>::names,
-    TypeHandler<Ctx, T>::fields,
-    TypeHandler<Ctx, T>::processors,
-    std::is_fundamental_v<T>
-  };
-}
-)";
-  }
-
   // TODO: bit of a hack - making ContainerInfo a node in the type graph and
   // traversing for it would remove the need for this set altogether.
   std::unordered_set<const ContainerInfo*> used{};
@@ -1318,9 +1309,6 @@ void CodeGen::generate(TypeGraph& typeGraph,
     code += "using namespace oi::detail;\n";
     code += "using oi::exporters::ParsedData;\n";
     code += "using namespace oi::exporters;\n";
-    code += "namespace OIInternal {\nnamespace {\n";
-    FuncGen::DefineBasicTypeHandlers(code, config_.features);
-    code += "} // namespace\n} // namespace OIInternal\n";
   }
 
   if (config_.features[Feature::CaptureThriftIsset]) {
@@ -1360,6 +1348,7 @@ void CodeGen::generate(TypeGraph& typeGraph,
   }
 
   if (config_.features[Feature::TreeBuilderV2]) {
+    FuncGen::DefineBasicTypeHandlers(code);
     addStandardTypeHandlers(typeGraph, config_.features, code);
     addTypeHandlers(typeGraph, code);
   } else {
diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index 5ac3abf..fbe6658 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -614,110 +614,127 @@ class BackInserter {
  * pointer's value always, then the value of the pointer if it is unique. void
  * is of type Unit and always stores nothing.
  */
-void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) {
+void FuncGen::DefineBasicTypeHandlers(std::string& code) {
   code += R"(
-    template <typename Ctx, typename T>
-    struct TypeHandler {
-        using DB = typename Ctx::DataBuffer;
-      private:
-        static auto choose_type() {
-            if constexpr(std::is_pointer_v<T>) {
-                return std::type_identity<types::st::Pair<DB,
-                  types::st::VarInt<DB>,
-                  types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>
-                >>();
-            } else {
-                return std::type_identity<types::st::Unit<DB>>();
-            }
-        }
-
-      public:
-        using type = typename decltype(choose_type())::type;
+template <typename Ctx, typename T>
+struct TypeHandler;
 )";
-  if (features[Feature::TreeBuilderV2]) {
-    code += R"(private:
-        static void process_pointer(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
-          el.pointer = std::get<ParsedData::VarInt>(d.val).value;
-        }
-        static void process_pointer_content(result::Element& el, std::function<void(inst::Inst)> stack_ins, ParsedData d) {
-          static constexpr std::array<std::string_view, 1> names{"TODO"};
-          static constexpr auto childField = make_field<Ctx, T>("*");
 
-          const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
-
-          el.container_stats.emplace(result::Element::ContainerStats{ .capacity = 1 });
-
-          if (sum.index == 0)
-            return;
-
-          el.container_stats->length = 1;
-          stack_ins(childField);
-        }
-
-        static constexpr auto choose_fields() {
-          if constexpr(std::is_pointer_v<T>) {
-            return std::array<exporters::inst::Field, 0>{};
-          } else {
-            return std::array<exporters::inst::Field, 0>{};
-          }
-        }
-        static constexpr auto choose_processors() {
-          if constexpr(std::is_pointer_v<T>) {
-            return std::array<inst::ProcessorInst, 2>{
-              {types::st::VarInt<DB>::describe, &process_pointer},
-              {types::st::Sum<DB, types::st::Unit<DB>, typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>::describe, &process_pointer_content},
-            };
-          } else {
-            return std::array<inst::ProcessorInst, 0>{};
-          }
-        }
-      public:
-        static constexpr auto fields = choose_fields();
-        static constexpr auto processors = choose_processors();
+  code += R"(
+template <typename Ctx, typename T>
+constexpr inst::Field make_field(std::string_view name) {
+  return inst::Field{
+      sizeof(T),
+      ExclusiveSizeProvider<T>::size,
+      name,
+      NameProvider<T>::names,
+      TypeHandler<Ctx, T>::fields,
+      TypeHandler<Ctx, T>::processors,
+      std::is_fundamental_v<T>,
+  };
+}
 )";
+  code += R"(
+template <typename Ctx, typename T>
+struct TypeHandler {
+  using DB = typename Ctx::DataBuffer;
+
+ private:
+  static void process_pointer(result::Element& el,
+                              std::function<void(inst::Inst)> stack_ins,
+                              ParsedData d) {
+    el.pointer = std::get<ParsedData::VarInt>(d.val).value;
   }
-  code += R"(
-        static types::st::Unit<DB> getSizeType(
-          Ctx& ctx,
-          const T& t,
-          typename TypeHandler<Ctx, T>::type returnArg) {
-              if constexpr(std::is_pointer_v<T>) {
-                JLOG("ptr val @");
-                JLOGPTR(t);
-                auto r0 = returnArg.write((uintptr_t)t);
-                if (t && ctx.pointers.add((uintptr_t)t)) {
-                  return r0.template delegate<1>([&t](auto ret) {
-                    if constexpr (!std::is_void<std::remove_pointer_t<T>>::value) {
-                      return TypeHandler<Ctx, std::remove_pointer_t<T>>::getSizeType(*t, ret);
-                    } else {
-                      return ret;
-                    }
-                  });
-                } else {
-                  return r0.template delegate<0>(std::identity());
-                }
-              } else {
-                return returnArg;
-              }
-        }
-    };
-  )";
+  static void process_pointer_content(result::Element& el,
+                                      std::function<void(inst::Inst)> stack_ins,
+                                      ParsedData d) {
+    using U = std::decay_t<std::remove_pointer_t<T>>;
+    const ParsedData::Sum& sum = std::get<ParsedData::Sum>(d.val);
+
+    el.container_stats.emplace(result::Element::ContainerStats{ .capacity = 1, .length = 0 });
+    if (sum.index == 0)
+      return;
+    el.container_stats->length = 1;
+
+    if constexpr (oi_is_complete<U>) {
+      static constexpr auto childField = make_field<Ctx, U>("*");
+      stack_ins(childField);
+    }
+  }
+
+  static auto choose_type() {
+    if constexpr (std::is_pointer_v<T>) {
+      return std::type_identity<types::st::Pair<
+          DB,
+          types::st::VarInt<DB>,
+          types::st::Sum<
+              DB,
+              types::st::Unit<DB>,
+              typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>>>();
+    } else {
+      return std::type_identity<types::st::Unit<DB>>();
+    }
+  }
+  static constexpr auto choose_processors() {
+    if constexpr (std::is_pointer_v<T>) {
+      return std::array<inst::ProcessorInst, 2>{
+          exporters::inst::ProcessorInst{types::st::VarInt<DB>::describe,
+                                         &process_pointer},
+          exporters::inst::ProcessorInst{
+              types::st::Sum<
+                  DB,
+                  types::st::Unit<DB>,
+                  typename TypeHandler<Ctx, std::remove_pointer_t<T>>::type>::
+                  describe,
+              &process_pointer_content},
+      };
+    } else {
+      return std::array<inst::ProcessorInst, 0>{};
+    }
+  }
+
+ public:
+  using type = typename decltype(choose_type())::type;
+
+  static constexpr std::array<exporters::inst::Field, 0> fields{};
+  static constexpr auto processors = choose_processors();
+
+  static types::st::Unit<DB> getSizeType(
+      Ctx& ctx, const T& t, typename TypeHandler<Ctx, T>::type returnArg) {
+    if constexpr (std::is_pointer_v<T>) {
+      JLOG("ptr val @");
+      JLOGPTR(t);
+      auto r0 = returnArg.write((uintptr_t)t);
+      if (t && ctx.pointers.add((uintptr_t)t)) {
+        return r0.template delegate<1>([&ctx, &t](auto ret) {
+          using U = std::decay_t<std::remove_pointer_t<T>>;
+          if constexpr (oi_is_complete<U>) {
+            return TypeHandler<Ctx, U>::getSizeType(ctx, *t, ret);
+          } else {
+            return ret;
+          }
+        });
+      } else {
+        return r0.template delegate<0>(std::identity());
+      }
+    } else {
+      return returnArg;
+    }
+  }
+};
+)";
 
   code += R"(
-    template <typename Ctx>
-    class TypeHandler<Ctx, void> {
-        using DB = typename Ctx::DataBuffer;
-      public:
-        using type = types::st::Unit<DB>;
+template <typename Ctx>
+class TypeHandler<Ctx, void> {
+  using DB = typename Ctx::DataBuffer;
+
+ public:
+  using type = types::st::Unit<DB>;
+  static constexpr std::array<exporters::inst::Field, 0> fields{};
+  static constexpr std::array<exporters::inst::ProcessorInst, 0> processors{};
+};
 )";
-  if (features[Feature::TreeBuilderV2]) {
-    code +=
-        "static constexpr std::array<exporters::inst::Field, 0> fields{};\n";
-    code +=
-        "static constexpr std::array<exporters::inst::ProcessorInst, 0> "
-        "processors{};\n";
-  }
-  code += "};\n";
 }
 
 ContainerInfo FuncGen::GetOiArrayContainerInfo() {
diff --git a/oi/FuncGen.h b/oi/FuncGen.h
index 33aaf9c..e2b2fba 100644
--- a/oi/FuncGen.h
+++ b/oi/FuncGen.h
@@ -76,7 +76,7 @@ class FuncGen {
 
   static void DefineDataSegmentDataBuffer(std::string& testCode);
   static void DefineBackInserterDataBuffer(std::string& code);
-  static void DefineBasicTypeHandlers(std::string& code, FeatureSet features);
+  static void DefineBasicTypeHandlers(std::string& code);
 
   static ContainerInfo GetOiArrayContainerInfo();
 };
diff --git a/oi/type_graph/TopoSorter.cpp b/oi/type_graph/TopoSorter.cpp
index 1be10f5..ccce3fa 100644
--- a/oi/type_graph/TopoSorter.cpp
+++ b/oi/type_graph/TopoSorter.cpp
@@ -126,12 +126,27 @@ void TopoSorter::visit(Pointer& p) {
     // Typedefs can not be forward declared, so we must sort them before
     // pointers which reference them
     accept(p.pointeeType());
-    return;
+  } else {
+    // Pointers do not create a dependency, but we do still care about the types
+    // they point to, so delay them until the end.
+    acceptAfter(p.pointeeType());
   }
 
-  // Pointers do not create a dependency, but we do still care about the types
-  // they point to, so delay them until the end.
-  acceptAfter(p.pointeeType());
+  sortedTypes_.push_back(p);
+}
+
+void TopoSorter::visit(Reference& r) {
+  if (dynamic_cast<Typedef*>(&r.pointeeType())) {
+    // Typedefs can not be forward declared, so we must sort them before
+    // pointers which reference them
+    accept(r.pointeeType());
+  } else {
+    // Pointers do not create a dependency, but we do still care about the types
+    // they point to, so delay them until the end.
+    acceptAfter(r.pointeeType());
+  }
+
+  sortedTypes_.push_back(r);
 }
 
 void TopoSorter::visit(CaptureKeys& c) {
diff --git a/oi/type_graph/TopoSorter.h b/oi/type_graph/TopoSorter.h
index 5c78ad1..6eb7083 100644
--- a/oi/type_graph/TopoSorter.h
+++ b/oi/type_graph/TopoSorter.h
@@ -46,6 +46,7 @@ class TopoSorter : public RecursiveVisitor {
   void visit(Enum& e) override;
   void visit(Typedef& td) override;
   void visit(Pointer& p) override;
+  void visit(Reference& r) override;
   void visit(Primitive& p) override;
   void visit(CaptureKeys& p) override;
   void visit(Incomplete& i) override;
diff --git a/test/integration/anonymous.toml b/test/integration/anonymous.toml
index 70600d1..d1e6286 100644
--- a/test/integration/anonymous.toml
+++ b/test/integration/anonymous.toml
@@ -97,7 +97,6 @@ definitions = '''
       ]}]'''
 
   [cases.anon_struct]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const AnonStructContainer&"]
     setup = '''
       return AnonStructContainer{
@@ -133,6 +132,40 @@ definitions = '''
         }]
       }]
     }]'''
+    expect_json_v2 = '''[{
+      "name":"a0",
+      "typeNames":["ns_anonymous::AnonStructContainer"],
+      "staticSize":8,
+      "exclusiveSize":0,
+      "size":20,
+      "members":[{
+        "name":"anon",
+        "typeNames":["__oi_anon_1"],
+        "staticSize":8,
+        "exclusiveSize":0,
+        "size":20,
+        "members":[{
+          "name":"node",
+          "typeNames":["ns_anonymous::Node*"],
+          "staticSize":8,
+          "exclusiveSize":8,
+          "size":20,
+          "length":1,
+          "capacity":1,
+          "members":[{
+            "name":"*",
+            "typeNames":["ns_anonymous::Node"],
+            "staticSize":12,
+            "exclusiveSize":0,
+            "size":12,
+            "members":[
+              { "name": "a", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "b", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "c", "staticSize": 4, "exclusiveSize": 4, "size": 4 }
+            ]
+        }]
+      }]}
+    ]}]'''
 
   [cases.anon_struct_ptr]
     skip = "We don't support pointer to anon-structs yet" # https://github.com/facebookexperimental/object-introspection/issues/20
@@ -146,7 +179,6 @@ definitions = '''
     features = ["chase-raw-pointers"]
 
   [cases.anon_typedef]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const AnonTypedefContainer&"]
     setup = '''
       return AnonTypedefContainer{
@@ -192,6 +224,34 @@ definitions = '''
         }]
       }]
     }]'''
+    expect_json_v2 = '''[{
+      "staticSize": 8,
+      "exclusiveSize": 0,
+      "size": 20,
+      "members": [{
+        "typeNames":["AnonStruct", "__oi_anon_2"],
+        "staticSize": 8,
+        "exclusiveSize": 0,
+        "size": 20,
+        "members": [{
+          "typeNames": ["ns_anonymous::Node*"],
+          "staticSize": 8,
+          "exclusiveSize": 8,
+          "size": 20,
+          "members": [{
+            "typeNames": ["ns_anonymous::Node"],
+            "staticSize": 12,
+            "exclusiveSize": 0,
+            "size": 12,
+            "members": [
+              { "name": "a", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "b", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+              { "name": "c", "staticSize": 4, "exclusiveSize": 4, "size": 4 }
+            ]
+          }]
+        }]
+      }]
+    }]'''
 
   [cases.anon_union]
     param_types = ["const AnonUnionContainer&"]
@@ -217,7 +277,6 @@ definitions = '''
     }]'''
 
   [cases.nested_anon_struct]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const NestedAnonContainer&"]
     features = ["chase-raw-pointers"]
     setup = 'return NestedAnonContainer{.m = { .v = {.as = {new Node{1, 2, 3}}}}};'
@@ -272,6 +331,64 @@ definitions = '''
         "dynamicSize": 0
       }]
     }]'''
+    expect_json_v2 = '''[{
+      "staticSize":80,
+      "exclusiveSize":0,
+      "size":92,
+      "members":[
+        {
+          "name":"m",
+          "typeNames": ["__oi_anon_1"],
+          "staticSize": 48,
+          "exclusiveSize":0,
+          "size": 60,
+          "members":[
+            {"name":"__oi_anon_0", "typeNames":["__oi_anon_2"], "staticSize":16, "exclusiveSize":16, "size":16},
+            {
+              "name":"v",
+              "typeNames":["__oi_anon_3"],
+              "staticSize":32,
+              "exclusiveSize":4,
+              "size":44,
+              "members":[
+                {"name":"a", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+                {"name":"b", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+                {"name":"c", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+                {"name":"__oi_anon_4", "typeNames":["__oi_anon_4"], "staticSize":8, "exclusiveSize":8, "size":8},
+                {
+                  "name":"as",
+                  "typeNames":["AnonStruct", "__oi_anon_6"],
+                  "staticSize":8,
+                  "exclusiveSize":0,
+                  "size":20,
+                  "members":[{
+                    "name":"node",
+                    "typeNames":["ns_anonymous::Node*"],
+                    "staticSize":8,
+                    "exclusiveSize":8,
+                    "size":20,
+                    "members":[{
+                      "name":"*",
+                      "typeNames":["ns_anonymous::Node"],
+                      "staticSize":12,
+                      "exclusiveSize":0,
+                      "size":12,
+                      "members":[
+                        { "name": "a", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+                        { "name": "b", "staticSize": 4, "exclusiveSize": 4, "size": 4 },
+                        { "name": "c", "staticSize": 4, "exclusiveSize": 4, "size": 4 }
+                      ]
+                    }]
+                  }]
+                }
+              ]
+            }
+          ]
+        },
+        {"name":"__oi_anon_1", "typeNames": ["__oi_anon_8"], "staticSize":24, "exclusiveSize":24, "size":24},
+        {"name":"__oi_anon_2", "typeNames": ["__oi_anon_9"], "staticSize":8, "exclusiveSize":8, "size":8}
+      ]
+    }]'''
 
   # This test is disabled due to GCC not supporting it
   # [cases.anon_array]
diff --git a/test/integration/pointers.toml b/test/integration/pointers.toml
index cfd3627..3096cb2 100644
--- a/test/integration/pointers.toml
+++ b/test/integration/pointers.toml
@@ -134,7 +134,6 @@ definitions = '''
 
 
   [cases.struct_primitive_ptrs]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, new int(0), new int(0)};"
     features = ["chase-raw-pointers"]
@@ -147,6 +146,16 @@ definitions = '''
         {"name":"b", "staticSize":8, "exclusiveSize":8, "dynamicSize":4},
         {"name":"c", "staticSize":8, "exclusiveSize":8, "dynamicSize":0}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":4,
+      "size":28,
+      "members":[
+        {"name":"a", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"b", "typeNames":["int32_t*"], "staticSize":8, "exclusiveSize":8, "size":12, "capacity":1, "length":1},
+        {"name":"c", "typeNames":["void*"], "staticSize":8, "exclusiveSize":8, "size":8, "capacity":1, "length":1}
+      ]
+    }]'''
   [cases.struct_primitive_ptrs_no_follow]
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, new int(0), new int(0)};"
@@ -161,7 +170,6 @@ definitions = '''
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_primitive_ptrs_null]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const PrimitivePtrs&"]
     setup = "return PrimitivePtrs{0, nullptr, nullptr};"
     features = ["chase-raw-pointers"]
@@ -175,10 +183,19 @@ definitions = '''
         {"name":"b", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8},
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":4,
+      "size":24,
+      "members":[
+        {"name":"a", "typeNames":["int32_t"], "staticSize":4, "exclusiveSize":4, "size":4},
+        {"name":"b", "typeNames":["int32_t*"], "staticSize":8, "exclusiveSize":8, "size":8, "capacity":1, "length":0},
+        {"name":"c", "typeNames":["void*"], "staticSize":8, "exclusiveSize":8, "size":8, "capacity":1, "length":0}
+      ]
+    }]'''
 
 
   [cases.struct_vector_ptr]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{new std::vector<int>{1,2,3}};"
     features = ["chase-raw-pointers"]
@@ -188,6 +205,22 @@ definitions = '''
       "members":[
         {"name":"vec", "staticSize":8, "dynamicSize":36}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":8,
+      "exclusiveSize":0,
+      "size":44,
+      "members": [
+        {
+          "typeNames":["std::vector<int32_t, std::allocator<int32_t>>*"],
+          "staticSize":8,
+          "exclusiveSize":8,
+          "size":44,
+          "length":1,
+          "capacity":1,
+          "members":[{ "staticSize":24, "exclusiveSize":24, "size":36 }]
+        }
+      ]
+    }]'''
   [cases.struct_vector_ptr_no_follow]
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{new std::vector<int>{1,2,3}};"
@@ -200,7 +233,6 @@ definitions = '''
         {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.struct_vector_ptr_null]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const VectorPtr&"]
     setup = "return VectorPtr{nullptr};"
     features = ["chase-raw-pointers"]
@@ -212,10 +244,25 @@ definitions = '''
       "members":[
         {"name":"vec", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":8,
+      "exclusiveSize":0,
+      "size":8,
+      "members": [
+        {
+          "typeNames":["std::vector<int32_t, std::allocator<int32_t>>*"],
+          "staticSize":8,
+          "exclusiveSize":8,
+          "size":8,
+          "length":0,
+          "capacity":1,
+          "members":[]
+        }
+      ]
+    }]'''
 
 
   [cases.vector_of_pointers]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const std::vector<int*>&"]
     setup = "return {{new int(1), nullptr, new int(3)}};"
     features = ["chase-raw-pointers"]
@@ -230,6 +277,18 @@ definitions = '''
         {"staticSize":8, "dynamicSize":0, "pointer":0},
         {"staticSize":8, "dynamicSize":4, "NOT": {"pointer":0}}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":24,
+      "size":56,
+      "length":3,
+      "capacity":3,
+      "members":[
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}},
+        {"staticSize":8, "exclusiveSize":8, "size":8, "pointer":0},
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}}
+      ]
+    }]'''
   [cases.vector_of_pointers_no_follow]
     skip = "pointer field is missing from results" # https://github.com/facebookexperimental/object-introspection/issues/21
     param_types = ["const std::vector<int*>&"]
@@ -260,7 +319,6 @@ definitions = '''
         {"name":"c", "staticSize":8, "dynamicSize":0, "exclusiveSize":8, "size":8}
       ]}]'''
   [cases.feature_config]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const std::vector<int*>&"]
     setup = "return {{new int(1), nullptr, new int(3)}};"
     config_prefix = 'features = ["chase-raw-pointers"]'
@@ -275,3 +333,15 @@ definitions = '''
         {"staticSize":8, "dynamicSize":0, "pointer":0},
         {"staticSize":8, "dynamicSize":4, "NOT": {"pointer":0}}
       ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":24,
+      "exclusiveSize":24,
+      "size":56,
+      "length":3,
+      "capacity":3,
+      "members":[
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}},
+        {"staticSize":8, "exclusiveSize":8, "size":8, "pointer":0},
+        {"staticSize":8, "exclusiveSize":8, "size":12, "NOT": {"pointer":0}}
+      ]
+    }]'''
diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml
index 87af9ad..b062043 100644
--- a/test/integration/pointers_incomplete.toml
+++ b/test/integration/pointers_incomplete.toml
@@ -86,7 +86,6 @@ definitions = '''
     expect_json_v2 = '[{"staticSize":16, "exclusiveSize":16}]'
 
   [cases.containing_struct]
-    oil_skip = "pointers are broken in tbv2" # https://github.com/facebookexperimental/object-introspection/issues/458
     param_types = ["const IncompleteTypeContainer&"]
     setup = "return IncompleteTypeContainer{};"
     features = ["chase-raw-pointers"]
@@ -115,6 +114,32 @@ definitions = '''
         }
       ]
     }]'''
+    expect_json_v2 = '''[{
+      "staticSize": 88,
+      "exclusiveSize": 21,
+      "size": 88,
+      "members": [
+        { "name": "ptrundef", "staticSize": 8, "exclusiveSize": 8, "size": 8 },
+        { "name": "__makePad1", "staticSize": 1, "exclusiveSize": 1, "size": 1 },
+        { "name": "shundef", "staticSize": 16, "exclusiveSize": 16, "size": 16 },
+        { "name": "__makePad2", "staticSize": 1, "exclusiveSize": 1, "size": 1 },
+        { "name": "shoptundef",
+          "staticSize": 24,
+          "exclusiveSize": 24,
+          "size": 24,
+          "length": 0,
+          "capacity": 1
+        },
+        { "name": "__makePad3", "staticSize": 1, "exclusiveSize": 1, "size": 1 },
+        { "name": "optundef",
+          "staticSize": 16,
+          "exclusiveSize": 16,
+          "size": 16,
+          "length": 0,
+          "capacity": 1
+        }
+      ]
+    }]'''
 
   [cases.containing_struct_no_follow]
     param_types = ["const IncompleteTypeContainer&"]
diff --git a/test/test_topo_sorter.cpp b/test/test_topo_sorter.cpp
index c97801f..7c00441 100644
--- a/test/test_topo_sorter.cpp
+++ b/test/test_topo_sorter.cpp
@@ -236,6 +236,22 @@ TEST(TopoSorterTest, Pointers) {
   myclass.members.push_back(Member{mypointer, "ptr", 0});
 
   test({myclass}, R"(
+ClassA*
+MyClass
+ClassA
+)");
+}
+
+TEST(TopoSorterTest, References) {
+  // References do not require pointee types to be defined first
+  auto classA = Class{0, Class::Kind::Class, "ClassA", 69};
+  auto myreference = Reference{1, classA};
+
+  auto myclass = Class{2, Class::Kind::Class, "MyClass", 69};
+  myclass.members.push_back(Member{myreference, "ref", 0});
+
+  test({myclass}, R"(
+ClassA*
 MyClass
 ClassA
 )");
@@ -258,6 +274,7 @@ TEST(TopoSorterTest, PointerCycle) {
   // the same sorted order for ClassA and ClassB.
   for (const auto& input : inputs) {
     test(input, R"(
+ClassA*
 ClassB
 ClassA
 )");
@@ -276,6 +293,7 @@ TEST(TopoSorterTest, PointerToTypedef) {
   test({myclass}, R"(
 ClassA
 aliasA
+aliasA*
 MyClass
 )");
 }

From 1e99b5b68a1f92a963b7e9fb65a5545cff976439 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 17 Jan 2024 14:05:58 +0000
Subject: [PATCH 145/188] tbv2: fix thrift isset with ClangTypeParser

Some of the logic that makes Thrift isset work for TreeBuilder-v2 in DrgnParser
(JIT OIL) wasn't ported to ClangTypeParser meaning it doesn't work in
Ahead-of-Time (AoT) OIL.

Add the template parameter name reconstruction for enum values to
ClangTypeParser.

Test plan:
- Tested with Thrift isset enabled on an internal type. Doesn't build before,
  does build after.
---
 oi/type_graph/ClangTypeParser.cpp | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index cb33377..29a3a6f 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -235,9 +235,28 @@ std::optional<TemplateParam> ClangTypeParser::enumerateTemplateParam(
     }
     case clang::TemplateArgument::Integral: {
       auto& ty = enumerateType(*p.getIntegralType());
-      llvm::SmallString<32> val;
-      p.getAsIntegral().toString(val);
-      return TemplateParam{ty, std::string(val)};
+
+      std::string value;
+      if (const auto* e = dynamic_cast<const Enum*>(&ty)) {
+        const auto& eTy = llvm::cast<const clang::EnumType>(*p.getIntegralType());
+        for (const auto* enumerator : eTy.getDecl()->enumerators()) {
+          if (enumerator->getInitVal() == p.getAsIntegral()) {
+            value = e->inputName();
+            value += "::";
+            value += enumerator->getName();
+            break;
+          }
+        }
+        if (value.empty()) {
+          throw std::runtime_error("unable to find enum name for value");
+        }
+      } else {
+        llvm::SmallString<32> val;
+        p.getAsIntegral().toString(val);
+        value = std::string{val};
+      }
+
+      return TemplateParam{ty, value};
     }
     case clang::TemplateArgument::Template: {
       return enumerateTemplateTemplateParam(p.getAsTemplate());

From d47fae19d08cb2890ce773ec680ce54d4f4bd1ac Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 17 Jan 2024 14:05:58 +0000
Subject: [PATCH 146/188] tbv2: fix thrift isset with ClangTypeParser

Some of the logic that makes Thrift isset work for TreeBuilder-v2 in DrgnParser
(JIT OIL) wasn't ported to ClangTypeParser meaning it doesn't work in
Ahead-of-Time (AoT) OIL.

Add the template parameter name reconstruction for enum values to
ClangTypeParser.

Test plan:
- Tested with Thrift isset enabled on an internal type. Doesn't build before,
  does build after.
---
 oi/type_graph/ClangTypeParser.cpp | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index cb33377..40f3a69 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -235,9 +235,29 @@ std::optional<TemplateParam> ClangTypeParser::enumerateTemplateParam(
     }
     case clang::TemplateArgument::Integral: {
       auto& ty = enumerateType(*p.getIntegralType());
-      llvm::SmallString<32> val;
-      p.getAsIntegral().toString(val);
-      return TemplateParam{ty, std::string(val)};
+
+      std::string value;
+      if (const auto* e = dynamic_cast<const Enum*>(&ty)) {
+        const auto& eTy =
+            llvm::cast<const clang::EnumType>(*p.getIntegralType());
+        for (const auto* enumerator : eTy.getDecl()->enumerators()) {
+          if (enumerator->getInitVal() == p.getAsIntegral()) {
+            value = e->inputName();
+            value += "::";
+            value += enumerator->getName();
+            break;
+          }
+        }
+        if (value.empty()) {
+          throw std::runtime_error("unable to find enum name for value");
+        }
+      } else {
+        llvm::SmallString<32> val;
+        p.getAsIntegral().toString(val);
+        value = std::string{val};
+      }
+
+      return TemplateParam{ty, value};
     }
     case clang::TemplateArgument::Template: {
       return enumerateTemplateTemplateParam(p.getAsTemplate());

From 8428cb97dab4f5dca8465029f435067834877881 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 18 Jan 2024 16:23:28 +0000
Subject: [PATCH 147/188] tbv2: remove unnecessary copy in Element

`IntrospectionResult::const_iterator` iterates through the `Element`s in an
`IntrospectionResult`. `Element` currently copies the `type_path` which is a
`std::vector<string_view>` every time the iterator is incremented. This is
unnecessary as the data in the vector only changes slightly between iterations.

This change changes the `type_path` field in `Element` to a
`std::span<const std::string_view>`. Doing this previously caused SEGVs because
of the iterator's potential to be copied. To make it possible we do two things:
1. Make all copies explicit using a clone interface as in `ContainerInfo`. This
   prevents accidental copies of an expensive structure.
2. After calling the copy constructor in `clone()` update the `span` in `next_`
   to point at the newly copied structure.

Moves are fine because the `span` points at the allocation of the `vector`, not
the vector itself.

Test plan:
- CI
- `FILTER='OilIntegration.*' make test`
- Ran `OilgenIntegration.std_vector_vector_int_some` which SEGVd with the
  `span` change before and now doesn't. This now passes cleanly with ASAN
  enabled on the target, though isn't available in `main` (only works on my
  machine).
---
 include/oi/IntrospectionResult-inl.h | 16 +++++++++++++++-
 include/oi/IntrospectionResult.h     | 10 ++++++++++
 include/oi/result/Element.h          |  3 +--
 include/oi/result/SizedResult-inl.h  |  9 ++++++---
 oi/IntrospectionResult.cpp           |  5 ++---
 5 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/include/oi/IntrospectionResult-inl.h b/include/oi/IntrospectionResult-inl.h
index 13928b5..a0866a7 100644
--- a/include/oi/IntrospectionResult-inl.h
+++ b/include/oi/IntrospectionResult-inl.h
@@ -45,7 +45,9 @@ inline IntrospectionResult::const_iterator IntrospectionResult::begin() const {
   return cbegin();
 }
 inline IntrospectionResult::const_iterator IntrospectionResult::cbegin() const {
-  return ++const_iterator{buf_.cbegin(), inst_};
+  auto it = const_iterator{buf_.cbegin(), inst_};
+  ++it;
+  return it;
 }
 inline IntrospectionResult::const_iterator IntrospectionResult::end() const {
   return cend();
@@ -70,6 +72,18 @@ inline const result::Element* IntrospectionResult::const_iterator::operator->()
   return &*next_;
 }
 
+inline IntrospectionResult::const_iterator
+IntrospectionResult::const_iterator::clone() const {
+  auto ret{*this};
+
+  // Fix the self referential type_path_ field in next_
+  if (ret.next_.has_value()) {
+    ret.next_->type_path = ret.type_path_;
+  }
+
+  return ret;
+}
+
 inline bool IntrospectionResult::const_iterator::operator==(
     const IntrospectionResult::const_iterator& that) const {
   // Case 1: The next data to read differs, thus the iterators are different.
diff --git a/include/oi/IntrospectionResult.h b/include/oi/IntrospectionResult.h
index 091028f..1675423 100644
--- a/include/oi/IntrospectionResult.h
+++ b/include/oi/IntrospectionResult.h
@@ -43,6 +43,16 @@ class IntrospectionResult {
     const_iterator& operator++();
     const_iterator operator++(int);
 
+   private:
+    const_iterator(const const_iterator&) = default;
+    const_iterator& operator=(const const_iterator& other) = default;
+
+   public:
+    const_iterator(const_iterator&&) = default;
+    const_iterator& operator=(const_iterator&&) = default;
+    // Explicit interface for copying
+    const_iterator clone() const;
+
    private:
     using stack_t =
         std::stack<exporters::inst::Inst, std::vector<exporters::inst::Inst>>;
diff --git a/include/oi/result/Element.h b/include/oi/result/Element.h
index c971236..4a47233 100644
--- a/include/oi/result/Element.h
+++ b/include/oi/result/Element.h
@@ -41,8 +41,7 @@ struct Element {
   };
 
   std::string_view name;
-  std::vector<std::string_view>
-      type_path;  // TODO: should be span<const std::string_view>
+  std::span<const std::string_view> type_path;
   std::span<const std::string_view> type_names;
   size_t static_size;
   size_t exclusive_size;
diff --git a/include/oi/result/SizedResult-inl.h b/include/oi/result/SizedResult-inl.h
index f86ff0c..94418a4 100644
--- a/include/oi/result/SizedResult-inl.h
+++ b/include/oi/result/SizedResult-inl.h
@@ -35,7 +35,9 @@ SizedResult<Res>::SizedResult(Res res) : res_{std::move(res)} {
 
 template <typename Res>
 typename SizedResult<Res>::const_iterator SizedResult<Res>::begin() const {
-  return ++const_iterator{res_.begin(), res_.end()};
+  const_iterator it{res_.begin(), res_.end()};
+  ++it;
+  return it;
 }
 template <typename Res>
 typename SizedResult<Res>::const_iterator SizedResult<Res>::end() const {
@@ -44,7 +46,7 @@ typename SizedResult<Res>::const_iterator SizedResult<Res>::end() const {
 
 template <typename Res>
 SizedResult<Res>::const_iterator::const_iterator(It it, const It& end)
-    : data_{it} {
+    : data_{it.clone()} {
   struct StackEntry {
     size_t index;
     size_t depth;
@@ -75,7 +77,8 @@ SizedResult<Res>::const_iterator::const_iterator(It it, const It& end)
 }
 
 template <typename Res>
-SizedResult<Res>::const_iterator::const_iterator(It end) : data_{end} {
+SizedResult<Res>::const_iterator::const_iterator(It end)
+    : data_{std::move(end)} {
 }
 
 template <typename Res>
diff --git a/oi/IntrospectionResult.cpp b/oi/IntrospectionResult.cpp
index 5310001..4b4839d 100644
--- a/oi/IntrospectionResult.cpp
+++ b/oi/IntrospectionResult.cpp
@@ -70,7 +70,7 @@ IntrospectionResult::const_iterator::operator++() {
           if constexpr (std::is_same_v<T, exporters::inst::Field>) {
             type_path_.emplace_back(ty.name);
             stack_.emplace(exporters::inst::PopTypePath{});
-            next_ = result::Element{
+            next_.emplace(result::Element{
                 .name = ty.name,
                 .type_path = type_path_,
                 .type_names = ty.type_names,
@@ -79,7 +79,7 @@ IntrospectionResult::const_iterator::operator++() {
                 .container_stats = std::nullopt,
                 .is_set_stats = std::nullopt,
                 .is_primitive = ty.is_primitive,
-            };
+            });
 
             for (const auto& [dy, handler] : ty.processors) {
               auto parsed = exporters::ParsedData::parse(data_, dy);
@@ -94,7 +94,6 @@ IntrospectionResult::const_iterator::operator++() {
                       .second;
 
               type_path_.back() = new_name_ref;
-              next_->type_path.back() = new_name_ref;
               next_->name = new_name_ref;
             }
 

From 09dd1cb7e06d8e3efcfd14d15d1040aa3cecd448 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 18 Jan 2024 16:24:15 +0000
Subject: [PATCH 148/188] type_graph: avoid overwriting explicitly set
 alignment

Previously AlignmentCalc calculates the alignment and sets packing for every
type except a member with explicit alignment. Change this to check whether an
alignment has been previously set for a type before calculating it. Use this in
ClangTypeParser where the full alignment of the type is available.

Remove explicitly aligning members by the type because that was previously
reserved for members with explicit alignment. AlignmentCalc will correctly
align a member to the underlying type without this. Explicit member alignment
is still missing, as before this change.

Test plan:
- CI
- Too little. Gets further into a production type than without this change.
---
 oi/type_graph/AlignmentCalc.cpp   | 30 ++++++++++++++++--------------
 oi/type_graph/ClangTypeParser.cpp |  4 ++--
 2 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/oi/type_graph/AlignmentCalc.cpp b/oi/type_graph/AlignmentCalc.cpp
index cf23eff..9abad71 100644
--- a/oi/type_graph/AlignmentCalc.cpp
+++ b/oi/type_graph/AlignmentCalc.cpp
@@ -50,24 +50,26 @@ void AlignmentCalc::accept(Type& type) {
 void AlignmentCalc::visit(Class& c) {
   RecursiveVisitor::visit(c);
 
-  uint64_t alignment = 1;
-  for (auto& member : c.members) {
-    if (member.align == 0) {
-      // If the member does not have an explicit alignment, calculate it from
-      // the member's type.
-      accept(member.type());
-      member.align = member.type().align();
-    }
-    alignment = std::max(alignment, member.align);
+  if (c.align() == 0) {
+    uint64_t alignment = 1;
+    for (auto& member : c.members) {
+      if (member.align == 0) {
+        // If the member does not have an explicit alignment, calculate it from
+        // the member's type.
+        accept(member.type());
+        member.align = member.type().align();
+      }
+      alignment = std::max(alignment, member.align);
 
-    if (member.align != 0 && (member.bitOffset / 8) % member.align != 0) {
-      // Mark as packed if members are not aligned
-      c.setPacked();
+      if (member.align != 0 && (member.bitOffset / 8) % member.align != 0) {
+        // Mark as packed if members are not aligned
+        c.setPacked();
+      }
     }
+
+    c.setAlign(alignment);
   }
 
-  c.setAlign(alignment);
-
   if (c.size() % c.align() != 0) {
     // Mark as packed if there is no tail padding
     c.setPacked();
diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index 40f3a69..5280aa1 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -178,7 +178,7 @@ Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
   if (auto* info = getContainerInfo(fqName)) {
     auto& c = makeType<Container>(ty, *info, size, nullptr);
     enumerateClassTemplateParams(ty, c.templateParams);
-    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)));
+    c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)) / 8);
     return c;
   }
 
@@ -197,6 +197,7 @@ Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
 
   auto& c = makeType<Class>(
       ty, kind, std::move(name), std::move(fqName), size, virtuality);
+  c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)) / 8);
 
   enumerateClassTemplateParams(ty, c.templateParams);
   // enumerateClassParents(type, c.parents);
@@ -317,7 +318,6 @@ void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
 
     auto& mtype = enumerateType(*qualType);
     Member m{mtype, std::move(member_name), offset_in_bits, size_in_bits};
-    m.align = decl->getASTContext().getTypeAlign(qualType) / 8;
     members.push_back(m);
   }
 

From 2dc53b6b548b50993313874c898a94766c9243bc Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 19 Jan 2024 18:56:07 +0000
Subject: [PATCH 149/188] tbv2: fix Thrift isset lookups with padding

Thrift isset was failing with a SEGV if the struct contained padding. This is
because we indexed the `isset_indexes` data structure using our field index
rather than the index of the field in Thrift. This then gave a rubbish index
for any exceeding which happens if we have added padding in the middle of the
struct, and this index was looked up in the bitset which can cause a SEGV.

Track a new index `thriftFieldIdx` which is only incremented if we've looked up
a Thrift index.

Namespaced the generated Thrift structs while I was there. This isn't necessary
anymore but cleans things up.

Test plan:
- Added a test case with lots of padding. These don't run in the CI but it
  passes locally.
- `FILTER='OilIntegration.*' make test` - no failures
- `FILTER='OidIntegration.*' make test` - no new failures
---
 oi/CodeGen.cpp                     |  6 +-
 test/integration/thrift_isset.toml | 92 ++++++++++++++++++++++++------
 2 files changed, 78 insertions(+), 20 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 47ffba8..fda52f5 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -505,6 +505,7 @@ void CodeGen::getClassSizeFuncConcrete(std::string_view funcName,
             c.fqName() + ">;\n";
   }
 
+  size_t thriftFieldIdx = 0;
   for (size_t i = 0; i < c.members.size(); i++) {
     const auto& member = c.members[i];
     if (member.name.starts_with(AddPadding::MemberPrefix))
@@ -514,7 +515,7 @@ void CodeGen::getClassSizeFuncConcrete(std::string_view funcName,
       // Capture Thrift's isset value for each field, except for __isset
       // itself
       std::string issetIdxStr =
-          "thrift_data::isset_indexes[" + std::to_string(i) + "]";
+          "thrift_data::isset_indexes[" + std::to_string(thriftFieldIdx++) + "]";
       code += "  if (&thrift_data::isset_indexes != nullptr && " + issetIdxStr +
               " != -1) {\n";
       code += "    SAVE_DATA(t." + thriftIssetMember->name + ".get(" +
@@ -684,6 +685,7 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
 
   size_t emptySize = code.size();
   size_t lastNonPaddingElement = getLastNonPaddingMemberIndex(c.members);
+  size_t thriftFieldIdx = 0;
   for (size_t i = 0; i < lastNonPaddingElement + 1; i++) {
     const auto& member = c.members[i];
     if (member.name.starts_with(AddPadding::MemberPrefix)) {
@@ -696,7 +698,7 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
 
     if (thriftIssetMember != nullptr && thriftIssetMember != &member) {
       code += "\n      .write(getThriftIsset(t, ";
-      code += std::to_string(i);
+      code += std::to_string(thriftFieldIdx++);
       code += "))";
     }
 
diff --git a/test/integration/thrift_isset.toml b/test/integration/thrift_isset.toml
index cdbcdb5..0925a88 100644
--- a/test/integration/thrift_isset.toml
+++ b/test/integration/thrift_isset.toml
@@ -1,4 +1,6 @@
 thrift_definitions = '''
+  namespace cpp2 ns_thrift_isset
+
   include "thrift/annotation/cpp.thrift"
   include "thrift/annotation/thrift.thrift"
 
@@ -8,6 +10,18 @@ thrift_definitions = '''
     3: optional i32 c;
   }
 
+  struct MyThriftStructUnpackedPadded {
+    1: optional i32 a;
+    2: optional i64 b;
+    3: optional i32 c;
+    4: optional i64 d;
+    5: optional i32 e;
+    6: optional i64 f;
+    7: optional i32 g;
+    8: optional i64 h;
+    13: optional i32 i;
+  }
+
   @cpp.PackIsset
   struct MyThriftStructPacked {
     1: optional i32 a;
@@ -56,7 +70,7 @@ thrift_definitions = '''
   }
 '''
 raw_definitions = '''
-namespace cpp2 {
+namespace ns_thrift_isset {
   MyThriftStructBoxed::MyThriftStructBoxed() :
         __fbthrift_field_b(),
         __fbthrift_field_c(),
@@ -64,7 +78,7 @@ namespace cpp2 {
         __fbthrift_field_e() {
   }
   MyThriftStructBoxed::~MyThriftStructBoxed() {}
-  MyThriftStructBoxed::MyThriftStructBoxed(::cpp2::MyThriftStructBoxed&& other) noexcept :
+  MyThriftStructBoxed::MyThriftStructBoxed(MyThriftStructBoxed&& other) noexcept :
       __fbthrift_field_a(std::move(other.__fbthrift_field_a)),
       __fbthrift_field_b(std::move(other.__fbthrift_field_b)),
       __fbthrift_field_c(std::move(other.__fbthrift_field_c)),
@@ -72,14 +86,14 @@ namespace cpp2 {
       __fbthrift_field_e(std::move(other.__fbthrift_field_e)),
       __isset(other.__isset) {
   }
-} // namespace cpp2
+} // namespace ns_thrift_isset
 '''
 
 [cases]
   [cases.unpacked]
-    param_types = ["const cpp2::MyThriftStructUnpacked&"]
+    param_types = ["const MyThriftStructUnpacked&"]
     setup = '''
-      cpp2::MyThriftStructUnpacked ret;
+      MyThriftStructUnpacked ret;
       ret.a_ref() = 1;
       ret.c_ref() = 1;
       return ret;
@@ -106,10 +120,52 @@ namespace cpp2 {
       ]
     }]'''
 
-  [cases.packed]
-    param_types = ["const cpp2::MyThriftStructPacked&"]
+  [cases.unpacked_padded]
+    param_types = ["const MyThriftStructUnpackedPadded&"]
     setup = '''
-      cpp2::MyThriftStructPacked ret;
+      MyThriftStructUnpackedPadded ret;
+      ret.a_ref() = 1;
+      ret.c_ref() = 1;
+      return ret;
+    '''
+    features = ["capture-thrift-isset"]
+    expect_json = '''[{
+      "staticSize":80,
+      "dynamicSize":0,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "isset":true},
+        {"name":"__fbthrift_field_b", "staticSize":8, "isset":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "isset":true},
+        {"name":"__fbthrift_field_d", "staticSize":8, "isset":false},
+        {"name":"__fbthrift_field_e", "staticSize":4, "isset":false},
+        {"name":"__fbthrift_field_f", "staticSize":8, "isset":false},
+        {"name":"__fbthrift_field_g", "staticSize":4, "isset":false},
+        {"name":"__fbthrift_field_h", "staticSize":8, "isset":false},
+        {"name":"__fbthrift_field_i", "staticSize":4, "isset":false},
+        {"name":"__isset", "staticSize":9}
+      ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":80,
+      "exclusiveSize":19,
+      "size":80,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_b", "staticSize":8, "is_set":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_d", "staticSize":8, "is_set":false},
+        {"name":"__fbthrift_field_e", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_f", "staticSize":8, "is_set":false},
+        {"name":"__fbthrift_field_g", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_h", "staticSize":8, "is_set":false},
+        {"name":"__fbthrift_field_i", "staticSize":4, "is_set":false},
+        {"name":"__isset", "staticSize":9, "NOT":"is_set"}
+      ]
+    }]'''
+
+  [cases.packed]
+    param_types = ["const MyThriftStructPacked&"]
+    setup = '''
+      MyThriftStructPacked ret;
       ret.a_ref() = 1;
       ret.c_ref() = 1;
       ret.d_ref() = 1;
@@ -154,9 +210,9 @@ namespace cpp2 {
       ]}]'''
 
   [cases.packed_non_atomic]
-    param_types = ["const cpp2::MyThriftStructPackedNonAtomic&"]
+    param_types = ["const MyThriftStructPackedNonAtomic&"]
     setup = '''
-      cpp2::MyThriftStructPackedNonAtomic ret;
+      MyThriftStructPackedNonAtomic ret;
       ret.a_ref() = 1;
       ret.c_ref() = 1;
       return ret;
@@ -185,9 +241,9 @@ namespace cpp2 {
       ]}]'''
 
   [cases.out_of_order]
-    param_types = ["const cpp2::MyThriftStructOutOfOrder&"]
+    param_types = ["const MyThriftStructOutOfOrder&"]
     setup = '''
-      cpp2::MyThriftStructOutOfOrder ret;
+      MyThriftStructOutOfOrder ret;
       ret.b_ref() = 1;
       return ret;
     '''
@@ -213,9 +269,9 @@ namespace cpp2 {
       ]}]'''
 
   [cases.required]
-    param_types = ["const cpp2::MyThriftStructRequired&"]
+    param_types = ["const MyThriftStructRequired&"]
     setup = '''
-      cpp2::MyThriftStructRequired ret;
+      MyThriftStructRequired ret;
       ret.b_ref() = 1;
       ret.f_ref() = 1;
       return ret;
@@ -248,9 +304,9 @@ namespace cpp2 {
       ]}]'''
 
   [cases.box]
-    param_types = ["const cpp2::MyThriftStructBoxed&"]
+    param_types = ["const MyThriftStructBoxed&"]
     setup = '''
-      cpp2::MyThriftStructBoxed ret;
+      MyThriftStructBoxed ret;
       ret.d_ref() = 1;
       ret.e_ref() = 1;
       return ret;
@@ -281,9 +337,9 @@ namespace cpp2 {
       ]}]'''
 
   [cases.no_capture]
-    param_types = ["const cpp2::MyThriftStructBoxed&"]
+    param_types = ["const MyThriftStructBoxed&"]
     setup = '''
-      cpp2::MyThriftStructBoxed ret;
+      MyThriftStructBoxed ret;
       ret.d_ref() = 1;
       ret.e_ref() = 1;
       return ret;

From d812c4eeed27915aba396edf336664c86d988013 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 19 Jan 2024 18:56:07 +0000
Subject: [PATCH 150/188] tbv2: fix Thrift isset lookups with padding

Thrift isset was failing with a SEGV if the struct contained padding. This is
because we indexed the `isset_indexes` data structure using our field index
rather than the index of the field in Thrift. This then gave a rubbish index
for any exceeding which happens if we have added padding in the middle of the
struct, and this index was looked up in the bitset which can cause a SEGV.

Track a new index `thriftFieldIdx` which is only incremented if we've looked up
a Thrift index.

Namespaced the generated Thrift structs while I was there. This isn't necessary
anymore but cleans things up.

Test plan:
- Added a test case with lots of padding. These don't run in the CI but it
  passes locally.
- `FILTER='OilIntegration.*' make test` - no failures
- `FILTER='OidIntegration.*' make test` - no new failures
---
 oi/CodeGen.cpp                     |  8 ++-
 test/integration/thrift_isset.toml | 92 ++++++++++++++++++++++++------
 2 files changed, 79 insertions(+), 21 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 47ffba8..df7340b 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -505,6 +505,7 @@ void CodeGen::getClassSizeFuncConcrete(std::string_view funcName,
             c.fqName() + ">;\n";
   }
 
+  size_t thriftFieldIdx = 0;
   for (size_t i = 0; i < c.members.size(); i++) {
     const auto& member = c.members[i];
     if (member.name.starts_with(AddPadding::MemberPrefix))
@@ -513,8 +514,8 @@ void CodeGen::getClassSizeFuncConcrete(std::string_view funcName,
     if (thriftIssetMember && thriftIssetMember != &member) {
       // Capture Thrift's isset value for each field, except for __isset
       // itself
-      std::string issetIdxStr =
-          "thrift_data::isset_indexes[" + std::to_string(i) + "]";
+      std::string issetIdxStr = "thrift_data::isset_indexes[" +
+                                std::to_string(thriftFieldIdx++) + "]";
       code += "  if (&thrift_data::isset_indexes != nullptr && " + issetIdxStr +
               " != -1) {\n";
       code += "    SAVE_DATA(t." + thriftIssetMember->name + ".get(" +
@@ -684,6 +685,7 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
 
   size_t emptySize = code.size();
   size_t lastNonPaddingElement = getLastNonPaddingMemberIndex(c.members);
+  size_t thriftFieldIdx = 0;
   for (size_t i = 0; i < lastNonPaddingElement + 1; i++) {
     const auto& member = c.members[i];
     if (member.name.starts_with(AddPadding::MemberPrefix)) {
@@ -696,7 +698,7 @@ void CodeGen::genClassTraversalFunction(const Class& c, std::string& code) {
 
     if (thriftIssetMember != nullptr && thriftIssetMember != &member) {
       code += "\n      .write(getThriftIsset(t, ";
-      code += std::to_string(i);
+      code += std::to_string(thriftFieldIdx++);
       code += "))";
     }
 
diff --git a/test/integration/thrift_isset.toml b/test/integration/thrift_isset.toml
index cdbcdb5..0925a88 100644
--- a/test/integration/thrift_isset.toml
+++ b/test/integration/thrift_isset.toml
@@ -1,4 +1,6 @@
 thrift_definitions = '''
+  namespace cpp2 ns_thrift_isset
+
   include "thrift/annotation/cpp.thrift"
   include "thrift/annotation/thrift.thrift"
 
@@ -8,6 +10,18 @@ thrift_definitions = '''
     3: optional i32 c;
   }
 
+  struct MyThriftStructUnpackedPadded {
+    1: optional i32 a;
+    2: optional i64 b;
+    3: optional i32 c;
+    4: optional i64 d;
+    5: optional i32 e;
+    6: optional i64 f;
+    7: optional i32 g;
+    8: optional i64 h;
+    13: optional i32 i;
+  }
+
   @cpp.PackIsset
   struct MyThriftStructPacked {
     1: optional i32 a;
@@ -56,7 +70,7 @@ thrift_definitions = '''
   }
 '''
 raw_definitions = '''
-namespace cpp2 {
+namespace ns_thrift_isset {
   MyThriftStructBoxed::MyThriftStructBoxed() :
         __fbthrift_field_b(),
         __fbthrift_field_c(),
@@ -64,7 +78,7 @@ namespace cpp2 {
         __fbthrift_field_e() {
   }
   MyThriftStructBoxed::~MyThriftStructBoxed() {}
-  MyThriftStructBoxed::MyThriftStructBoxed(::cpp2::MyThriftStructBoxed&& other) noexcept :
+  MyThriftStructBoxed::MyThriftStructBoxed(MyThriftStructBoxed&& other) noexcept :
       __fbthrift_field_a(std::move(other.__fbthrift_field_a)),
       __fbthrift_field_b(std::move(other.__fbthrift_field_b)),
       __fbthrift_field_c(std::move(other.__fbthrift_field_c)),
@@ -72,14 +86,14 @@ namespace cpp2 {
       __fbthrift_field_e(std::move(other.__fbthrift_field_e)),
       __isset(other.__isset) {
   }
-} // namespace cpp2
+} // namespace ns_thrift_isset
 '''
 
 [cases]
   [cases.unpacked]
-    param_types = ["const cpp2::MyThriftStructUnpacked&"]
+    param_types = ["const MyThriftStructUnpacked&"]
     setup = '''
-      cpp2::MyThriftStructUnpacked ret;
+      MyThriftStructUnpacked ret;
       ret.a_ref() = 1;
       ret.c_ref() = 1;
       return ret;
@@ -106,10 +120,52 @@ namespace cpp2 {
       ]
     }]'''
 
-  [cases.packed]
-    param_types = ["const cpp2::MyThriftStructPacked&"]
+  [cases.unpacked_padded]
+    param_types = ["const MyThriftStructUnpackedPadded&"]
     setup = '''
-      cpp2::MyThriftStructPacked ret;
+      MyThriftStructUnpackedPadded ret;
+      ret.a_ref() = 1;
+      ret.c_ref() = 1;
+      return ret;
+    '''
+    features = ["capture-thrift-isset"]
+    expect_json = '''[{
+      "staticSize":80,
+      "dynamicSize":0,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "isset":true},
+        {"name":"__fbthrift_field_b", "staticSize":8, "isset":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "isset":true},
+        {"name":"__fbthrift_field_d", "staticSize":8, "isset":false},
+        {"name":"__fbthrift_field_e", "staticSize":4, "isset":false},
+        {"name":"__fbthrift_field_f", "staticSize":8, "isset":false},
+        {"name":"__fbthrift_field_g", "staticSize":4, "isset":false},
+        {"name":"__fbthrift_field_h", "staticSize":8, "isset":false},
+        {"name":"__fbthrift_field_i", "staticSize":4, "isset":false},
+        {"name":"__isset", "staticSize":9}
+      ]}]'''
+    expect_json_v2 = '''[{
+      "staticSize":80,
+      "exclusiveSize":19,
+      "size":80,
+      "members":[
+        {"name":"__fbthrift_field_a", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_b", "staticSize":8, "is_set":false},
+        {"name":"__fbthrift_field_c", "staticSize":4, "is_set":true},
+        {"name":"__fbthrift_field_d", "staticSize":8, "is_set":false},
+        {"name":"__fbthrift_field_e", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_f", "staticSize":8, "is_set":false},
+        {"name":"__fbthrift_field_g", "staticSize":4, "is_set":false},
+        {"name":"__fbthrift_field_h", "staticSize":8, "is_set":false},
+        {"name":"__fbthrift_field_i", "staticSize":4, "is_set":false},
+        {"name":"__isset", "staticSize":9, "NOT":"is_set"}
+      ]
+    }]'''
+
+  [cases.packed]
+    param_types = ["const MyThriftStructPacked&"]
+    setup = '''
+      MyThriftStructPacked ret;
       ret.a_ref() = 1;
       ret.c_ref() = 1;
       ret.d_ref() = 1;
@@ -154,9 +210,9 @@ namespace cpp2 {
       ]}]'''
 
   [cases.packed_non_atomic]
-    param_types = ["const cpp2::MyThriftStructPackedNonAtomic&"]
+    param_types = ["const MyThriftStructPackedNonAtomic&"]
     setup = '''
-      cpp2::MyThriftStructPackedNonAtomic ret;
+      MyThriftStructPackedNonAtomic ret;
       ret.a_ref() = 1;
       ret.c_ref() = 1;
       return ret;
@@ -185,9 +241,9 @@ namespace cpp2 {
       ]}]'''
 
   [cases.out_of_order]
-    param_types = ["const cpp2::MyThriftStructOutOfOrder&"]
+    param_types = ["const MyThriftStructOutOfOrder&"]
     setup = '''
-      cpp2::MyThriftStructOutOfOrder ret;
+      MyThriftStructOutOfOrder ret;
       ret.b_ref() = 1;
       return ret;
     '''
@@ -213,9 +269,9 @@ namespace cpp2 {
       ]}]'''
 
   [cases.required]
-    param_types = ["const cpp2::MyThriftStructRequired&"]
+    param_types = ["const MyThriftStructRequired&"]
     setup = '''
-      cpp2::MyThriftStructRequired ret;
+      MyThriftStructRequired ret;
       ret.b_ref() = 1;
       ret.f_ref() = 1;
       return ret;
@@ -248,9 +304,9 @@ namespace cpp2 {
       ]}]'''
 
   [cases.box]
-    param_types = ["const cpp2::MyThriftStructBoxed&"]
+    param_types = ["const MyThriftStructBoxed&"]
     setup = '''
-      cpp2::MyThriftStructBoxed ret;
+      MyThriftStructBoxed ret;
       ret.d_ref() = 1;
       ret.e_ref() = 1;
       return ret;
@@ -281,9 +337,9 @@ namespace cpp2 {
       ]}]'''
 
   [cases.no_capture]
-    param_types = ["const cpp2::MyThriftStructBoxed&"]
+    param_types = ["const MyThriftStructBoxed&"]
     setup = '''
-      cpp2::MyThriftStructBoxed ret;
+      MyThriftStructBoxed ret;
       ret.d_ref() = 1;
       ret.e_ref() = 1;
       return ret;

From 523a9a4c707181b0df59b0cbbc0052639a81f0d7 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 31 Jan 2024 17:13:43 +0000
Subject: [PATCH 151/188] tbv2: use std::decay_t with smart pointers

CodeGen v2 permits template parameters to be qualified. This means that if we
call `make_field` with a template parameter it will be qualified. However, we
don't qualify the types when generating meta functions such as `NameProvider`
and `TypeHandler`. This means these qualified types don't match up with the
expected type.

Use `std::decay_t` when forwarding the type to `NameProvider` and `TypeHandler`
so they're always the base type that they were generated with. Most of this is
covered by `make_field`, but there are direct references to `TypeHandler<Ctx,
T>` in a lot of `TypeHandler::type` fields. Fix the problematic types manually
for now, there may need to be a better solution with meta functions for this in
the future.

Test Plan:
- CI
- Added a test for `std::unique_ptr<const uint64_t>` to exercise this. Failed
  before, passes after.
- Added a test for `std::unique_ptr<const std::vector<uint64_t>>` to test a
  non-primitive type. Failed before, passes after.
---
 oi/FuncGen.cpp                      |  8 +--
 test/integration/std_smart_ptr.toml | 97 +++++++++++++++++++++++++++++
 types/shrd_ptr_type.toml            |  2 +-
 types/uniq_ptr_type.toml            |  2 +-
 4 files changed, 103 insertions(+), 6 deletions(-)

diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp
index fbe6658..11298ca 100644
--- a/oi/FuncGen.cpp
+++ b/oi/FuncGen.cpp
@@ -625,11 +625,11 @@ template <typename Ctx, typename T>
 constexpr inst::Field make_field(std::string_view name) {
   return inst::Field{
       sizeof(T),
-      ExclusiveSizeProvider<T>::size,
+      ExclusiveSizeProvider<std::decay_t<T>>::size,
       name,
-      NameProvider<T>::names,
-      TypeHandler<Ctx, T>::fields,
-      TypeHandler<Ctx, T>::processors,
+      NameProvider<std::decay_t<T>>::names,
+      TypeHandler<Ctx, std::decay_t<T>>::fields,
+      TypeHandler<Ctx, std::decay_t<T>>::processors,
       std::is_fundamental_v<T>,
   };
 }
diff --git a/test/integration/std_smart_ptr.toml b/test/integration/std_smart_ptr.toml
index c9ab383..d0bddb6 100644
--- a/test/integration/std_smart_ptr.toml
+++ b/test/integration/std_smart_ptr.toml
@@ -32,6 +32,31 @@ definitions = '''
       }
     ]
     '''
+  [cases.unique_ptr_const_uint64_empty]
+    param_types = ["std::unique_ptr<const std::uint64_t>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 8,
+        "dynamicSize": 0,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 8
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 8,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.unique_ptr_uint64_present]
     param_types = ["std::unique_ptr<std::uint64_t>&"]
     setup = "return {std::make_unique<std::uint64_t>(64)};"
@@ -82,6 +107,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.unique_ptr_const_vector_empty]
+    param_types = ["std::unique_ptr<const std::vector<std::uint64_t>>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 8,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 24
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 8,
+        "exclusiveSize": 8,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.unique_ptr_vector_present]
     param_types = ["std::unique_ptr<std::vector<std::uint64_t>>&"]
     setup = "return {std::make_unique<std::vector<std::uint64_t>>(std::initializer_list<std::uint64_t>({1,2,3,4,5}))};"
@@ -188,6 +237,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.shared_ptr_const_uint64_empty]
+    param_types = ["std::shared_ptr<const std::uint64_t>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 16,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 8
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 16,
+        "exclusiveSize": 16,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.shared_ptr_uint64_present]
     param_types = ["std::shared_ptr<std::uint64_t>&"]
     setup = "return std::make_shared<std::uint64_t>(64);"
@@ -241,6 +314,30 @@ definitions = '''
       }
     ]
     '''
+  [cases.shared_ptr_const_vector_empty]
+    param_types = ["std::shared_ptr<const std::vector<std::uint64_t>>&"]
+    setup = "return {nullptr};"
+    expect_json = '''
+    [
+      {
+        "staticSize": 16,
+        "dynamicSize": 0,
+        "length": 0,
+        "capacity": 1,
+        "elementStaticSize": 24
+      }
+    ]
+    '''
+    expect_json_v2 = '''
+    [
+      {
+        "staticSize": 16,
+        "exclusiveSize": 16,
+        "length": 0,
+        "capacity": 1
+      }
+    ]
+    '''
   [cases.shared_ptr_vector_present]
     param_types = ["std::shared_ptr<std::vector<std::uint64_t>>&"]
     setup = "return std::make_shared<std::vector<std::uint64_t>>(std::initializer_list<std::uint64_t>({1,2,3,4,5}));"
diff --git a/types/shrd_ptr_type.toml b/types/shrd_ptr_type.toml
index 3439bf3..ab42eef 100644
--- a/types/shrd_ptr_type.toml
+++ b/types/shrd_ptr_type.toml
@@ -60,7 +60,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<Ctx, T0>::type>
+  typename TypeHandler<Ctx, std::decay_t<T0>>::type>
 """
 func = """
 #ifdef __GLIBCXX__
diff --git a/types/uniq_ptr_type.toml b/types/uniq_ptr_type.toml
index f71cff4..90663a4 100644
--- a/types/uniq_ptr_type.toml
+++ b/types/uniq_ptr_type.toml
@@ -61,7 +61,7 @@ el.pointer = std::get<ParsedData::VarInt>(d.val).value;
 type = """
 types::st::Sum<DB,
   types::st::Unit<DB>,
-  typename TypeHandler<Ctx, T0>::type>
+  typename TypeHandler<Ctx, std::decay_t<T0>>::type>
 """
 func = """
 auto sum = std::get<ParsedData::Sum>(d.val);

From 8899570a85971f17df6b4173acfd650d0c1cde80 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Thu, 1 Feb 2024 13:30:31 +0000
Subject: [PATCH 152/188] clangparser: add support for parents

TODO: Check the assumption that a "base" always has a Type that can be cast to
RecordType.
TODO: Check the assumption that a "base" always has a Decl that can be cast to
CXXRecordDecl.

Add basic support for class parents. Focus purely on bases for now and ignore
vbases.

Test Plan:
- Tested with a simple example. Base containing a long and a class containing a
  float. Both fields appear in the final flattened code.
---
 oi/type_graph/ClangTypeParser.cpp | 32 ++++++++++++++++++++++++++++++-
 oi/type_graph/ClangTypeParser.h   |  2 ++
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/oi/type_graph/ClangTypeParser.cpp b/oi/type_graph/ClangTypeParser.cpp
index c36fad7..0549289 100644
--- a/oi/type_graph/ClangTypeParser.cpp
+++ b/oi/type_graph/ClangTypeParser.cpp
@@ -17,8 +17,10 @@
 
 #include <clang/AST/ASTContext.h>
 #include <clang/AST/Decl.h>
+#include <clang/AST/DeclCXX.h>
 #include <clang/AST/DeclTemplate.h>
 #include <clang/AST/QualTypeNames.h>
+#include <clang/AST/RecordLayout.h>
 #include <clang/AST/Type.h>
 #include <clang/Basic/DiagnosticSema.h>
 #include <clang/Sema/Sema.h>
@@ -199,7 +201,7 @@ Type& ClangTypeParser::enumerateClass(const clang::RecordType& ty) {
   c.setAlign(ast->getTypeAlign(clang::QualType(&ty, 0)) / 8);
 
   enumerateClassTemplateParams(ty, c.templateParams);
-  // enumerateClassParents(type, c.parents);
+  enumerateClassParents(ty, c.parents);
   enumerateClassMembers(ty, c.members);
   // enumerateClassFunctions(type, c.functions);
 
@@ -298,6 +300,34 @@ std::optional<TemplateParam> ClangTypeParser::enumerateTemplateTemplateParam(
   }
 }
 
+void ClangTypeParser::enumerateClassParents(const clang::RecordType& ty,
+                                            std::vector<Parent>& parents) {
+  assert(parents.empty());
+
+  auto* decl = ty.getDecl();
+  auto* cxxDecl = llvm::dyn_cast<clang::CXXRecordDecl>(decl);
+  if (cxxDecl == nullptr)
+    return;
+
+  const auto& layout = decl->getASTContext().getASTRecordLayout(decl);
+  for (const auto& base : cxxDecl->bases()) {
+    auto baseType = base.getType().getDesugaredType(decl->getASTContext());
+    const auto* baseRecordType = llvm::dyn_cast<clang::RecordType>(&*baseType);
+    if (baseRecordType == nullptr)
+      continue;
+
+    auto* baseDecl = baseRecordType->getDecl();
+    auto* baseCxxDecl = llvm::dyn_cast<clang::CXXRecordDecl>(baseDecl);
+    if (baseCxxDecl == nullptr)
+      continue;
+
+    auto offset = layout.getBaseClassOffset(baseCxxDecl).getQuantity();
+    auto& ptype = enumerateType(*baseType);
+    Parent p{ptype, static_cast<uint64_t>(offset * 8)};
+    parents.push_back(p);
+  }
+}
+
 void ClangTypeParser::enumerateClassMembers(const clang::RecordType& ty,
                                             std::vector<Member>& members) {
   assert(members.empty());
diff --git a/oi/type_graph/ClangTypeParser.h b/oi/type_graph/ClangTypeParser.h
index 9b27d64..98f0f67 100644
--- a/oi/type_graph/ClangTypeParser.h
+++ b/oi/type_graph/ClangTypeParser.h
@@ -50,6 +50,7 @@ class Array;
 class Class;
 class Enum;
 class Member;
+struct Parent;
 class Primitive;
 class Reference;
 class Type;
@@ -114,6 +115,7 @@ class ClangTypeParser {
   std::optional<TemplateParam> enumerateTemplateTemplateParam(
       const clang::TemplateName&);
 
+  void enumerateClassParents(const clang::RecordType&, std::vector<Parent>&);
   void enumerateClassMembers(const clang::RecordType&, std::vector<Member>&);
 
   ContainerInfo* getContainerInfo(const std::string& fqName) const;

From 4ffdc51ef19ff7a8e9255e754801d5404a79673e Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 14 Feb 2024 11:34:52 +0000
Subject: [PATCH 153/188] remove internal build config and update for CentOS 9

to fold: more build stuff
---
 CMakeLists.txt                      | 35 ++++++++++++++++++++++-------
 cmake/StandardProjectSettings.cmake |  1 +
 oi/CMakeLists.txt                   |  5 -----
 oi/SymbolService.cpp                |  2 +-
 oi/type_graph/CMakeLists.txt        |  2 --
 test/integration/CMakeLists.txt     |  2 +-
 6 files changed, 30 insertions(+), 17 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b6fecd1..78cc3a8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -152,7 +152,7 @@ if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn")
 endif()
 
 ### Select Python version
-find_program(PYTHON NAMES python3.8 python3)
+find_program(PYTHON NAMES python3.9 python3)
 
 add_library(folly_headers INTERFACE)
 target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
@@ -218,7 +218,7 @@ find_package(zstd REQUIRED)
 # clang-12 does NOT work. clang fails with the following error :-
 # configure: error: gcc with GNU99 support required
 
-set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no")
+set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no" "--disable-libdebuginfod")
 if (ASAN)
   list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes")
 endif()
@@ -252,7 +252,25 @@ set(CMAKE_BUILD_RPATH
 )
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
-include_directories(SYSTEM "${DRGN_PATH}")
+# This header from elfutils is not in the right place. Fake the path manually.
+configure_file(extern/drgn/libdrgn/velfutils/libdw/known-dwarf.h
+elfutils/known-dwarf.h COPYONLY)
+
+add_library(drgn INTERFACE)
+add_dependencies(drgn libdrgn)
+target_include_directories(drgn INTERFACE
+  "${DRGN_PATH}"
+  "${DRGN_PATH}/include"
+  "${PROJECT_SOURCE_DIR}/extern/drgn/libdrgn/velfutils"
+)
+target_link_options(drgn INTERFACE
+  "-L${DRGN_PATH}/.libs"
+  "-ldrgn"
+  "-L${DRGN_PATH}/velfutils/libdw"
+  "-ldw"
+)
+
+
 
 if (STATIC_LINK)
   # glog links against the `gflags` target, which is an alias for `gflags_shared`
@@ -288,7 +306,6 @@ add_library(oicore
   oi/PaddingHunter.cpp
   oi/Serialize.cpp
 )
-add_dependencies(oicore libdrgn)
 target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -299,6 +316,7 @@ target_link_libraries(oicore
 
   ${Boost_LIBRARIES}
   Boost::headers
+  drgn
   glog::glog
   range-v3
   resources
@@ -322,9 +340,6 @@ else()
 endif()
 
 target_link_libraries(oicore
-  "-L${DRGN_PATH}/.libs"
-  drgn
-  dw
   pthread
 )
 
@@ -365,8 +380,12 @@ add_executable(oilgen
 target_link_libraries(oilgen
   drgn_utils
   oicore
-  clangTooling
 )
+if (FORCE_LLVM_STATIC)
+  target_link_libraries(oilgen
+    clangTooling
+  )
+endif()
 
 ### Object Introspection cache Printer (OIP)
 add_executable(oip tools/OIP.cpp)
diff --git a/cmake/StandardProjectSettings.cmake b/cmake/StandardProjectSettings.cmake
index 77519bb..e90bf8d 100644
--- a/cmake/StandardProjectSettings.cmake
+++ b/cmake/StandardProjectSettings.cmake
@@ -16,6 +16,7 @@ endif()
 
 # Generate compile_commands.json to make it easier to work with clang based tools
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES})
 
 option(ENABLE_IPO "Enable Interprocedural Optimization, aka Link Time Optimization (LTO)" OFF)
 
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index bb73871..ad5efa7 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus)
 add_library(drgn_utils DrgnUtils.cpp)
 target_link_libraries(drgn_utils
   glog::glog
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
-add_dependencies(drgn_utils libdrgn)
 
 add_library(symbol_service
   Descs.cpp
@@ -22,8 +19,6 @@ target_link_libraries(symbol_service
   Boost::headers
   ${Boost_LIBRARIES}
   glog::glog
-
-  dw
 )
 
 add_library(features Features.cpp)
diff --git a/oi/SymbolService.cpp b/oi/SymbolService.cpp
index 15d19d2..6952cca 100644
--- a/oi/SymbolService.cpp
+++ b/oi/SymbolService.cpp
@@ -31,7 +31,7 @@ extern "C" {
 #include <elfutils/libdwfl.h>
 
 #include "drgn.h"
-#include "dwarf.h"
+#include "libdw/dwarf.h"
 }
 
 namespace fs = std::filesystem;
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 6c4da13..6cd201e 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -20,12 +20,10 @@ add_library(type_graph
   TypeIdentifier.cpp
   Types.cpp
 )
-add_dependencies(type_graph libdrgn)
 target_link_libraries(type_graph
   container_info
   symbol_service
 
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
 target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 4d235b5..aee09ad 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -28,7 +28,7 @@ add_link_options(-no-pie)
 set(INTEGRATION_TEST_TARGET_SRC integration_test_target.cpp)
 set(INTEGRATION_TEST_RUNNER_SRC integration_test_runner.cpp)
 
-find_program(PYTHON_CMD NAMES python3.6 python3)
+find_program(PYTHON_CMD NAMES python3.9 python3)
 
 set(INTEGRATION_TEST_THRIFT_SRCS ${THRIFT_TESTS})
 list(TRANSFORM INTEGRATION_TEST_THRIFT_SRCS APPEND ".thrift")

From ff96b2a814ecb50f873dac912b978e09fbc30cbc Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 14 Feb 2024 11:34:34 +0000
Subject: [PATCH 154/188] tests: add ClangTypeParserTest

Currently there is no testing for ClangTypeParser even though it's used in
production. This is because adding integration tests is very hard: they require
testing the build time behaviour at runtime, or else they'd be build failures
intead of test failures. There's a PR available for integration tests but it's
incomplete.

In contrast ClangTypeParser can be sort of unit tested. This follows the
structure of `test/test_drgn_parser.cpp` with some differences. There is a
tonne of boilerplate for setting up the Clang tool, and this set of testing
operates on type names instead of OID functions. The new tests are also
incredibly slow as they compile the entire `integration_test_target.cpp` (which
is huge) for every test case. I don't think this is avoidable without
compromising the separation of the tests somewhat due to the way Clang tooling
forces the code to be structured.

Test plan:
- Tested locally
- CI
---
 oi/type_graph/ClangTypeParserTest.cpp | 177 ++++++++++++++++++++++++++
 test/CMakeLists.txt                   |  25 ++++
 2 files changed, 202 insertions(+)
 create mode 100644 oi/type_graph/ClangTypeParserTest.cpp

diff --git a/oi/type_graph/ClangTypeParserTest.cpp b/oi/type_graph/ClangTypeParserTest.cpp
new file mode 100644
index 0000000..fd56cca
--- /dev/null
+++ b/oi/type_graph/ClangTypeParserTest.cpp
@@ -0,0 +1,177 @@
+#include <gtest/gtest.h>
+
+#include <clang/AST/Mangle.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Lookup.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
+#include <clang/Tooling/CompilationDatabase.h>
+#include <range/v3/core.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <string>
+#include <string_view>
+
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+
+using namespace oi::detail;
+using namespace oi::detail::type_graph;
+
+class ClangTypeParserTest : public ::testing::Test {
+ public:
+  std::string run(std::string_view function, ClangTypeParserOptions opt);
+  void test(std::string_view function,
+            std::string_view expected,
+            ClangTypeParserOptions opts = {
+                .chaseRawPointers = true,
+                .readEnumValues = true,
+            });
+};
+
+// This stuff is a total mess. Set up factories as ClangTooling expects them.
+class ConsumerContext;
+class CreateTypeGraphConsumer;
+
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class ConsumerContext {
+ public:
+  ConsumerContext(const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos_) : containerInfos{containerInfos_} {}
+  std::string_view fullyQualifiedName;
+  ClangTypeParserOptions opts;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+  TypeGraph typeGraph;
+  Type* result = nullptr;
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    const clang::Type* type = nullptr;
+    for (const clang::Type* ty : Context.getTypes()) {
+      std::string fqnWithoutTemplateParams;
+      switch (ty->getTypeClass()) {
+        case clang::Type::Record:
+          fqnWithoutTemplateParams = llvm::cast<const clang::RecordType>(ty)->getDecl()->getQualifiedNameAsString();
+          break;
+        default:
+          continue;
+      }
+      if (fqnWithoutTemplateParams != ctx.fullyQualifiedName)
+        continue;
+
+      type = ty;
+      break;
+    }
+    EXPECT_NE(type, nullptr);
+
+    ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, ctx.opts};
+    ctx.result = &parser.parse(Context, *ctx.sema, *type);
+  }
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
+}
+
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    [[maybe_unused]] clang::CompilerInstance& CI,
+    [[maybe_unused]] clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+std::string ClangTypeParserTest::run(std::string_view type,
+                                     ClangTypeParserOptions opts) {
+  std::string err{"failed to load compilation database"};
+  auto db = clang::tooling::CompilationDatabase::loadFromDirectory(BUILD_DIR, err);
+  if (!db) {
+    throw std::runtime_error("failed to load compilation database");
+  }
+
+  std::vector<std::unique_ptr<ContainerInfo>> cis;
+  ConsumerContext ctx{ cis };
+  ctx.fullyQualifiedName = type;
+  ctx.opts = std::move(opts);
+  CreateTypeGraphActionFactory factory{ctx};
+
+  std::vector<std::string> sourcePaths{TARGET_CPP_PATH};
+  clang::tooling::ClangTool tool{*db, sourcePaths};
+  if (auto retCode = tool.run(&factory); retCode != 0) {
+    throw std::runtime_error("clang type parsing failed");
+  }
+
+  std::stringstream out;
+  NodeTracker tracker;
+  Printer printer{out, tracker, ctx.typeGraph.size()};
+  printer.print(*ctx.result);
+
+  return std::move(out).str();
+}
+
+void ClangTypeParserTest::test(std::string_view type,
+                               std::string_view expected,
+                               ClangTypeParserOptions opts) {
+  std::string actual = run(type, std::move(opts));
+  expected.remove_prefix(1);  // Remove initial '\n'
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(ClangTypeParserTest, SimpleStruct) {
+  test("ns_simple::SimpleStruct", R"(
+[0] Struct: SimpleStruct [ns_simple::SimpleStruct] (size: 16, align: 8)
+      Member: a (offset: 0)
+        Primitive: int32_t
+      Member: b (offset: 4)
+        Primitive: int8_t
+      Member: c (offset: 8)
+        Primitive: int64_t
+)");
+}
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 287ebcd..ccdb606 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -69,6 +69,31 @@ target_link_libraries(test_type_graph
 include(GoogleTest)
 gtest_discover_tests(test_type_graph)
 
+add_executable(test_clang_type_parser
+  main.cpp
+  ../oi/type_graph/ClangTypeParserTest.cpp
+)
+add_dependencies(test_clang_type_parser integration_test_target)
+target_compile_definitions(test_clang_type_parser PRIVATE
+  TARGET_CPP_PATH="${CMAKE_CURRENT_BINARY_DIR}/integration/integration_test_target.cpp"
+  BUILD_DIR="${CMAKE_BINARY_DIR}"
+)
+target_link_libraries(test_clang_type_parser
+  codegen
+  container_info
+  type_graph
+
+  range-v3
+
+  GTest::gmock_main
+)
+if (FORCE_LLVM_STATIC)
+  target_link_libraries(test_clang_type_parser clangTooling ${llvm_libs})
+else()
+  target_link_libraries(test_clang_type_parser clang-cpp LLVM)
+endif()
+gtest_discover_tests(test_clang_type_parser)
+
 cpp_unittest(
   NAME test_parser
   SRCS test_parser.cpp

From 105c26f803f932c8d8014ee0ca80fc5003299599 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 14 Feb 2024 14:22:25 +0000
Subject: [PATCH 155/188] remove internal build config and update for CentOS 9

Previously we maintained three types of builds: a fully internal BUCK build, a
CMake build with modifications to use things from an internal toolchain, and an
open source CMake build.

As far as I'm concerned the intermediate build is not useful because our source
is readily available in both an internal and external form. Use cases as
follows:
1. BUCK build for distributing widely.
2. BUCK build for getting a static binary that can be run on any machine.
3. CMake build for primary development.
4. CMake build for external CI.

With the internal update to CentOS Stream 9 an unmodified CMake build now works
readily. This change patches up some things that were relying on system headers
that should have been vendored and cleans up drgn dependencies.

Test plan:
- It builds.
- TODO: Document CentOS 9 installation.
---
 CMakeLists.txt                      | 35 ++++++++++++++++++++++-------
 cmake/StandardProjectSettings.cmake |  1 +
 oi/CMakeLists.txt                   |  5 -----
 oi/SymbolService.cpp                |  2 +-
 oi/type_graph/CMakeLists.txt        |  2 --
 test/integration/CMakeLists.txt     |  2 +-
 6 files changed, 30 insertions(+), 17 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b6fecd1..78cc3a8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -152,7 +152,7 @@ if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn")
 endif()
 
 ### Select Python version
-find_program(PYTHON NAMES python3.8 python3)
+find_program(PYTHON NAMES python3.9 python3)
 
 add_library(folly_headers INTERFACE)
 target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
@@ -218,7 +218,7 @@ find_package(zstd REQUIRED)
 # clang-12 does NOT work. clang fails with the following error :-
 # configure: error: gcc with GNU99 support required
 
-set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no")
+set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no" "--disable-libdebuginfod")
 if (ASAN)
   list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes")
 endif()
@@ -252,7 +252,25 @@ set(CMAKE_BUILD_RPATH
 )
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
-include_directories(SYSTEM "${DRGN_PATH}")
+# This header from elfutils is not in the right place. Fake the path manually.
+configure_file(extern/drgn/libdrgn/velfutils/libdw/known-dwarf.h
+elfutils/known-dwarf.h COPYONLY)
+
+add_library(drgn INTERFACE)
+add_dependencies(drgn libdrgn)
+target_include_directories(drgn INTERFACE
+  "${DRGN_PATH}"
+  "${DRGN_PATH}/include"
+  "${PROJECT_SOURCE_DIR}/extern/drgn/libdrgn/velfutils"
+)
+target_link_options(drgn INTERFACE
+  "-L${DRGN_PATH}/.libs"
+  "-ldrgn"
+  "-L${DRGN_PATH}/velfutils/libdw"
+  "-ldw"
+)
+
+
 
 if (STATIC_LINK)
   # glog links against the `gflags` target, which is an alias for `gflags_shared`
@@ -288,7 +306,6 @@ add_library(oicore
   oi/PaddingHunter.cpp
   oi/Serialize.cpp
 )
-add_dependencies(oicore libdrgn)
 target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -299,6 +316,7 @@ target_link_libraries(oicore
 
   ${Boost_LIBRARIES}
   Boost::headers
+  drgn
   glog::glog
   range-v3
   resources
@@ -322,9 +340,6 @@ else()
 endif()
 
 target_link_libraries(oicore
-  "-L${DRGN_PATH}/.libs"
-  drgn
-  dw
   pthread
 )
 
@@ -365,8 +380,12 @@ add_executable(oilgen
 target_link_libraries(oilgen
   drgn_utils
   oicore
-  clangTooling
 )
+if (FORCE_LLVM_STATIC)
+  target_link_libraries(oilgen
+    clangTooling
+  )
+endif()
 
 ### Object Introspection cache Printer (OIP)
 add_executable(oip tools/OIP.cpp)
diff --git a/cmake/StandardProjectSettings.cmake b/cmake/StandardProjectSettings.cmake
index 77519bb..e90bf8d 100644
--- a/cmake/StandardProjectSettings.cmake
+++ b/cmake/StandardProjectSettings.cmake
@@ -16,6 +16,7 @@ endif()
 
 # Generate compile_commands.json to make it easier to work with clang based tools
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES})
 
 option(ENABLE_IPO "Enable Interprocedural Optimization, aka Link Time Optimization (LTO)" OFF)
 
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index bb73871..ad5efa7 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus)
 add_library(drgn_utils DrgnUtils.cpp)
 target_link_libraries(drgn_utils
   glog::glog
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
-add_dependencies(drgn_utils libdrgn)
 
 add_library(symbol_service
   Descs.cpp
@@ -22,8 +19,6 @@ target_link_libraries(symbol_service
   Boost::headers
   ${Boost_LIBRARIES}
   glog::glog
-
-  dw
 )
 
 add_library(features Features.cpp)
diff --git a/oi/SymbolService.cpp b/oi/SymbolService.cpp
index 15d19d2..6952cca 100644
--- a/oi/SymbolService.cpp
+++ b/oi/SymbolService.cpp
@@ -31,7 +31,7 @@ extern "C" {
 #include <elfutils/libdwfl.h>
 
 #include "drgn.h"
-#include "dwarf.h"
+#include "libdw/dwarf.h"
 }
 
 namespace fs = std::filesystem;
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 6c4da13..6cd201e 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -20,12 +20,10 @@ add_library(type_graph
   TypeIdentifier.cpp
   Types.cpp
 )
-add_dependencies(type_graph libdrgn)
 target_link_libraries(type_graph
   container_info
   symbol_service
 
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
 target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 4d235b5..aee09ad 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -28,7 +28,7 @@ add_link_options(-no-pie)
 set(INTEGRATION_TEST_TARGET_SRC integration_test_target.cpp)
 set(INTEGRATION_TEST_RUNNER_SRC integration_test_runner.cpp)
 
-find_program(PYTHON_CMD NAMES python3.6 python3)
+find_program(PYTHON_CMD NAMES python3.9 python3)
 
 set(INTEGRATION_TEST_THRIFT_SRCS ${THRIFT_TESTS})
 list(TRANSFORM INTEGRATION_TEST_THRIFT_SRCS APPEND ".thrift")

From 33286a8f423d15f2625fa5182b1895a95bc1fbdc Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 14 Feb 2024 14:22:25 +0000
Subject: [PATCH 156/188] remove internal build config and update for CentOS 9

Previously we maintained three types of builds: a fully internal BUCK build, a
CMake build with modifications to use things from an internal toolchain, and an
open source CMake build.

As far as I'm concerned the intermediate build is not useful because our source
is readily available in both an internal and external form. Use cases as
follows:
1. BUCK build for distributing widely.
2. BUCK build for getting a static binary that can be run on any machine.
3. CMake build for primary development.
4. CMake build for external CI.

With the internal update to CentOS Stream 9 an unmodified CMake build now works
readily. This change patches up some things that were relying on system headers
that should have been vendored and cleans up drgn dependencies.

Test plan:
- It builds.
- TODO: Document CentOS 9 installation.
---
 CMakeLists.txt                      | 36 ++++++++++++++++++++++-------
 cmake/StandardProjectSettings.cmake |  1 +
 oi/CMakeLists.txt                   |  5 ----
 oi/SymbolService.cpp                |  2 +-
 oi/type_graph/CMakeLists.txt        |  2 --
 test/integration/CMakeLists.txt     |  2 +-
 6 files changed, 31 insertions(+), 17 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b6fecd1..c256901 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -152,7 +152,7 @@ if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn")
 endif()
 
 ### Select Python version
-find_program(PYTHON NAMES python3.8 python3)
+find_program(PYTHON NAMES python3.9 python3)
 
 add_library(folly_headers INTERFACE)
 target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
@@ -218,7 +218,7 @@ find_package(zstd REQUIRED)
 # clang-12 does NOT work. clang fails with the following error :-
 # configure: error: gcc with GNU99 support required
 
-set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no")
+set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no" "--disable-libdebuginfod")
 if (ASAN)
   list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes")
 endif()
@@ -252,7 +252,26 @@ set(CMAKE_BUILD_RPATH
 )
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
-include_directories(SYSTEM "${DRGN_PATH}")
+# This header from elfutils is not in the right place. Fake the path manually.
+add_custom_command(TARGET libdrgn POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy
+${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/libdrgn/velfutils/libdw/known-dwarf.h
+${CMAKE_CURRENT_BINARY_DIR}/elfutils/known-dwarf.h)
+
+add_library(drgn INTERFACE)
+add_dependencies(drgn libdrgn)
+target_include_directories(drgn INTERFACE
+  "${DRGN_PATH}"
+  "${DRGN_PATH}/include"
+  "${PROJECT_SOURCE_DIR}/extern/drgn/libdrgn/velfutils"
+)
+target_link_options(drgn INTERFACE
+  "-L${DRGN_PATH}/.libs"
+  "-ldrgn"
+  "-L${DRGN_PATH}/velfutils/libdw"
+  "-ldw"
+)
+
+
 
 if (STATIC_LINK)
   # glog links against the `gflags` target, which is an alias for `gflags_shared`
@@ -288,7 +307,6 @@ add_library(oicore
   oi/PaddingHunter.cpp
   oi/Serialize.cpp
 )
-add_dependencies(oicore libdrgn)
 target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -299,6 +317,7 @@ target_link_libraries(oicore
 
   ${Boost_LIBRARIES}
   Boost::headers
+  drgn
   glog::glog
   range-v3
   resources
@@ -322,9 +341,6 @@ else()
 endif()
 
 target_link_libraries(oicore
-  "-L${DRGN_PATH}/.libs"
-  drgn
-  dw
   pthread
 )
 
@@ -365,8 +381,12 @@ add_executable(oilgen
 target_link_libraries(oilgen
   drgn_utils
   oicore
-  clangTooling
 )
+if (FORCE_LLVM_STATIC)
+  target_link_libraries(oilgen
+    clangTooling
+  )
+endif()
 
 ### Object Introspection cache Printer (OIP)
 add_executable(oip tools/OIP.cpp)
diff --git a/cmake/StandardProjectSettings.cmake b/cmake/StandardProjectSettings.cmake
index 77519bb..e90bf8d 100644
--- a/cmake/StandardProjectSettings.cmake
+++ b/cmake/StandardProjectSettings.cmake
@@ -16,6 +16,7 @@ endif()
 
 # Generate compile_commands.json to make it easier to work with clang based tools
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES})
 
 option(ENABLE_IPO "Enable Interprocedural Optimization, aka Link Time Optimization (LTO)" OFF)
 
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index bb73871..ad5efa7 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus)
 add_library(drgn_utils DrgnUtils.cpp)
 target_link_libraries(drgn_utils
   glog::glog
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
-add_dependencies(drgn_utils libdrgn)
 
 add_library(symbol_service
   Descs.cpp
@@ -22,8 +19,6 @@ target_link_libraries(symbol_service
   Boost::headers
   ${Boost_LIBRARIES}
   glog::glog
-
-  dw
 )
 
 add_library(features Features.cpp)
diff --git a/oi/SymbolService.cpp b/oi/SymbolService.cpp
index 15d19d2..6952cca 100644
--- a/oi/SymbolService.cpp
+++ b/oi/SymbolService.cpp
@@ -31,7 +31,7 @@ extern "C" {
 #include <elfutils/libdwfl.h>
 
 #include "drgn.h"
-#include "dwarf.h"
+#include "libdw/dwarf.h"
 }
 
 namespace fs = std::filesystem;
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 6c4da13..6cd201e 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -20,12 +20,10 @@ add_library(type_graph
   TypeIdentifier.cpp
   Types.cpp
 )
-add_dependencies(type_graph libdrgn)
 target_link_libraries(type_graph
   container_info
   symbol_service
 
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
 target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 4d235b5..aee09ad 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -28,7 +28,7 @@ add_link_options(-no-pie)
 set(INTEGRATION_TEST_TARGET_SRC integration_test_target.cpp)
 set(INTEGRATION_TEST_RUNNER_SRC integration_test_runner.cpp)
 
-find_program(PYTHON_CMD NAMES python3.6 python3)
+find_program(PYTHON_CMD NAMES python3.9 python3)
 
 set(INTEGRATION_TEST_THRIFT_SRCS ${THRIFT_TESTS})
 list(TRANSFORM INTEGRATION_TEST_THRIFT_SRCS APPEND ".thrift")

From a908d3282422a4fd1c17a9e3bae2f96df5d16ef9 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 14 Feb 2024 14:22:25 +0000
Subject: [PATCH 157/188] remove internal build config and update for CentOS 9

Previously we maintained three types of builds: a fully internal BUCK build, a
CMake build with modifications to use things from an internal toolchain, and an
open source CMake build.

As far as I'm concerned the intermediate build is not useful because our source
is readily available in both an internal and external form. Use cases as
follows:
1. BUCK build for distributing widely.
2. BUCK build for getting a static binary that can be run on any machine.
3. CMake build for primary development.
4. CMake build for external CI.

With the internal update to CentOS Stream 9 an unmodified CMake build now works
readily. This change patches up some things that were relying on system headers
that should have been vendored and cleans up drgn dependencies.

Test plan:
- It builds.
- TODO: Document CentOS 9 installation.
---
 CMakeLists.txt                      | 36 ++++++++++++++++++++++-------
 cmake/StandardProjectSettings.cmake |  1 +
 oi/CMakeLists.txt                   |  5 ----
 oi/SymbolService.cpp                |  2 +-
 oi/type_graph/CMakeLists.txt        |  2 --
 test/integration/CMakeLists.txt     |  2 +-
 6 files changed, 31 insertions(+), 17 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b6fecd1..212df55 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -152,7 +152,7 @@ if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn")
 endif()
 
 ### Select Python version
-find_program(PYTHON NAMES python3.8 python3)
+find_program(PYTHON NAMES python3.9 python3)
 
 add_library(folly_headers INTERFACE)
 target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
@@ -218,7 +218,7 @@ find_package(zstd REQUIRED)
 # clang-12 does NOT work. clang fails with the following error :-
 # configure: error: gcc with GNU99 support required
 
-set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no")
+set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no" "--disable-libdebuginfod")
 if (ASAN)
   list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes")
 endif()
@@ -252,7 +252,26 @@ set(CMAKE_BUILD_RPATH
 )
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
-include_directories(SYSTEM "${DRGN_PATH}")
+# This header from elfutils is not in the right place. Fake the path manually.
+add_custom_command(TARGET libdrgn POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy
+${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/libdrgn/velfutils/libdw/known-dwarf.h
+${CMAKE_CURRENT_BINARY_DIR}/elfutils/known-dwarf.h)
+
+add_library(drgn INTERFACE)
+add_dependencies(drgn libdrgn)
+target_include_directories(drgn SYSTEM INTERFACE
+  "${DRGN_PATH}"
+  "${DRGN_PATH}/include"
+  "${PROJECT_SOURCE_DIR}/extern/drgn/libdrgn/velfutils"
+)
+target_link_options(drgn INTERFACE
+  "-L${DRGN_PATH}/.libs"
+  "-ldrgn"
+  "-L${DRGN_PATH}/velfutils/libdw"
+  "-ldw"
+)
+
+
 
 if (STATIC_LINK)
   # glog links against the `gflags` target, which is an alias for `gflags_shared`
@@ -288,7 +307,6 @@ add_library(oicore
   oi/PaddingHunter.cpp
   oi/Serialize.cpp
 )
-add_dependencies(oicore libdrgn)
 target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -299,6 +317,7 @@ target_link_libraries(oicore
 
   ${Boost_LIBRARIES}
   Boost::headers
+  drgn
   glog::glog
   range-v3
   resources
@@ -322,9 +341,6 @@ else()
 endif()
 
 target_link_libraries(oicore
-  "-L${DRGN_PATH}/.libs"
-  drgn
-  dw
   pthread
 )
 
@@ -365,8 +381,12 @@ add_executable(oilgen
 target_link_libraries(oilgen
   drgn_utils
   oicore
-  clangTooling
 )
+if (FORCE_LLVM_STATIC)
+  target_link_libraries(oilgen
+    clangTooling
+  )
+endif()
 
 ### Object Introspection cache Printer (OIP)
 add_executable(oip tools/OIP.cpp)
diff --git a/cmake/StandardProjectSettings.cmake b/cmake/StandardProjectSettings.cmake
index 77519bb..e90bf8d 100644
--- a/cmake/StandardProjectSettings.cmake
+++ b/cmake/StandardProjectSettings.cmake
@@ -16,6 +16,7 @@ endif()
 
 # Generate compile_commands.json to make it easier to work with clang based tools
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES})
 
 option(ENABLE_IPO "Enable Interprocedural Optimization, aka Link Time Optimization (LTO)" OFF)
 
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index bb73871..ad5efa7 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus)
 add_library(drgn_utils DrgnUtils.cpp)
 target_link_libraries(drgn_utils
   glog::glog
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
-add_dependencies(drgn_utils libdrgn)
 
 add_library(symbol_service
   Descs.cpp
@@ -22,8 +19,6 @@ target_link_libraries(symbol_service
   Boost::headers
   ${Boost_LIBRARIES}
   glog::glog
-
-  dw
 )
 
 add_library(features Features.cpp)
diff --git a/oi/SymbolService.cpp b/oi/SymbolService.cpp
index 15d19d2..6952cca 100644
--- a/oi/SymbolService.cpp
+++ b/oi/SymbolService.cpp
@@ -31,7 +31,7 @@ extern "C" {
 #include <elfutils/libdwfl.h>
 
 #include "drgn.h"
-#include "dwarf.h"
+#include "libdw/dwarf.h"
 }
 
 namespace fs = std::filesystem;
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 6c4da13..6cd201e 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -20,12 +20,10 @@ add_library(type_graph
   TypeIdentifier.cpp
   Types.cpp
 )
-add_dependencies(type_graph libdrgn)
 target_link_libraries(type_graph
   container_info
   symbol_service
 
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
 target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 4d235b5..aee09ad 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -28,7 +28,7 @@ add_link_options(-no-pie)
 set(INTEGRATION_TEST_TARGET_SRC integration_test_target.cpp)
 set(INTEGRATION_TEST_RUNNER_SRC integration_test_runner.cpp)
 
-find_program(PYTHON_CMD NAMES python3.6 python3)
+find_program(PYTHON_CMD NAMES python3.9 python3)
 
 set(INTEGRATION_TEST_THRIFT_SRCS ${THRIFT_TESTS})
 list(TRANSFORM INTEGRATION_TEST_THRIFT_SRCS APPEND ".thrift")

From 8d3feedaea610010db2da6629a6f6a56b61ba2c7 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Mon, 19 Feb 2024 14:44:24 +0000
Subject: [PATCH 158/188] remove internal build config and update for CentOS 9

Previously we maintained three types of builds: a fully internal BUCK build, a
CMake build with modifications to use things from an internal toolchain, and an
open source CMake build.

As far as I'm concerned the intermediate build is not useful because our source
is readily available in both an internal and external form. Use cases as
follows:
1. BUCK build for distributing widely.
2. BUCK build for getting a static binary that can be run on any machine.
3. CMake build for primary development.
4. CMake build for external CI.

With the internal update to CentOS Stream 9 an unmodified CMake build now works
readily. This change patches up some things that were relying on system headers
that should have been vendored and cleans up drgn dependencies.

Test plan:
- It builds.
- TODO: Document CentOS 9 installation.
---
 CMakeLists.txt                      | 36 ++++++++++++++++++++++-------
 cmake/StandardProjectSettings.cmake |  1 +
 oi/CMakeLists.txt                   |  5 ----
 oi/SymbolService.cpp                |  2 +-
 oi/type_graph/CMakeLists.txt        |  2 --
 test/integration/CMakeLists.txt     |  2 +-
 6 files changed, 31 insertions(+), 17 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b6fecd1..212df55 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -152,7 +152,7 @@ if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn")
 endif()
 
 ### Select Python version
-find_program(PYTHON NAMES python3.8 python3)
+find_program(PYTHON NAMES python3.9 python3)
 
 add_library(folly_headers INTERFACE)
 target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
@@ -218,7 +218,7 @@ find_package(zstd REQUIRED)
 # clang-12 does NOT work. clang fails with the following error :-
 # configure: error: gcc with GNU99 support required
 
-set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no")
+set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no" "--disable-libdebuginfod")
 if (ASAN)
   list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes")
 endif()
@@ -252,7 +252,26 @@ set(CMAKE_BUILD_RPATH
 )
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
-include_directories(SYSTEM "${DRGN_PATH}")
+# This header from elfutils is not in the right place. Fake the path manually.
+add_custom_command(TARGET libdrgn POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy
+${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/libdrgn/velfutils/libdw/known-dwarf.h
+${CMAKE_CURRENT_BINARY_DIR}/elfutils/known-dwarf.h)
+
+add_library(drgn INTERFACE)
+add_dependencies(drgn libdrgn)
+target_include_directories(drgn SYSTEM INTERFACE
+  "${DRGN_PATH}"
+  "${DRGN_PATH}/include"
+  "${PROJECT_SOURCE_DIR}/extern/drgn/libdrgn/velfutils"
+)
+target_link_options(drgn INTERFACE
+  "-L${DRGN_PATH}/.libs"
+  "-ldrgn"
+  "-L${DRGN_PATH}/velfutils/libdw"
+  "-ldw"
+)
+
+
 
 if (STATIC_LINK)
   # glog links against the `gflags` target, which is an alias for `gflags_shared`
@@ -288,7 +307,6 @@ add_library(oicore
   oi/PaddingHunter.cpp
   oi/Serialize.cpp
 )
-add_dependencies(oicore libdrgn)
 target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -299,6 +317,7 @@ target_link_libraries(oicore
 
   ${Boost_LIBRARIES}
   Boost::headers
+  drgn
   glog::glog
   range-v3
   resources
@@ -322,9 +341,6 @@ else()
 endif()
 
 target_link_libraries(oicore
-  "-L${DRGN_PATH}/.libs"
-  drgn
-  dw
   pthread
 )
 
@@ -365,8 +381,12 @@ add_executable(oilgen
 target_link_libraries(oilgen
   drgn_utils
   oicore
-  clangTooling
 )
+if (FORCE_LLVM_STATIC)
+  target_link_libraries(oilgen
+    clangTooling
+  )
+endif()
 
 ### Object Introspection cache Printer (OIP)
 add_executable(oip tools/OIP.cpp)
diff --git a/cmake/StandardProjectSettings.cmake b/cmake/StandardProjectSettings.cmake
index 77519bb..e90bf8d 100644
--- a/cmake/StandardProjectSettings.cmake
+++ b/cmake/StandardProjectSettings.cmake
@@ -16,6 +16,7 @@ endif()
 
 # Generate compile_commands.json to make it easier to work with clang based tools
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES})
 
 option(ENABLE_IPO "Enable Interprocedural Optimization, aka Link Time Optimization (LTO)" OFF)
 
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index bb73871..ad5efa7 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus)
 add_library(drgn_utils DrgnUtils.cpp)
 target_link_libraries(drgn_utils
   glog::glog
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
-add_dependencies(drgn_utils libdrgn)
 
 add_library(symbol_service
   Descs.cpp
@@ -22,8 +19,6 @@ target_link_libraries(symbol_service
   Boost::headers
   ${Boost_LIBRARIES}
   glog::glog
-
-  dw
 )
 
 add_library(features Features.cpp)
diff --git a/oi/SymbolService.cpp b/oi/SymbolService.cpp
index 15d19d2..6952cca 100644
--- a/oi/SymbolService.cpp
+++ b/oi/SymbolService.cpp
@@ -31,7 +31,7 @@ extern "C" {
 #include <elfutils/libdwfl.h>
 
 #include "drgn.h"
-#include "dwarf.h"
+#include "libdw/dwarf.h"
 }
 
 namespace fs = std::filesystem;
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 6c4da13..6cd201e 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -20,12 +20,10 @@ add_library(type_graph
   TypeIdentifier.cpp
   Types.cpp
 )
-add_dependencies(type_graph libdrgn)
 target_link_libraries(type_graph
   container_info
   symbol_service
 
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
 target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 4d235b5..aee09ad 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -28,7 +28,7 @@ add_link_options(-no-pie)
 set(INTEGRATION_TEST_TARGET_SRC integration_test_target.cpp)
 set(INTEGRATION_TEST_RUNNER_SRC integration_test_runner.cpp)
 
-find_program(PYTHON_CMD NAMES python3.6 python3)
+find_program(PYTHON_CMD NAMES python3.9 python3)
 
 set(INTEGRATION_TEST_THRIFT_SRCS ${THRIFT_TESTS})
 list(TRANSFORM INTEGRATION_TEST_THRIFT_SRCS APPEND ".thrift")

From ef3bab148010b70722389858de9970a7be9bd38c Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Mon, 19 Feb 2024 14:44:25 +0000
Subject: [PATCH 159/188] tests: add ClangTypeParserTest

Currently there is no testing for ClangTypeParser even though it's used in
production. This is because adding integration tests is very hard: they require
testing the build time behaviour at runtime, or else they'd be build failures
intead of test failures. There's a PR available for integration tests but it's
incomplete.

In contrast ClangTypeParser can be sort of unit tested. This follows the
structure of `test/test_drgn_parser.cpp` with some differences. There is a
tonne of boilerplate for setting up the Clang tool, and this set of testing
operates on type names instead of OID functions. The new tests are also
incredibly slow as they compile the entire `integration_test_target.cpp` (which
is huge) for every test case. I don't think this is avoidable without
compromising the separation of the tests somewhat due to the way Clang tooling
forces the code to be structured.

Test plan:
- Tested locally
- CI
---
 oi/type_graph/ClangTypeParserTest.cpp | 181 ++++++++++++++++++++++++++
 test/CMakeLists.txt                   |  25 ++++
 2 files changed, 206 insertions(+)
 create mode 100644 oi/type_graph/ClangTypeParserTest.cpp

diff --git a/oi/type_graph/ClangTypeParserTest.cpp b/oi/type_graph/ClangTypeParserTest.cpp
new file mode 100644
index 0000000..33385bb
--- /dev/null
+++ b/oi/type_graph/ClangTypeParserTest.cpp
@@ -0,0 +1,181 @@
+#include <gtest/gtest.h>
+
+#include <clang/AST/Mangle.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Lookup.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/Tooling.h>
+#include <clang/Tooling/CompilationDatabase.h>
+#include <range/v3/core.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <string>
+#include <string_view>
+
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+
+using namespace oi::detail;
+using namespace oi::detail::type_graph;
+
+class ClangTypeParserTest : public ::testing::Test {
+ public:
+  std::string run(std::string_view function, ClangTypeParserOptions opt);
+  void test(std::string_view function,
+            std::string_view expected,
+            ClangTypeParserOptions opts = {
+                .chaseRawPointers = true,
+                .readEnumValues = true,
+            });
+};
+
+// This stuff is a total mess. Set up factories as ClangTooling expects them.
+class ConsumerContext;
+class CreateTypeGraphConsumer;
+
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class ConsumerContext {
+ public:
+  ConsumerContext(const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos_) : containerInfos{containerInfos_} {}
+  std::string_view fullyQualifiedName;
+  ClangTypeParserOptions opts;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+  TypeGraph typeGraph;
+  Type* result = nullptr;
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    const clang::Type* type = nullptr;
+    for (const clang::Type* ty : Context.getTypes()) {
+      std::string fqnWithoutTemplateParams;
+      switch (ty->getTypeClass()) {
+        case clang::Type::Record:
+          fqnWithoutTemplateParams = llvm::cast<const clang::RecordType>(ty)->getDecl()->getQualifiedNameAsString();
+          break;
+        default:
+          continue;
+      }
+      if (fqnWithoutTemplateParams != ctx.fullyQualifiedName)
+        continue;
+
+      type = ty;
+      break;
+    }
+    EXPECT_NE(type, nullptr);
+
+    ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, ctx.opts};
+    ctx.result = &parser.parse(Context, *ctx.sema, *type);
+  }
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
+}
+
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    [[maybe_unused]] clang::CompilerInstance& CI,
+    [[maybe_unused]] clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+std::string ClangTypeParserTest::run(std::string_view type,
+                                     ClangTypeParserOptions opts) {
+  std::string err{"failed to load compilation database"};
+  auto db = clang::tooling::CompilationDatabase::loadFromDirectory(BUILD_DIR, err);
+  if (!db) {
+    throw std::runtime_error("failed to load compilation database");
+  }
+
+  std::vector<std::unique_ptr<ContainerInfo>> cis;
+  ConsumerContext ctx{ cis };
+  ctx.fullyQualifiedName = type;
+  ctx.opts = std::move(opts);
+  CreateTypeGraphActionFactory factory{ctx};
+
+  std::vector<std::string> sourcePaths{TARGET_CPP_PATH};
+  clang::tooling::ClangTool tool{*db, sourcePaths};
+  if (auto retCode = tool.run(&factory); retCode != 0) {
+    throw std::runtime_error("clang type parsing failed");
+  }
+
+  std::stringstream out;
+  NodeTracker tracker;
+  Printer printer{out, tracker, ctx.typeGraph.size()};
+  printer.print(*ctx.result);
+
+  return std::move(out).str();
+}
+
+void ClangTypeParserTest::test(std::string_view type,
+                               std::string_view expected,
+                               ClangTypeParserOptions opts) {
+  std::string actual = run(type, std::move(opts));
+  expected.remove_prefix(1);  // Remove initial '\n'
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(ClangTypeParserTest, SimpleStruct) {
+  test("ns_simple::SimpleStruct", R"(
+[0] Struct: SimpleStruct [ns_simple::SimpleStruct] (size: 16, align: 8)
+      Member: a (offset: 0)
+        Primitive: int32_t
+      Member: b (offset: 4)
+        Primitive: int8_t
+      Member: c (offset: 8)
+        Primitive: int64_t
+)");
+}
+
+TEST_F(ClangTypeParserTest, MemberAlignment) {
+  test("ns_alignment::MemberAlignment", R"(stuff)");
+}
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 287ebcd..ccdb606 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -69,6 +69,31 @@ target_link_libraries(test_type_graph
 include(GoogleTest)
 gtest_discover_tests(test_type_graph)
 
+add_executable(test_clang_type_parser
+  main.cpp
+  ../oi/type_graph/ClangTypeParserTest.cpp
+)
+add_dependencies(test_clang_type_parser integration_test_target)
+target_compile_definitions(test_clang_type_parser PRIVATE
+  TARGET_CPP_PATH="${CMAKE_CURRENT_BINARY_DIR}/integration/integration_test_target.cpp"
+  BUILD_DIR="${CMAKE_BINARY_DIR}"
+)
+target_link_libraries(test_clang_type_parser
+  codegen
+  container_info
+  type_graph
+
+  range-v3
+
+  GTest::gmock_main
+)
+if (FORCE_LLVM_STATIC)
+  target_link_libraries(test_clang_type_parser clangTooling ${llvm_libs})
+else()
+  target_link_libraries(test_clang_type_parser clang-cpp LLVM)
+endif()
+gtest_discover_tests(test_clang_type_parser)
+
 cpp_unittest(
   NAME test_parser
   SRCS test_parser.cpp

From 88c0446ef447d1d7ce7d60ac97c6ad0a98f3200e Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Mon, 19 Feb 2024 14:44:24 +0000
Subject: [PATCH 160/188] remove internal build config and update for CentOS 9

Previously we maintained three types of builds: a fully internal BUCK build, a
CMake build with modifications to use things from an internal toolchain, and an
open source CMake build.

As far as I'm concerned the intermediate build is not useful because our source
is readily available in both an internal and external form. Use cases as
follows:
1. BUCK build for distributing widely.
2. BUCK build for getting a static binary that can be run on any machine.
3. CMake build for primary development.
4. CMake build for external CI.

With the internal update to CentOS Stream 9 an unmodified CMake build now works
readily. This change patches up some things that were relying on system headers
that should have been vendored and cleans up drgn dependencies.

Test plan:
- It builds.
- TODO: Document CentOS 9 installation.
---
 CMakeLists.txt                      | 38 +++++++++++++++++++++++------
 cmake/StandardProjectSettings.cmake |  1 +
 oi/CMakeLists.txt                   |  5 ----
 oi/SymbolService.cpp                |  2 +-
 oi/type_graph/CMakeLists.txt        |  2 --
 test/integration/CMakeLists.txt     |  2 +-
 6 files changed, 33 insertions(+), 17 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b6fecd1..e1717ce 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -152,7 +152,7 @@ if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn")
 endif()
 
 ### Select Python version
-find_program(PYTHON NAMES python3.8 python3)
+find_program(PYTHON NAMES python3.9 python3)
 
 add_library(folly_headers INTERFACE)
 target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
@@ -218,7 +218,7 @@ find_package(zstd REQUIRED)
 # clang-12 does NOT work. clang fails with the following error :-
 # configure: error: gcc with GNU99 support required
 
-set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no")
+set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no" "--disable-libdebuginfod")
 if (ASAN)
   list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes")
 endif()
@@ -252,7 +252,28 @@ set(CMAKE_BUILD_RPATH
 )
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
-include_directories(SYSTEM "${DRGN_PATH}")
+# This header from elfutils is not in the right place. Fake the path manually.
+add_custom_command(TARGET libdrgn POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy
+${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/libdrgn/velfutils/libdw/known-dwarf.h
+${CMAKE_CURRENT_BINARY_DIR}/elfutils/known-dwarf.h)
+
+add_library(drgn INTERFACE)
+add_dependencies(drgn libdrgn)
+target_include_directories(drgn SYSTEM INTERFACE
+  "${DRGN_PATH}"
+  "${DRGN_PATH}/include"
+  "${PROJECT_SOURCE_DIR}/extern/drgn/libdrgn/velfutils"
+)
+target_link_options(drgn INTERFACE
+  "-L${DRGN_PATH}/.libs"
+  "-L${DRGN_PATH}/velfutils/libdw"
+)
+target_link_libraries(drgn INTERFACE
+  "-ldrgn"
+  "-ldw"
+)
+
+
 
 if (STATIC_LINK)
   # glog links against the `gflags` target, which is an alias for `gflags_shared`
@@ -288,7 +309,6 @@ add_library(oicore
   oi/PaddingHunter.cpp
   oi/Serialize.cpp
 )
-add_dependencies(oicore libdrgn)
 target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -299,6 +319,7 @@ target_link_libraries(oicore
 
   ${Boost_LIBRARIES}
   Boost::headers
+  drgn
   glog::glog
   range-v3
   resources
@@ -322,9 +343,6 @@ else()
 endif()
 
 target_link_libraries(oicore
-  "-L${DRGN_PATH}/.libs"
-  drgn
-  dw
   pthread
 )
 
@@ -365,8 +383,12 @@ add_executable(oilgen
 target_link_libraries(oilgen
   drgn_utils
   oicore
-  clangTooling
 )
+if (FORCE_LLVM_STATIC)
+  target_link_libraries(oilgen
+    clangTooling
+  )
+endif()
 
 ### Object Introspection cache Printer (OIP)
 add_executable(oip tools/OIP.cpp)
diff --git a/cmake/StandardProjectSettings.cmake b/cmake/StandardProjectSettings.cmake
index 77519bb..e90bf8d 100644
--- a/cmake/StandardProjectSettings.cmake
+++ b/cmake/StandardProjectSettings.cmake
@@ -16,6 +16,7 @@ endif()
 
 # Generate compile_commands.json to make it easier to work with clang based tools
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES})
 
 option(ENABLE_IPO "Enable Interprocedural Optimization, aka Link Time Optimization (LTO)" OFF)
 
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index bb73871..ad5efa7 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus)
 add_library(drgn_utils DrgnUtils.cpp)
 target_link_libraries(drgn_utils
   glog::glog
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
-add_dependencies(drgn_utils libdrgn)
 
 add_library(symbol_service
   Descs.cpp
@@ -22,8 +19,6 @@ target_link_libraries(symbol_service
   Boost::headers
   ${Boost_LIBRARIES}
   glog::glog
-
-  dw
 )
 
 add_library(features Features.cpp)
diff --git a/oi/SymbolService.cpp b/oi/SymbolService.cpp
index 15d19d2..6952cca 100644
--- a/oi/SymbolService.cpp
+++ b/oi/SymbolService.cpp
@@ -31,7 +31,7 @@ extern "C" {
 #include <elfutils/libdwfl.h>
 
 #include "drgn.h"
-#include "dwarf.h"
+#include "libdw/dwarf.h"
 }
 
 namespace fs = std::filesystem;
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 6c4da13..6cd201e 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -20,12 +20,10 @@ add_library(type_graph
   TypeIdentifier.cpp
   Types.cpp
 )
-add_dependencies(type_graph libdrgn)
 target_link_libraries(type_graph
   container_info
   symbol_service
 
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
 target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 4d235b5..aee09ad 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -28,7 +28,7 @@ add_link_options(-no-pie)
 set(INTEGRATION_TEST_TARGET_SRC integration_test_target.cpp)
 set(INTEGRATION_TEST_RUNNER_SRC integration_test_runner.cpp)
 
-find_program(PYTHON_CMD NAMES python3.6 python3)
+find_program(PYTHON_CMD NAMES python3.9 python3)
 
 set(INTEGRATION_TEST_THRIFT_SRCS ${THRIFT_TESTS})
 list(TRANSFORM INTEGRATION_TEST_THRIFT_SRCS APPEND ".thrift")

From 58886c3571b3acee63dbd63632304b66437ae968 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Mon, 19 Feb 2024 15:29:01 +0000
Subject: [PATCH 161/188] tests: add ClangTypeParserTest

Currently there is no testing for ClangTypeParser even though it's used in
production. This is because adding integration tests is very hard: they require
testing the build time behaviour at runtime, or else they'd be build failures
intead of test failures. There's a PR available for integration tests but it's
incomplete.

In contrast ClangTypeParser can be sort of unit tested. This follows the
structure of `test/test_drgn_parser.cpp` with some differences. There is a
tonne of boilerplate for setting up the Clang tool, and this set of testing
operates on type names instead of OID functions. The new tests are also
incredibly slow as they compile the entire `integration_test_target.cpp` (which
is huge) for every test case. I don't think this is avoidable without
compromising the separation of the tests somewhat due to the way Clang tooling
forces the code to be structured.

Test plan:
- Tested locally
- CI
---
 oi/type_graph/ClangTypeParserTest.cpp | 188 ++++++++++++++++++++++++++
 test/CMakeLists.txt                   |  25 ++++
 2 files changed, 213 insertions(+)
 create mode 100644 oi/type_graph/ClangTypeParserTest.cpp

diff --git a/oi/type_graph/ClangTypeParserTest.cpp b/oi/type_graph/ClangTypeParserTest.cpp
new file mode 100644
index 0000000..f3fa832
--- /dev/null
+++ b/oi/type_graph/ClangTypeParserTest.cpp
@@ -0,0 +1,188 @@
+#include <clang/AST/Mangle.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Lookup.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/CompilationDatabase.h>
+#include <clang/Tooling/Tooling.h>
+#include <gtest/gtest.h>
+
+#include <range/v3/core.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <string>
+#include <string_view>
+
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+
+using namespace oi::detail;
+using namespace oi::detail::type_graph;
+
+class ClangTypeParserTest : public ::testing::Test {
+ public:
+  std::string run(std::string_view function, ClangTypeParserOptions opt);
+  void test(std::string_view function,
+            std::string_view expected,
+            ClangTypeParserOptions opts = {
+                .chaseRawPointers = true,
+                .readEnumValues = true,
+            });
+};
+
+// This stuff is a total mess. Set up factories as ClangTooling expects them.
+class ConsumerContext;
+class CreateTypeGraphConsumer;
+
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class ConsumerContext {
+ public:
+  ConsumerContext(
+      const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos_)
+      : containerInfos{containerInfos_} {
+  }
+  std::string_view fullyQualifiedName;
+  ClangTypeParserOptions opts;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+  TypeGraph typeGraph;
+  Type* result = nullptr;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    const clang::Type* type = nullptr;
+    for (const clang::Type* ty : Context.getTypes()) {
+      std::string fqnWithoutTemplateParams;
+      switch (ty->getTypeClass()) {
+        case clang::Type::Record:
+          fqnWithoutTemplateParams = llvm::cast<const clang::RecordType>(ty)
+                                         ->getDecl()
+                                         ->getQualifiedNameAsString();
+          break;
+        default:
+          continue;
+      }
+      if (fqnWithoutTemplateParams != ctx.fullyQualifiedName)
+        continue;
+
+      type = ty;
+      break;
+    }
+    EXPECT_NE(type, nullptr);
+
+    ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, ctx.opts};
+    ctx.result = &parser.parse(Context, *ctx.sema, *type);
+  }
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
+}
+
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    [[maybe_unused]] clang::CompilerInstance& CI,
+    [[maybe_unused]] clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+std::string ClangTypeParserTest::run(std::string_view type,
+                                     ClangTypeParserOptions opts) {
+  std::string err{"failed to load compilation database"};
+  auto db =
+      clang::tooling::CompilationDatabase::loadFromDirectory(BUILD_DIR, err);
+  if (!db) {
+    throw std::runtime_error("failed to load compilation database");
+  }
+
+  std::vector<std::unique_ptr<ContainerInfo>> cis;
+  ConsumerContext ctx{cis};
+  ctx.fullyQualifiedName = type;
+  ctx.opts = std::move(opts);
+  CreateTypeGraphActionFactory factory{ctx};
+
+  std::vector<std::string> sourcePaths{TARGET_CPP_PATH};
+  clang::tooling::ClangTool tool{*db, sourcePaths};
+  if (auto retCode = tool.run(&factory); retCode != 0) {
+    throw std::runtime_error("clang type parsing failed");
+  }
+
+  std::stringstream out;
+  NodeTracker tracker;
+  Printer printer{out, tracker, ctx.typeGraph.size()};
+  printer.print(*ctx.result);
+
+  return std::move(out).str();
+}
+
+void ClangTypeParserTest::test(std::string_view type,
+                               std::string_view expected,
+                               ClangTypeParserOptions opts) {
+  std::string actual = run(type, std::move(opts));
+  expected.remove_prefix(1);  // Remove initial '\n'
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(ClangTypeParserTest, SimpleStruct) {
+  test("ns_simple::SimpleStruct", R"(
+[0] Struct: SimpleStruct [ns_simple::SimpleStruct] (size: 16, align: 8)
+      Member: a (offset: 0)
+        Primitive: int32_t
+      Member: b (offset: 4)
+        Primitive: int8_t
+      Member: c (offset: 8)
+        Primitive: int64_t
+)");
+}
+
+TEST_F(ClangTypeParserTest, MemberAlignment) {
+  test("ns_alignment::MemberAlignment", R"(stuff)");
+}
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 287ebcd..ccdb606 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -69,6 +69,31 @@ target_link_libraries(test_type_graph
 include(GoogleTest)
 gtest_discover_tests(test_type_graph)
 
+add_executable(test_clang_type_parser
+  main.cpp
+  ../oi/type_graph/ClangTypeParserTest.cpp
+)
+add_dependencies(test_clang_type_parser integration_test_target)
+target_compile_definitions(test_clang_type_parser PRIVATE
+  TARGET_CPP_PATH="${CMAKE_CURRENT_BINARY_DIR}/integration/integration_test_target.cpp"
+  BUILD_DIR="${CMAKE_BINARY_DIR}"
+)
+target_link_libraries(test_clang_type_parser
+  codegen
+  container_info
+  type_graph
+
+  range-v3
+
+  GTest::gmock_main
+)
+if (FORCE_LLVM_STATIC)
+  target_link_libraries(test_clang_type_parser clangTooling ${llvm_libs})
+else()
+  target_link_libraries(test_clang_type_parser clang-cpp LLVM)
+endif()
+gtest_discover_tests(test_clang_type_parser)
+
 cpp_unittest(
   NAME test_parser
   SRCS test_parser.cpp

From 3bfaa01cf92d1b2b8f7b9ada1fe5c8ce3de31ce4 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 20 Feb 2024 14:54:14 +0000
Subject: [PATCH 162/188] convert circleci lint job to github actions

Start the migration from CircleCI to GitHub Actions with migrating the lint
job. Used the structure from @robandpdx to setup Nix and use a GitHub key.
Restructured the check from `nix flake check` to
`nix fmt; git diff --exit-code` so we get a full patch again.

Test plan:
- Submitted this PR with a formatting error. CI failed. Submitted without and
  it passed.

Co-Authored By: Rob Anderson <robandpdx@github.com>
---
 .circleci/config.yml                       | 14 --------------
 .github/workflows/object-introspection.yml | 15 +++++++++++++++
 oi/OIDebugger.cpp                          |  2 +-
 3 files changed, 16 insertions(+), 15 deletions(-)
 create mode 100644 .github/workflows/object-introspection.yml

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 443439f..69a0807 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -3,8 +3,6 @@ version: 2.1
 workflows:
   object-introspection:
     jobs:
-      - lint
-
       - build:
           name: build-gcc
           cc: /usr/bin/gcc
@@ -33,10 +31,6 @@ workflows:
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_member|OilIntegration.fbstring_.*|OilIntegration.capture_keys_string|OilIntegration.capture_keys_multi_level"
 
 executors:
-  nix-docker:
-    docker:
-      - image: nixos/nix:latest
-    resource_class: small
   ubuntu-docker:
     docker:
       - image: ubuntu:jammy
@@ -47,14 +41,6 @@ executors:
     resource_class: 2xlarge
 
 jobs:
-  lint:
-    executor: nix-docker
-    steps:
-      - checkout
-      - run:
-          name: Flake check
-          command: nix --experimental-features 'nix-command flakes' flake check
-
   build:
     # TODO this job could be run in Docker
     executor: big-boy
diff --git a/.github/workflows/object-introspection.yml b/.github/workflows/object-introspection.yml
new file mode 100644
index 0000000..921b6e3
--- /dev/null
+++ b/.github/workflows/object-introspection.yml
@@ -0,0 +1,15 @@
+name: facebookexperimental/object-introspection
+on:
+  pull_request:
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: cachix/install-nix-action@v25
+      with:
+        github_access_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Flake check
+      run: |-
+        nix --experimental-features 'nix-command flakes' fmt
+        git diff --exit-code
diff --git a/oi/OIDebugger.cpp b/oi/OIDebugger.cpp
index ee9f040..24842c7 100644
--- a/oi/OIDebugger.cpp
+++ b/oi/OIDebugger.cpp
@@ -30,7 +30,7 @@
 #include <chrono>
 #include <cstddef>
 #include <cstdio>
-#include <cstring>
+#include  <cstring>
 #include <numeric>
 #include <span>
 

From 579ee4686c4420551e760ffcb6add81b247b4603 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 20 Feb 2024 14:57:55 +0000
Subject: [PATCH 163/188] convert circleci lint job to github actions

Start the migration from CircleCI to GitHub Actions with migrating the lint
job. Used the structure from @robandpdx to setup Nix and use a GitHub key.
Restructured the check from `nix flake check` to
`nix fmt; git diff --exit-code` so we get a full patch again.

Test plan:
- Submitted this PR with a formatting error. CI failed. Submitted without and
  it passed.

Co-authored-by: Rob Anderson <robandpdx@github.com>
---
 .circleci/config.yml                       | 14 --------------
 .github/workflows/object-introspection.yml | 15 +++++++++++++++
 oi/OIDebugger.cpp                          |  2 +-
 3 files changed, 16 insertions(+), 15 deletions(-)
 create mode 100644 .github/workflows/object-introspection.yml

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 443439f..69a0807 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -3,8 +3,6 @@ version: 2.1
 workflows:
   object-introspection:
     jobs:
-      - lint
-
       - build:
           name: build-gcc
           cc: /usr/bin/gcc
@@ -33,10 +31,6 @@ workflows:
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_member|OilIntegration.fbstring_.*|OilIntegration.capture_keys_string|OilIntegration.capture_keys_multi_level"
 
 executors:
-  nix-docker:
-    docker:
-      - image: nixos/nix:latest
-    resource_class: small
   ubuntu-docker:
     docker:
       - image: ubuntu:jammy
@@ -47,14 +41,6 @@ executors:
     resource_class: 2xlarge
 
 jobs:
-  lint:
-    executor: nix-docker
-    steps:
-      - checkout
-      - run:
-          name: Flake check
-          command: nix --experimental-features 'nix-command flakes' flake check
-
   build:
     # TODO this job could be run in Docker
     executor: big-boy
diff --git a/.github/workflows/object-introspection.yml b/.github/workflows/object-introspection.yml
new file mode 100644
index 0000000..921b6e3
--- /dev/null
+++ b/.github/workflows/object-introspection.yml
@@ -0,0 +1,15 @@
+name: facebookexperimental/object-introspection
+on:
+  pull_request:
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: cachix/install-nix-action@v25
+      with:
+        github_access_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Flake check
+      run: |-
+        nix --experimental-features 'nix-command flakes' fmt
+        git diff --exit-code
diff --git a/oi/OIDebugger.cpp b/oi/OIDebugger.cpp
index ee9f040..24842c7 100644
--- a/oi/OIDebugger.cpp
+++ b/oi/OIDebugger.cpp
@@ -30,7 +30,7 @@
 #include <chrono>
 #include <cstddef>
 #include <cstdio>
-#include <cstring>
+#include  <cstring>
 #include <numeric>
 #include <span>
 

From 121079f31b5b8a86be4076daf63f4d182a2a3e07 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 20 Feb 2024 15:05:17 +0000
Subject: [PATCH 164/188] convert circleci lint job to github actions

Start the migration from CircleCI to GitHub Actions with migrating the lint
job. Used the structure from @robandpdx to setup Nix and use a GitHub key.
Restructured the check from `nix flake check` to
`nix fmt; git diff --exit-code` so we get a full patch again.

Test plan:
- Submitted this PR with a formatting error. CI failed. Submitted without and
  it passed.

Co-authored-by: Rob Anderson <robandpdx@github.com>
---
 .circleci/config.yml                       | 14 --------------
 .github/workflows/object-introspection.yml | 15 +++++++++++++++
 2 files changed, 15 insertions(+), 14 deletions(-)
 create mode 100644 .github/workflows/object-introspection.yml

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 443439f..69a0807 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -3,8 +3,6 @@ version: 2.1
 workflows:
   object-introspection:
     jobs:
-      - lint
-
       - build:
           name: build-gcc
           cc: /usr/bin/gcc
@@ -33,10 +31,6 @@ workflows:
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_member|OilIntegration.fbstring_.*|OilIntegration.capture_keys_string|OilIntegration.capture_keys_multi_level"
 
 executors:
-  nix-docker:
-    docker:
-      - image: nixos/nix:latest
-    resource_class: small
   ubuntu-docker:
     docker:
       - image: ubuntu:jammy
@@ -47,14 +41,6 @@ executors:
     resource_class: 2xlarge
 
 jobs:
-  lint:
-    executor: nix-docker
-    steps:
-      - checkout
-      - run:
-          name: Flake check
-          command: nix --experimental-features 'nix-command flakes' flake check
-
   build:
     # TODO this job could be run in Docker
     executor: big-boy
diff --git a/.github/workflows/object-introspection.yml b/.github/workflows/object-introspection.yml
new file mode 100644
index 0000000..921b6e3
--- /dev/null
+++ b/.github/workflows/object-introspection.yml
@@ -0,0 +1,15 @@
+name: facebookexperimental/object-introspection
+on:
+  pull_request:
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: cachix/install-nix-action@v25
+      with:
+        github_access_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Flake check
+      run: |-
+        nix --experimental-features 'nix-command flakes' fmt
+        git diff --exit-code

From b3f6fb7bf6c4ff579b9225a54acd87aa7557c13c Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 20 Feb 2024 15:07:15 +0000
Subject: [PATCH 165/188] convert circleci lint job to github actions

Start the migration from CircleCI to GitHub Actions with migrating the lint
job. Used the structure from @robandpdx to setup Nix and use a GitHub key.
Restructured the check from `nix flake check` to
`nix fmt; git diff --exit-code` so we get a full patch again.

Test plan:
- Submitted this PR with a formatting error. CI failed. Submitted without and
  it passed.

Co-authored-by: Rob Anderson <robandpdx@github.com>
---
 .circleci/config.yml                       | 14 --------------
 .github/workflows/object-introspection.yml | 15 +++++++++++++++
 2 files changed, 15 insertions(+), 14 deletions(-)
 create mode 100644 .github/workflows/object-introspection.yml

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 443439f..69a0807 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -3,8 +3,6 @@ version: 2.1
 workflows:
   object-introspection:
     jobs:
-      - lint
-
       - build:
           name: build-gcc
           cc: /usr/bin/gcc
@@ -33,10 +31,6 @@ workflows:
           exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|.*fbstring.*|.*std_string_*|.*multi_arg_tb_.*|.*ignored_member|OilIntegration.fbstring_.*|OilIntegration.capture_keys_string|OilIntegration.capture_keys_multi_level"
 
 executors:
-  nix-docker:
-    docker:
-      - image: nixos/nix:latest
-    resource_class: small
   ubuntu-docker:
     docker:
       - image: ubuntu:jammy
@@ -47,14 +41,6 @@ executors:
     resource_class: 2xlarge
 
 jobs:
-  lint:
-    executor: nix-docker
-    steps:
-      - checkout
-      - run:
-          name: Flake check
-          command: nix --experimental-features 'nix-command flakes' flake check
-
   build:
     # TODO this job could be run in Docker
     executor: big-boy
diff --git a/.github/workflows/object-introspection.yml b/.github/workflows/object-introspection.yml
new file mode 100644
index 0000000..70b41aa
--- /dev/null
+++ b/.github/workflows/object-introspection.yml
@@ -0,0 +1,15 @@
+name: facebookexperimental/object-introspection
+on:
+  pull_request:
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: cachix/install-nix-action@v25
+      with:
+        github_access_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: nix fmt
+      run: |-
+        nix --experimental-features 'nix-command flakes' fmt
+        git diff --exit-code

From 418e4a1cde980932ab862feb9d9001946b07ac7a Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 20 Feb 2024 16:03:39 +0000
Subject: [PATCH 166/188] tests: add ClangTypeParserTest

Currently there is no testing for ClangTypeParser even though it's used in
production. This is because adding integration tests is very hard: they require
testing the build time behaviour at runtime, or else they'd be build failures
intead of test failures. There's a PR available for integration tests but it's
incomplete.

In contrast ClangTypeParser can be sort of unit tested. This follows the
structure of `test/test_drgn_parser.cpp` with some differences. There is a
tonne of boilerplate for setting up the Clang tool, and this set of testing
operates on type names instead of OID functions. The new tests are also
incredibly slow as they compile the entire `integration_test_target.cpp` (which
is huge) for every test case. I don't think this is avoidable without
compromising the separation of the tests somewhat due to the way Clang tooling
forces the code to be structured.

Test plan:
- Tested locally
- CI
---
 oi/type_graph/ClangTypeParserTest.cpp | 188 ++++++++++++++++++++++++++
 test/CMakeLists.txt                   |  25 ++++
 2 files changed, 213 insertions(+)
 create mode 100644 oi/type_graph/ClangTypeParserTest.cpp

diff --git a/oi/type_graph/ClangTypeParserTest.cpp b/oi/type_graph/ClangTypeParserTest.cpp
new file mode 100644
index 0000000..f3fa832
--- /dev/null
+++ b/oi/type_graph/ClangTypeParserTest.cpp
@@ -0,0 +1,188 @@
+#include <clang/AST/Mangle.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Lookup.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/CompilationDatabase.h>
+#include <clang/Tooling/Tooling.h>
+#include <gtest/gtest.h>
+
+#include <range/v3/core.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <string>
+#include <string_view>
+
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+
+using namespace oi::detail;
+using namespace oi::detail::type_graph;
+
+class ClangTypeParserTest : public ::testing::Test {
+ public:
+  std::string run(std::string_view function, ClangTypeParserOptions opt);
+  void test(std::string_view function,
+            std::string_view expected,
+            ClangTypeParserOptions opts = {
+                .chaseRawPointers = true,
+                .readEnumValues = true,
+            });
+};
+
+// This stuff is a total mess. Set up factories as ClangTooling expects them.
+class ConsumerContext;
+class CreateTypeGraphConsumer;
+
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class ConsumerContext {
+ public:
+  ConsumerContext(
+      const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos_)
+      : containerInfos{containerInfos_} {
+  }
+  std::string_view fullyQualifiedName;
+  ClangTypeParserOptions opts;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+  TypeGraph typeGraph;
+  Type* result = nullptr;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    const clang::Type* type = nullptr;
+    for (const clang::Type* ty : Context.getTypes()) {
+      std::string fqnWithoutTemplateParams;
+      switch (ty->getTypeClass()) {
+        case clang::Type::Record:
+          fqnWithoutTemplateParams = llvm::cast<const clang::RecordType>(ty)
+                                         ->getDecl()
+                                         ->getQualifiedNameAsString();
+          break;
+        default:
+          continue;
+      }
+      if (fqnWithoutTemplateParams != ctx.fullyQualifiedName)
+        continue;
+
+      type = ty;
+      break;
+    }
+    EXPECT_NE(type, nullptr);
+
+    ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, ctx.opts};
+    ctx.result = &parser.parse(Context, *ctx.sema, *type);
+  }
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
+}
+
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    [[maybe_unused]] clang::CompilerInstance& CI,
+    [[maybe_unused]] clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+std::string ClangTypeParserTest::run(std::string_view type,
+                                     ClangTypeParserOptions opts) {
+  std::string err{"failed to load compilation database"};
+  auto db =
+      clang::tooling::CompilationDatabase::loadFromDirectory(BUILD_DIR, err);
+  if (!db) {
+    throw std::runtime_error("failed to load compilation database");
+  }
+
+  std::vector<std::unique_ptr<ContainerInfo>> cis;
+  ConsumerContext ctx{cis};
+  ctx.fullyQualifiedName = type;
+  ctx.opts = std::move(opts);
+  CreateTypeGraphActionFactory factory{ctx};
+
+  std::vector<std::string> sourcePaths{TARGET_CPP_PATH};
+  clang::tooling::ClangTool tool{*db, sourcePaths};
+  if (auto retCode = tool.run(&factory); retCode != 0) {
+    throw std::runtime_error("clang type parsing failed");
+  }
+
+  std::stringstream out;
+  NodeTracker tracker;
+  Printer printer{out, tracker, ctx.typeGraph.size()};
+  printer.print(*ctx.result);
+
+  return std::move(out).str();
+}
+
+void ClangTypeParserTest::test(std::string_view type,
+                               std::string_view expected,
+                               ClangTypeParserOptions opts) {
+  std::string actual = run(type, std::move(opts));
+  expected.remove_prefix(1);  // Remove initial '\n'
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(ClangTypeParserTest, SimpleStruct) {
+  test("ns_simple::SimpleStruct", R"(
+[0] Struct: SimpleStruct [ns_simple::SimpleStruct] (size: 16, align: 8)
+      Member: a (offset: 0)
+        Primitive: int32_t
+      Member: b (offset: 4)
+        Primitive: int8_t
+      Member: c (offset: 8)
+        Primitive: int64_t
+)");
+}
+
+TEST_F(ClangTypeParserTest, MemberAlignment) {
+  test("ns_alignment::MemberAlignment", R"(stuff)");
+}
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 287ebcd..ccdb606 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -69,6 +69,31 @@ target_link_libraries(test_type_graph
 include(GoogleTest)
 gtest_discover_tests(test_type_graph)
 
+add_executable(test_clang_type_parser
+  main.cpp
+  ../oi/type_graph/ClangTypeParserTest.cpp
+)
+add_dependencies(test_clang_type_parser integration_test_target)
+target_compile_definitions(test_clang_type_parser PRIVATE
+  TARGET_CPP_PATH="${CMAKE_CURRENT_BINARY_DIR}/integration/integration_test_target.cpp"
+  BUILD_DIR="${CMAKE_BINARY_DIR}"
+)
+target_link_libraries(test_clang_type_parser
+  codegen
+  container_info
+  type_graph
+
+  range-v3
+
+  GTest::gmock_main
+)
+if (FORCE_LLVM_STATIC)
+  target_link_libraries(test_clang_type_parser clangTooling ${llvm_libs})
+else()
+  target_link_libraries(test_clang_type_parser clang-cpp LLVM)
+endif()
+gtest_discover_tests(test_clang_type_parser)
+
 cpp_unittest(
   NAME test_parser
   SRCS test_parser.cpp

From d1b62bc7c136756ce984809a53c844961b198052 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 20 Feb 2024 16:03:39 +0000
Subject: [PATCH 167/188] remove internal build config and update for CentOS 9

Previously we maintained three types of builds: a fully internal BUCK build, a
CMake build with modifications to use things from an internal toolchain, and an
open source CMake build.

As far as I'm concerned the intermediate build is not useful because our source
is readily available in both an internal and external form. Use cases as
follows:
1. BUCK build for distributing widely.
2. BUCK build for getting a static binary that can be run on any machine.
3. CMake build for primary development.
4. CMake build for external CI.

With the internal update to CentOS Stream 9 an unmodified CMake build now works
readily. This change patches up some things that were relying on system headers
that should have been vendored and cleans up drgn dependencies.

Test plan:
- It builds.
- TODO: Document CentOS 9 installation.
---
 CMakeLists.txt                      | 38 +++++++++++++++++++++++------
 cmake/StandardProjectSettings.cmake |  1 +
 oi/CMakeLists.txt                   |  5 ----
 oi/SymbolService.cpp                |  2 +-
 oi/type_graph/CMakeLists.txt        |  2 --
 test/integration/CMakeLists.txt     |  2 +-
 6 files changed, 33 insertions(+), 17 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b6fecd1..e1717ce 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -152,7 +152,7 @@ if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn")
 endif()
 
 ### Select Python version
-find_program(PYTHON NAMES python3.8 python3)
+find_program(PYTHON NAMES python3.9 python3)
 
 add_library(folly_headers INTERFACE)
 target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
@@ -218,7 +218,7 @@ find_package(zstd REQUIRED)
 # clang-12 does NOT work. clang fails with the following error :-
 # configure: error: gcc with GNU99 support required
 
-set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no")
+set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no" "--disable-libdebuginfod")
 if (ASAN)
   list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes")
 endif()
@@ -252,7 +252,28 @@ set(CMAKE_BUILD_RPATH
 )
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
-include_directories(SYSTEM "${DRGN_PATH}")
+# This header from elfutils is not in the right place. Fake the path manually.
+add_custom_command(TARGET libdrgn POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy
+${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/libdrgn/velfutils/libdw/known-dwarf.h
+${CMAKE_CURRENT_BINARY_DIR}/elfutils/known-dwarf.h)
+
+add_library(drgn INTERFACE)
+add_dependencies(drgn libdrgn)
+target_include_directories(drgn SYSTEM INTERFACE
+  "${DRGN_PATH}"
+  "${DRGN_PATH}/include"
+  "${PROJECT_SOURCE_DIR}/extern/drgn/libdrgn/velfutils"
+)
+target_link_options(drgn INTERFACE
+  "-L${DRGN_PATH}/.libs"
+  "-L${DRGN_PATH}/velfutils/libdw"
+)
+target_link_libraries(drgn INTERFACE
+  "-ldrgn"
+  "-ldw"
+)
+
+
 
 if (STATIC_LINK)
   # glog links against the `gflags` target, which is an alias for `gflags_shared`
@@ -288,7 +309,6 @@ add_library(oicore
   oi/PaddingHunter.cpp
   oi/Serialize.cpp
 )
-add_dependencies(oicore libdrgn)
 target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -299,6 +319,7 @@ target_link_libraries(oicore
 
   ${Boost_LIBRARIES}
   Boost::headers
+  drgn
   glog::glog
   range-v3
   resources
@@ -322,9 +343,6 @@ else()
 endif()
 
 target_link_libraries(oicore
-  "-L${DRGN_PATH}/.libs"
-  drgn
-  dw
   pthread
 )
 
@@ -365,8 +383,12 @@ add_executable(oilgen
 target_link_libraries(oilgen
   drgn_utils
   oicore
-  clangTooling
 )
+if (FORCE_LLVM_STATIC)
+  target_link_libraries(oilgen
+    clangTooling
+  )
+endif()
 
 ### Object Introspection cache Printer (OIP)
 add_executable(oip tools/OIP.cpp)
diff --git a/cmake/StandardProjectSettings.cmake b/cmake/StandardProjectSettings.cmake
index 77519bb..e90bf8d 100644
--- a/cmake/StandardProjectSettings.cmake
+++ b/cmake/StandardProjectSettings.cmake
@@ -16,6 +16,7 @@ endif()
 
 # Generate compile_commands.json to make it easier to work with clang based tools
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES})
 
 option(ENABLE_IPO "Enable Interprocedural Optimization, aka Link Time Optimization (LTO)" OFF)
 
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index bb73871..ad5efa7 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus)
 add_library(drgn_utils DrgnUtils.cpp)
 target_link_libraries(drgn_utils
   glog::glog
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
-add_dependencies(drgn_utils libdrgn)
 
 add_library(symbol_service
   Descs.cpp
@@ -22,8 +19,6 @@ target_link_libraries(symbol_service
   Boost::headers
   ${Boost_LIBRARIES}
   glog::glog
-
-  dw
 )
 
 add_library(features Features.cpp)
diff --git a/oi/SymbolService.cpp b/oi/SymbolService.cpp
index 15d19d2..6952cca 100644
--- a/oi/SymbolService.cpp
+++ b/oi/SymbolService.cpp
@@ -31,7 +31,7 @@ extern "C" {
 #include <elfutils/libdwfl.h>
 
 #include "drgn.h"
-#include "dwarf.h"
+#include "libdw/dwarf.h"
 }
 
 namespace fs = std::filesystem;
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 6c4da13..6cd201e 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -20,12 +20,10 @@ add_library(type_graph
   TypeIdentifier.cpp
   Types.cpp
 )
-add_dependencies(type_graph libdrgn)
 target_link_libraries(type_graph
   container_info
   symbol_service
 
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
 target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt
index 4d235b5..aee09ad 100644
--- a/test/integration/CMakeLists.txt
+++ b/test/integration/CMakeLists.txt
@@ -28,7 +28,7 @@ add_link_options(-no-pie)
 set(INTEGRATION_TEST_TARGET_SRC integration_test_target.cpp)
 set(INTEGRATION_TEST_RUNNER_SRC integration_test_runner.cpp)
 
-find_program(PYTHON_CMD NAMES python3.6 python3)
+find_program(PYTHON_CMD NAMES python3.9 python3)
 
 set(INTEGRATION_TEST_THRIFT_SRCS ${THRIFT_TESTS})
 list(TRANSFORM INTEGRATION_TEST_THRIFT_SRCS APPEND ".thrift")

From af84c3c93b3bddacfa5fae63d217fc86f74d7490 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 20 Feb 2024 16:03:39 +0000
Subject: [PATCH 168/188] tests: add ClangTypeParserTest

Currently there is no testing for ClangTypeParser even though it's used in
production. This is because adding integration tests is very hard: they require
testing the build time behaviour at runtime, or else they'd be build failures
intead of test failures. There's a PR available for integration tests but it's
incomplete.

In contrast ClangTypeParser can be sort of unit tested. This follows the
structure of `test/test_drgn_parser.cpp` with some differences. There is a
tonne of boilerplate for setting up the Clang tool, and this set of testing
operates on type names instead of OID functions. The new tests are also
incredibly slow as they compile the entire `integration_test_target.cpp` (which
is huge) for every test case. I don't think this is avoidable without
compromising the separation of the tests somewhat due to the way Clang tooling
forces the code to be structured.

Test plan:
- Tested locally
- CI
---
 .circleci/config.yml                  |   1 +
 oi/type_graph/ClangTypeParserTest.cpp | 188 ++++++++++++++++++++++++++
 test/CMakeLists.txt                   |  25 ++++
 3 files changed, 214 insertions(+)
 create mode 100644 oi/type_graph/ClangTypeParserTest.cpp

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 69a0807..a5203cb 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -104,6 +104,7 @@ jobs:
           paths:
             - build/*
             - extern/*
+            - include/*
             - types/*
 
   test:
diff --git a/oi/type_graph/ClangTypeParserTest.cpp b/oi/type_graph/ClangTypeParserTest.cpp
new file mode 100644
index 0000000..f3fa832
--- /dev/null
+++ b/oi/type_graph/ClangTypeParserTest.cpp
@@ -0,0 +1,188 @@
+#include <clang/AST/Mangle.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Lookup.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/CompilationDatabase.h>
+#include <clang/Tooling/Tooling.h>
+#include <gtest/gtest.h>
+
+#include <range/v3/core.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <string>
+#include <string_view>
+
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+
+using namespace oi::detail;
+using namespace oi::detail::type_graph;
+
+class ClangTypeParserTest : public ::testing::Test {
+ public:
+  std::string run(std::string_view function, ClangTypeParserOptions opt);
+  void test(std::string_view function,
+            std::string_view expected,
+            ClangTypeParserOptions opts = {
+                .chaseRawPointers = true,
+                .readEnumValues = true,
+            });
+};
+
+// This stuff is a total mess. Set up factories as ClangTooling expects them.
+class ConsumerContext;
+class CreateTypeGraphConsumer;
+
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class ConsumerContext {
+ public:
+  ConsumerContext(
+      const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos_)
+      : containerInfos{containerInfos_} {
+  }
+  std::string_view fullyQualifiedName;
+  ClangTypeParserOptions opts;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+  TypeGraph typeGraph;
+  Type* result = nullptr;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    const clang::Type* type = nullptr;
+    for (const clang::Type* ty : Context.getTypes()) {
+      std::string fqnWithoutTemplateParams;
+      switch (ty->getTypeClass()) {
+        case clang::Type::Record:
+          fqnWithoutTemplateParams = llvm::cast<const clang::RecordType>(ty)
+                                         ->getDecl()
+                                         ->getQualifiedNameAsString();
+          break;
+        default:
+          continue;
+      }
+      if (fqnWithoutTemplateParams != ctx.fullyQualifiedName)
+        continue;
+
+      type = ty;
+      break;
+    }
+    EXPECT_NE(type, nullptr);
+
+    ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, ctx.opts};
+    ctx.result = &parser.parse(Context, *ctx.sema, *type);
+  }
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
+}
+
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    [[maybe_unused]] clang::CompilerInstance& CI,
+    [[maybe_unused]] clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+std::string ClangTypeParserTest::run(std::string_view type,
+                                     ClangTypeParserOptions opts) {
+  std::string err{"failed to load compilation database"};
+  auto db =
+      clang::tooling::CompilationDatabase::loadFromDirectory(BUILD_DIR, err);
+  if (!db) {
+    throw std::runtime_error("failed to load compilation database");
+  }
+
+  std::vector<std::unique_ptr<ContainerInfo>> cis;
+  ConsumerContext ctx{cis};
+  ctx.fullyQualifiedName = type;
+  ctx.opts = std::move(opts);
+  CreateTypeGraphActionFactory factory{ctx};
+
+  std::vector<std::string> sourcePaths{TARGET_CPP_PATH};
+  clang::tooling::ClangTool tool{*db, sourcePaths};
+  if (auto retCode = tool.run(&factory); retCode != 0) {
+    throw std::runtime_error("clang type parsing failed");
+  }
+
+  std::stringstream out;
+  NodeTracker tracker;
+  Printer printer{out, tracker, ctx.typeGraph.size()};
+  printer.print(*ctx.result);
+
+  return std::move(out).str();
+}
+
+void ClangTypeParserTest::test(std::string_view type,
+                               std::string_view expected,
+                               ClangTypeParserOptions opts) {
+  std::string actual = run(type, std::move(opts));
+  expected.remove_prefix(1);  // Remove initial '\n'
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(ClangTypeParserTest, SimpleStruct) {
+  test("ns_simple::SimpleStruct", R"(
+[0] Struct: SimpleStruct [ns_simple::SimpleStruct] (size: 16, align: 8)
+      Member: a (offset: 0)
+        Primitive: int32_t
+      Member: b (offset: 4)
+        Primitive: int8_t
+      Member: c (offset: 8)
+        Primitive: int64_t
+)");
+}
+
+TEST_F(ClangTypeParserTest, MemberAlignment) {
+  test("ns_alignment::MemberAlignment", R"(stuff)");
+}
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 287ebcd..ccdb606 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -69,6 +69,31 @@ target_link_libraries(test_type_graph
 include(GoogleTest)
 gtest_discover_tests(test_type_graph)
 
+add_executable(test_clang_type_parser
+  main.cpp
+  ../oi/type_graph/ClangTypeParserTest.cpp
+)
+add_dependencies(test_clang_type_parser integration_test_target)
+target_compile_definitions(test_clang_type_parser PRIVATE
+  TARGET_CPP_PATH="${CMAKE_CURRENT_BINARY_DIR}/integration/integration_test_target.cpp"
+  BUILD_DIR="${CMAKE_BINARY_DIR}"
+)
+target_link_libraries(test_clang_type_parser
+  codegen
+  container_info
+  type_graph
+
+  range-v3
+
+  GTest::gmock_main
+)
+if (FORCE_LLVM_STATIC)
+  target_link_libraries(test_clang_type_parser clangTooling ${llvm_libs})
+else()
+  target_link_libraries(test_clang_type_parser clang-cpp LLVM)
+endif()
+gtest_discover_tests(test_clang_type_parser)
+
 cpp_unittest(
   NAME test_parser
   SRCS test_parser.cpp

From c496f9dfaadeeecf7c529fa710dd989305d04103 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 20 Feb 2024 16:03:39 +0000
Subject: [PATCH 169/188] tests: add ClangTypeParserTest

Currently there is no testing for ClangTypeParser even though it's used in
production. This is because adding integration tests is very hard: they require
testing the build time behaviour at runtime, or else they'd be build failures
intead of test failures. There's a PR available for integration tests but it's
incomplete.

In contrast ClangTypeParser can be sort of unit tested. This follows the
structure of `test/test_drgn_parser.cpp` with some differences. There is a
tonne of boilerplate for setting up the Clang tool, and this set of testing
operates on type names instead of OID functions. The new tests are also
incredibly slow as they compile the entire `integration_test_target.cpp` (which
is huge) for every test case. I don't think this is avoidable without
compromising the separation of the tests somewhat due to the way Clang tooling
forces the code to be structured.

Test plan:
- Tested locally
- CI
---
 .circleci/config.yml                  |   1 +
 oi/type_graph/ClangTypeParserTest.cpp | 194 ++++++++++++++++++++++++++
 test/CMakeLists.txt                   |  25 ++++
 3 files changed, 220 insertions(+)
 create mode 100644 oi/type_graph/ClangTypeParserTest.cpp

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 69a0807..a5203cb 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -104,6 +104,7 @@ jobs:
           paths:
             - build/*
             - extern/*
+            - include/*
             - types/*
 
   test:
diff --git a/oi/type_graph/ClangTypeParserTest.cpp b/oi/type_graph/ClangTypeParserTest.cpp
new file mode 100644
index 0000000..4671773
--- /dev/null
+++ b/oi/type_graph/ClangTypeParserTest.cpp
@@ -0,0 +1,194 @@
+#include <clang/AST/Mangle.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Lookup.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/CompilationDatabase.h>
+#include <clang/Tooling/Tooling.h>
+#include <gtest/gtest.h>
+
+#include <range/v3/core.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <string>
+#include <string_view>
+
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+
+using namespace oi::detail;
+using namespace oi::detail::type_graph;
+
+class ClangTypeParserTest : public ::testing::Test {
+ public:
+  std::string run(std::string_view function, ClangTypeParserOptions opt);
+  void test(std::string_view function,
+            std::string_view expected,
+            ClangTypeParserOptions opts = {
+                .chaseRawPointers = true,
+                .readEnumValues = true,
+            });
+};
+
+// This stuff is a total mess. Set up factories as ClangTooling expects them.
+class ConsumerContext;
+class CreateTypeGraphConsumer;
+
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class ConsumerContext {
+ public:
+  ConsumerContext(
+      const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos_)
+      : containerInfos{containerInfos_} {
+  }
+  std::string_view fullyQualifiedName;
+  ClangTypeParserOptions opts;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+  TypeGraph typeGraph;
+  Type* result = nullptr;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    const clang::Type* type = nullptr;
+    for (const clang::Type* ty : Context.getTypes()) {
+      std::string fqnWithoutTemplateParams;
+      switch (ty->getTypeClass()) {
+        case clang::Type::Record:
+          fqnWithoutTemplateParams = llvm::cast<const clang::RecordType>(ty)
+                                         ->getDecl()
+                                         ->getQualifiedNameAsString();
+          break;
+        default:
+          continue;
+      }
+      if (fqnWithoutTemplateParams != ctx.fullyQualifiedName)
+        continue;
+
+      type = ty;
+      break;
+    }
+    EXPECT_NE(type, nullptr);
+
+    ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, ctx.opts};
+    ctx.result = &parser.parse(Context, *ctx.sema, *type);
+  }
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
+}
+
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    [[maybe_unused]] clang::CompilerInstance& CI,
+    [[maybe_unused]] clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+std::string ClangTypeParserTest::run(std::string_view type,
+                                     ClangTypeParserOptions opts) {
+  std::string err{"failed to load compilation database"};
+  auto db =
+      clang::tooling::CompilationDatabase::loadFromDirectory(BUILD_DIR, err);
+  if (!db) {
+    throw std::runtime_error("failed to load compilation database");
+  }
+
+  std::vector<std::unique_ptr<ContainerInfo>> cis;
+  ConsumerContext ctx{cis};
+  ctx.fullyQualifiedName = type;
+  ctx.opts = std::move(opts);
+  CreateTypeGraphActionFactory factory{ctx};
+
+  std::vector<std::string> sourcePaths{TARGET_CPP_PATH};
+  clang::tooling::ClangTool tool{*db, sourcePaths};
+  if (auto retCode = tool.run(&factory); retCode != 0) {
+    throw std::runtime_error("clang type parsing failed");
+  }
+
+  std::stringstream out;
+  NodeTracker tracker;
+  Printer printer{out, tracker, ctx.typeGraph.size()};
+  printer.print(*ctx.result);
+
+  return std::move(out).str();
+}
+
+void ClangTypeParserTest::test(std::string_view type,
+                               std::string_view expected,
+                               ClangTypeParserOptions opts) {
+  std::string actual = run(type, std::move(opts));
+  expected.remove_prefix(1);  // Remove initial '\n'
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(ClangTypeParserTest, SimpleStruct) {
+  test("ns_simple::SimpleStruct", R"(
+[0] Struct: SimpleStruct [ns_simple::SimpleStruct] (size: 16, align: 8)
+      Member: a (offset: 0)
+        Primitive: int32_t
+      Member: b (offset: 4)
+        Primitive: int8_t
+      Member: c (offset: 8)
+        Primitive: int64_t
+)");
+}
+
+TEST_F(ClangTypeParserTest, MemberAlignment) {
+  test("ns_alignment::MemberAlignment", R"(
+[0] Struct: MemberAlignment [ns_alignment::MemberAlignment] (size: 64, align: 32)
+      Member: c (offset: 0)
+        Primitive: int8_t
+      Member: c32 (offset: 32, align: 32)
+        Primitive: int8_t
+)");
+}
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 287ebcd..ccdb606 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -69,6 +69,31 @@ target_link_libraries(test_type_graph
 include(GoogleTest)
 gtest_discover_tests(test_type_graph)
 
+add_executable(test_clang_type_parser
+  main.cpp
+  ../oi/type_graph/ClangTypeParserTest.cpp
+)
+add_dependencies(test_clang_type_parser integration_test_target)
+target_compile_definitions(test_clang_type_parser PRIVATE
+  TARGET_CPP_PATH="${CMAKE_CURRENT_BINARY_DIR}/integration/integration_test_target.cpp"
+  BUILD_DIR="${CMAKE_BINARY_DIR}"
+)
+target_link_libraries(test_clang_type_parser
+  codegen
+  container_info
+  type_graph
+
+  range-v3
+
+  GTest::gmock_main
+)
+if (FORCE_LLVM_STATIC)
+  target_link_libraries(test_clang_type_parser clangTooling ${llvm_libs})
+else()
+  target_link_libraries(test_clang_type_parser clang-cpp LLVM)
+endif()
+gtest_discover_tests(test_clang_type_parser)
+
 cpp_unittest(
   NAME test_parser
   SRCS test_parser.cpp

From 21fdcd273421eab328551646daa2df0e0b36d25d Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 20 Feb 2024 16:03:39 +0000
Subject: [PATCH 170/188] tests: add ClangTypeParserTest

Currently there is no testing for ClangTypeParser even though it's used in
production. This is because adding integration tests is very hard: they require
testing the build time behaviour at runtime, or else they'd be build failures
intead of test failures. There's a PR available for integration tests but it's
incomplete.

In contrast ClangTypeParser can be sort of unit tested. This follows the
structure of `test/test_drgn_parser.cpp` with some differences. There is a
tonne of boilerplate for setting up the Clang tool, and this set of testing
operates on type names instead of OID functions. The new tests are also
incredibly slow as they compile the entire `integration_test_target.cpp` (which
is huge) for every test case. I don't think this is avoidable without
compromising the separation of the tests somewhat due to the way Clang tooling
forces the code to be structured.

Test plan:
- Tested locally
- CI
---
 .circleci/config.yml                  |   3 +-
 oi/type_graph/ClangTypeParserTest.cpp | 194 ++++++++++++++++++++++++++
 test/CMakeLists.txt                   |  25 ++++
 3 files changed, 221 insertions(+), 1 deletion(-)
 create mode 100644 oi/type_graph/ClangTypeParserTest.cpp

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 69a0807..589245a 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -12,7 +12,7 @@ workflows:
           name: test-gcc
           requires:
             - build-gcc
-          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
+          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|ClangTypeParserTest.*"
       - coverage:
           name: coverage
           requires:
@@ -104,6 +104,7 @@ jobs:
           paths:
             - build/*
             - extern/*
+            - include/*
             - types/*
 
   test:
diff --git a/oi/type_graph/ClangTypeParserTest.cpp b/oi/type_graph/ClangTypeParserTest.cpp
new file mode 100644
index 0000000..4671773
--- /dev/null
+++ b/oi/type_graph/ClangTypeParserTest.cpp
@@ -0,0 +1,194 @@
+#include <clang/AST/Mangle.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Lookup.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/CompilationDatabase.h>
+#include <clang/Tooling/Tooling.h>
+#include <gtest/gtest.h>
+
+#include <range/v3/core.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <string>
+#include <string_view>
+
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+
+using namespace oi::detail;
+using namespace oi::detail::type_graph;
+
+class ClangTypeParserTest : public ::testing::Test {
+ public:
+  std::string run(std::string_view function, ClangTypeParserOptions opt);
+  void test(std::string_view function,
+            std::string_view expected,
+            ClangTypeParserOptions opts = {
+                .chaseRawPointers = true,
+                .readEnumValues = true,
+            });
+};
+
+// This stuff is a total mess. Set up factories as ClangTooling expects them.
+class ConsumerContext;
+class CreateTypeGraphConsumer;
+
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class ConsumerContext {
+ public:
+  ConsumerContext(
+      const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos_)
+      : containerInfos{containerInfos_} {
+  }
+  std::string_view fullyQualifiedName;
+  ClangTypeParserOptions opts;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+  TypeGraph typeGraph;
+  Type* result = nullptr;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    const clang::Type* type = nullptr;
+    for (const clang::Type* ty : Context.getTypes()) {
+      std::string fqnWithoutTemplateParams;
+      switch (ty->getTypeClass()) {
+        case clang::Type::Record:
+          fqnWithoutTemplateParams = llvm::cast<const clang::RecordType>(ty)
+                                         ->getDecl()
+                                         ->getQualifiedNameAsString();
+          break;
+        default:
+          continue;
+      }
+      if (fqnWithoutTemplateParams != ctx.fullyQualifiedName)
+        continue;
+
+      type = ty;
+      break;
+    }
+    EXPECT_NE(type, nullptr);
+
+    ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, ctx.opts};
+    ctx.result = &parser.parse(Context, *ctx.sema, *type);
+  }
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
+}
+
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    [[maybe_unused]] clang::CompilerInstance& CI,
+    [[maybe_unused]] clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+std::string ClangTypeParserTest::run(std::string_view type,
+                                     ClangTypeParserOptions opts) {
+  std::string err{"failed to load compilation database"};
+  auto db =
+      clang::tooling::CompilationDatabase::loadFromDirectory(BUILD_DIR, err);
+  if (!db) {
+    throw std::runtime_error("failed to load compilation database");
+  }
+
+  std::vector<std::unique_ptr<ContainerInfo>> cis;
+  ConsumerContext ctx{cis};
+  ctx.fullyQualifiedName = type;
+  ctx.opts = std::move(opts);
+  CreateTypeGraphActionFactory factory{ctx};
+
+  std::vector<std::string> sourcePaths{TARGET_CPP_PATH};
+  clang::tooling::ClangTool tool{*db, sourcePaths};
+  if (auto retCode = tool.run(&factory); retCode != 0) {
+    throw std::runtime_error("clang type parsing failed");
+  }
+
+  std::stringstream out;
+  NodeTracker tracker;
+  Printer printer{out, tracker, ctx.typeGraph.size()};
+  printer.print(*ctx.result);
+
+  return std::move(out).str();
+}
+
+void ClangTypeParserTest::test(std::string_view type,
+                               std::string_view expected,
+                               ClangTypeParserOptions opts) {
+  std::string actual = run(type, std::move(opts));
+  expected.remove_prefix(1);  // Remove initial '\n'
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(ClangTypeParserTest, SimpleStruct) {
+  test("ns_simple::SimpleStruct", R"(
+[0] Struct: SimpleStruct [ns_simple::SimpleStruct] (size: 16, align: 8)
+      Member: a (offset: 0)
+        Primitive: int32_t
+      Member: b (offset: 4)
+        Primitive: int8_t
+      Member: c (offset: 8)
+        Primitive: int64_t
+)");
+}
+
+TEST_F(ClangTypeParserTest, MemberAlignment) {
+  test("ns_alignment::MemberAlignment", R"(
+[0] Struct: MemberAlignment [ns_alignment::MemberAlignment] (size: 64, align: 32)
+      Member: c (offset: 0)
+        Primitive: int8_t
+      Member: c32 (offset: 32, align: 32)
+        Primitive: int8_t
+)");
+}
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 287ebcd..ccdb606 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -69,6 +69,31 @@ target_link_libraries(test_type_graph
 include(GoogleTest)
 gtest_discover_tests(test_type_graph)
 
+add_executable(test_clang_type_parser
+  main.cpp
+  ../oi/type_graph/ClangTypeParserTest.cpp
+)
+add_dependencies(test_clang_type_parser integration_test_target)
+target_compile_definitions(test_clang_type_parser PRIVATE
+  TARGET_CPP_PATH="${CMAKE_CURRENT_BINARY_DIR}/integration/integration_test_target.cpp"
+  BUILD_DIR="${CMAKE_BINARY_DIR}"
+)
+target_link_libraries(test_clang_type_parser
+  codegen
+  container_info
+  type_graph
+
+  range-v3
+
+  GTest::gmock_main
+)
+if (FORCE_LLVM_STATIC)
+  target_link_libraries(test_clang_type_parser clangTooling ${llvm_libs})
+else()
+  target_link_libraries(test_clang_type_parser clang-cpp LLVM)
+endif()
+gtest_discover_tests(test_clang_type_parser)
+
 cpp_unittest(
   NAME test_parser
   SRCS test_parser.cpp

From 9289fbe8d8c5be8724ca8c88eec3ceee1086058b Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 23 Feb 2024 14:49:06 +0000
Subject: [PATCH 171/188] tbv2: update std::variant

`std::variant` is the last archetypal container missing in TreeBuilder-v2. The
code for it isn't hugely complicated and relies on pack expansion.

This change introduces a new field to the container specification:
`scoped_extra`. This field allows you to write extra code that will be included
within the TypeHandler in CodeGen. This means it will not have collisions with
other containers, unlike the existing `extra` field. It's used here to write
the recursive `getSizeType` function for `std::variant`.

Tech debt is introduced here by comparing the container name to `std::variant`
in CodeGen to conditionally generate some code. We've worked hard to remove
references to containers in code and move them to `.toml` files. On balance,
this is worth having to include the example of `std::variant`. It should be
moved into a container spec field at some point, the design of which is still
to be determined.

Test plan:
- Activated the OIL `std::variant` tests.
- CI
---
 oi/CodeGen.cpp                    | 63 ++++++++++++++++++-------------
 oi/ContainerInfo.cpp              |  4 ++
 oi/ContainerInfo.h                |  1 +
 test/integration/std_variant.toml |  8 ----
 types/README.md                   |  5 +++
 types/std_variant.toml            | 53 +++++++++++++++++++++++++-
 6 files changed, 98 insertions(+), 36 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 98bedf7..5614174 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -941,29 +941,45 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     containerWithTypes = "OICaptureKeys<" + containerWithTypes + ">";
   }
 
-  code += "template <typename Ctx";
-  types = 0, values = 0;
-  for (const auto& p : templateParams) {
-    if (p.value) {
-      code += ", ";
+  // TODO: This is tech debt. This should be moved to a field in the container
+  // spec/`.toml` called something like `codegen.handler_header` or have an
+  // explicit option for variable template parameters. However I'm landing it
+  // anyway to demonstrate how to handle tagged unions in TreeBuilder-v2.
+  if (c.typeName == "std::variant") {
+    code += R"(
+template <typename Ctx, typename... Types>
+struct TypeHandler<Ctx, std::variant<Types...>> {
+  using container_type = std::variant<Types...>;
+)";
+  } else {
+    code += "template <typename Ctx";
+    types = 0, values = 0;
+    for (const auto& p : templateParams) {
+      if (p.value) {
+        code += ", ";
 
-      // HACK: forward all enums directly. this might turn out to be a problem
-      // if there are enums we should be regenerating/use in the body.
-      if (const auto* e = dynamic_cast<const Enum*>(&p.type())) {
-        code += e->inputName();
+        // HACK: forward all enums directly. this might turn out to be a problem
+        // if there are enums we should be regenerating/use in the body.
+        if (const auto* e = dynamic_cast<const Enum*>(&p.type())) {
+          code += e->inputName();
+        } else {
+          code += p.type().name();
+        }
+
+        code += " N" + std::to_string(values++);
       } else {
-        code += p.type().name();
+        code += ", typename T" + std::to_string(types++);
       }
-
-      code += " N" + std::to_string(values++);
-    } else {
-      code += ", typename T" + std::to_string(types++);
     }
+    code += ">\n";
+    code += "struct TypeHandler<Ctx, ";
+    code += containerWithTypes;
+    code += "> {\n";
+    code += "  using container_type = ";
+    code += containerWithTypes;
+    code += ";\n";
   }
-  code += ">\n";
-  code += "struct TypeHandler<Ctx, ";
-  code += containerWithTypes;
-  code += "> {\n";
+
   code += "  using DB = typename Ctx::DataBuffer;\n";
 
   if (c.captureKeys) {
@@ -972,10 +988,6 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     code += "  static constexpr bool captureKeys = false;\n";
   }
 
-  code += "  using container_type = ";
-  code += containerWithTypes;
-  code += ";\n";
-
   code += "  using type = ";
   if (processors.empty()) {
     code += "types::st::Unit<DB>";
@@ -991,11 +1003,11 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   }
   code += ";\n";
 
+  code += c.codegen.scopedExtra;
+
   code += "  static types::st::Unit<DB> getSizeType(\n";
   code += "      Ctx& ctx,\n";
-  code += "      const ";
-  code += containerWithTypes;
-  code += "& container,\n";
+  code += "      const container_type& container,\n";
   code += "      typename TypeHandler<Ctx, ";
   code += containerWithTypes;
   code += ">::type returnArg) {\n";
@@ -1029,7 +1041,6 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     code += "},\n";
   }
   code += "  };\n";
-
   code += "};\n\n";
 }
 
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 980ec61..a0d095e 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -298,6 +298,10 @@ ContainerInfo::ContainerInfo(const fs::path& path) {
           codegenToml["extra"].value<std::string>()) {
     codegen.extra = std::move(*str);
   }
+  if (std::optional<std::string> str =
+          codegenToml["scoped_extra"].value<std::string>()) {
+    codegen.scopedExtra = std::move(*str);
+  }
 
   if (toml::array* arr = codegenToml["processor"].as_array()) {
     codegen.processors.reserve(arr->size());
diff --git a/oi/ContainerInfo.h b/oi/ContainerInfo.h
index a43404c..faf7ed5 100644
--- a/oi/ContainerInfo.h
+++ b/oi/ContainerInfo.h
@@ -38,6 +38,7 @@ struct ContainerInfo {
     std::string func;
     std::string traversalFunc = "";
     std::string extra = "";
+    std::string scopedExtra = "";
     std::vector<Processor> processors{};
   };
 
diff --git a/test/integration/std_variant.toml b/test/integration/std_variant.toml
index 38ad343..518e3e5 100644
--- a/test/integration/std_variant.toml
+++ b/test/integration/std_variant.toml
@@ -9,7 +9,6 @@ definitions = '''
 
 [cases]
   [cases.char_int64_1]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<char, int64_t>&"]
     setup = "return 'a';"
     expect_json = '''[{
@@ -23,7 +22,6 @@ definitions = '''
         {"typeName":"int8_t", "staticSize":1, "exclusiveSize":1, "dynamicSize":0}
       ]}]'''
   [cases.char_int64_2]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<char, int64_t>&"]
     setup = "return 1234;"
     expect_json = '''[{
@@ -38,7 +36,6 @@ definitions = '''
       ]}]'''
 
   [cases.vector_int_1]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<std::vector<int>, int>&"]
     setup = "return std::vector<int>{1,2,3};"
     expect_json = '''[{
@@ -60,7 +57,6 @@ definitions = '''
         }
       ]}]'''
   [cases.vector_int_2]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<std::vector<int>, int>&"]
     setup = "return 123;"
     expect_json = '''[{
@@ -78,7 +74,6 @@ definitions = '''
       ]}]'''
 
   [cases.optional]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     # This test case ensures that the alignment of std::variant is set
     # correctly, as otherwise the size of the std::optional would be wrong
     param_types = ["const std::optional<std::variant<char, int64_t>>&"]
@@ -105,7 +100,6 @@ definitions = '''
       ]}]'''
 
   [cases.empty]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     # https://en.cppreference.com/w/cpp/utility/variant/valueless_by_exception
     param_types = ["const std::variant<int, Thrower>&"]
     setup = '''
@@ -133,7 +127,6 @@ definitions = '''
   # 0xff can be a valid index if there are at least 256 parameters, and that
   # the invalid index value is raised to 0xffff.
   [cases.256_params_256]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<Thrower,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,char>&"]
     setup = "return 'a';"
     expect_json = '''[{
@@ -147,7 +140,6 @@ definitions = '''
         {"typeName":"int8_t", "staticSize":1, "exclusiveSize":1, "dynamicSize":0}
       ]}]'''
   [cases.256_params_empty]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<Thrower,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,char>&"]
     setup = '''
       std::variant<Thrower,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,char> var{'a'};
diff --git a/types/README.md b/types/README.md
index e2b70fd..30acf0b 100644
--- a/types/README.md
+++ b/types/README.md
@@ -55,6 +55,11 @@ This document describes the format of the container definition files contained i
   `static types::st::Unit<DB> getSizeType(const T& container, ST returnArg)`
   where `ST` defines the combined static type of each supplied processor.
 
+- `scoped_extra`
+
+  Extra C++ code to be included in the generated `TypeHandler` for this
+  container.
+
 - `extra`
 
   Any extra C++ code to be included directly. This code is not automatically
diff --git a/types/std_variant.toml b/types/std_variant.toml
index c7b9dcc..fc7559d 100644
--- a/types/std_variant.toml
+++ b/types/std_variant.toml
@@ -36,5 +36,54 @@ void getSizeType(const %1%<Types...> &container, size_t& returnArg)
 }
 """
 
-# TODO: Add tbv2 definitions. The removed intermediate handler is a good
-# template for this, find it in the git logs.
+scoped_extra = """
+template <size_t I = 0>
+static types::st::Unit<DB>
+getSizeTypeRecursive(
+    Ctx& ctx,
+    const typename container_type& container,
+    typename TypeHandler<Ctx, typename container_type>::type returnArg
+) {
+  if constexpr (I < sizeof...(Types)) {
+    if (I == container.index()) {
+      return returnArg.template delegate<I>([&ctx, &container](auto ret) {
+        return OIInternal::getSizeType<Ctx>(ctx, std::get<I>(container), ret);
+      });
+    } else {
+      return getSizeTypeRecursive<I+1>(ctx, container, returnArg);
+    }
+  } else {
+    return returnArg.template delegate<sizeof...(Types)>(std::identity());
+  }
+}
+"""
+
+handler_header = """
+template <typename Ctx, typename... Types>
+struct TypeHandler<Ctx, std::variant<Types...>>
+"""
+
+traversal_func = """
+return getSizeTypeRecursive(ctx, container, returnArg);
+"""
+
+[[codegen.processor]]
+type = "types::st::Sum<Ctx, typename TypeHandler<DB, Types>::type..., types::st::Unit<DB>>"
+func = """
+static constexpr std::array<inst::Field, sizeof...(Types)> children{
+  make_field<Ctx, Types>("*")...,
+};
+
+auto sum = std::get<ParsedData::Sum>(d.val);
+
+el.container_stats = result::Element::ContainerStats {
+  .capacity = 1,
+  .length = sum.index == sizeof...(Types) ? 0 : 1,
+};
+
+if (el.container_stats->length == 0)
+  return;
+
+el.exclusive_size -= children[sum.index].static_size;
+stack_ins(children[sum.index]);
+"""

From b07ab920de1ce4405ec120996be2322a80c689b1 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 23 Feb 2024 14:49:06 +0000
Subject: [PATCH 172/188] tbv2: update std::variant

`std::variant` is the last archetypal container missing in TreeBuilder-v2. The
code for it isn't hugely complicated and relies on pack expansion.

This change introduces a new field to the container specification:
`scoped_extra`. This field allows you to write extra code that will be included
within the TypeHandler in CodeGen. This means it will not have collisions with
other containers, unlike the existing `extra` field. It's used here to write
the recursive `getSizeType` function for `std::variant`.

Tech debt is introduced here by comparing the container name to `std::variant`
in CodeGen to conditionally generate some code. We've worked hard to remove
references to containers in code and move them to `.toml` files. On balance,
this is worth having to include the example of `std::variant`. It should be
moved into a container spec field at some point, the design of which is still
to be determined.

Test plan:
- Activated the OIL `std::variant` tests.
- CI
---
 oi/CodeGen.cpp                    | 67 ++++++++++++----------
 oi/ContainerInfo.cpp              |  4 ++
 oi/ContainerInfo.h                |  1 +
 test/integration/std_variant.toml | 95 ++++++++++++++++++++++++++++---
 types/README.md                   |  5 ++
 types/std_variant.toml            | 53 ++++++++++++++++-
 6 files changed, 186 insertions(+), 39 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 98bedf7..c6372f2 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -941,29 +941,45 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     containerWithTypes = "OICaptureKeys<" + containerWithTypes + ">";
   }
 
-  code += "template <typename Ctx";
-  types = 0, values = 0;
-  for (const auto& p : templateParams) {
-    if (p.value) {
-      code += ", ";
+  // TODO: This is tech debt. This should be moved to a field in the container
+  // spec/`.toml` called something like `codegen.handler_header` or have an
+  // explicit option for variable template parameters. However I'm landing it
+  // anyway to demonstrate how to handle tagged unions in TreeBuilder-v2.
+  if (c.typeName == "std::variant") {
+    code += R"(
+template <typename Ctx, typename... Types>
+struct TypeHandler<Ctx, std::variant<Types...>> {
+  using container_type = std::variant<Types...>;
+)";
+  } else {
+    code += "template <typename Ctx";
+    types = 0, values = 0;
+    for (const auto& p : templateParams) {
+      if (p.value) {
+        code += ", ";
 
-      // HACK: forward all enums directly. this might turn out to be a problem
-      // if there are enums we should be regenerating/use in the body.
-      if (const auto* e = dynamic_cast<const Enum*>(&p.type())) {
-        code += e->inputName();
+        // HACK: forward all enums directly. this might turn out to be a problem
+        // if there are enums we should be regenerating/use in the body.
+        if (const auto* e = dynamic_cast<const Enum*>(&p.type())) {
+          code += e->inputName();
+        } else {
+          code += p.type().name();
+        }
+
+        code += " N" + std::to_string(values++);
       } else {
-        code += p.type().name();
+        code += ", typename T" + std::to_string(types++);
       }
-
-      code += " N" + std::to_string(values++);
-    } else {
-      code += ", typename T" + std::to_string(types++);
     }
+    code += ">\n";
+    code += "struct TypeHandler<Ctx, ";
+    code += containerWithTypes;
+    code += "> {\n";
+    code += "  using container_type = ";
+    code += containerWithTypes;
+    code += ";\n";
   }
-  code += ">\n";
-  code += "struct TypeHandler<Ctx, ";
-  code += containerWithTypes;
-  code += "> {\n";
+
   code += "  using DB = typename Ctx::DataBuffer;\n";
 
   if (c.captureKeys) {
@@ -972,10 +988,6 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     code += "  static constexpr bool captureKeys = false;\n";
   }
 
-  code += "  using container_type = ";
-  code += containerWithTypes;
-  code += ";\n";
-
   code += "  using type = ";
   if (processors.empty()) {
     code += "types::st::Unit<DB>";
@@ -991,14 +1003,12 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   }
   code += ";\n";
 
+  code += c.codegen.scopedExtra;
+
   code += "  static types::st::Unit<DB> getSizeType(\n";
   code += "      Ctx& ctx,\n";
-  code += "      const ";
-  code += containerWithTypes;
-  code += "& container,\n";
-  code += "      typename TypeHandler<Ctx, ";
-  code += containerWithTypes;
-  code += ">::type returnArg) {\n";
+  code += "      const container_type& container,\n";
+  code += "      typename TypeHandler<Ctx, container_type>::type returnArg) {\n";
   code += func;  // has rubbish indentation
   code += "  }\n";
 
@@ -1029,7 +1039,6 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     code += "},\n";
   }
   code += "  };\n";
-
   code += "};\n\n";
 }
 
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 980ec61..a0d095e 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -298,6 +298,10 @@ ContainerInfo::ContainerInfo(const fs::path& path) {
           codegenToml["extra"].value<std::string>()) {
     codegen.extra = std::move(*str);
   }
+  if (std::optional<std::string> str =
+          codegenToml["scoped_extra"].value<std::string>()) {
+    codegen.scopedExtra = std::move(*str);
+  }
 
   if (toml::array* arr = codegenToml["processor"].as_array()) {
     codegen.processors.reserve(arr->size());
diff --git a/oi/ContainerInfo.h b/oi/ContainerInfo.h
index a43404c..faf7ed5 100644
--- a/oi/ContainerInfo.h
+++ b/oi/ContainerInfo.h
@@ -38,6 +38,7 @@ struct ContainerInfo {
     std::string func;
     std::string traversalFunc = "";
     std::string extra = "";
+    std::string scopedExtra = "";
     std::vector<Processor> processors{};
   };
 
diff --git a/test/integration/std_variant.toml b/test/integration/std_variant.toml
index 38ad343..c63a35c 100644
--- a/test/integration/std_variant.toml
+++ b/test/integration/std_variant.toml
@@ -9,7 +9,6 @@ definitions = '''
 
 [cases]
   [cases.char_int64_1]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<char, int64_t>&"]
     setup = "return 'a';"
     expect_json = '''[{
@@ -22,8 +21,17 @@ definitions = '''
       "members":[
         {"typeName":"int8_t", "staticSize":1, "exclusiveSize":1, "dynamicSize":0}
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":16,
+      "staticSize":16,
+      "exclusiveSize":15,
+      "length":1,
+      "capacity":1,
+      "members":[
+        {"typeNames":["int8_t"], "size":1, "staticSize":1, "exclusiveSize":1}
+      ]
+    }]'''
   [cases.char_int64_2]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<char, int64_t>&"]
     setup = "return 1234;"
     expect_json = '''[{
@@ -36,9 +44,18 @@ definitions = '''
       "members":[
         {"typeName":"int64_t", "staticSize":8, "exclusiveSize":8, "dynamicSize":0}
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":16,
+      "staticSize":16,
+      "exclusiveSize":8,
+      "length":1,
+      "capacity":1,
+      "members":[
+        {"typeNames":["int64_t"], "size":8, "staticSize":8, "exclusiveSize":8}
+      ]
+    }]'''
 
   [cases.vector_int_1]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<std::vector<int>, int>&"]
     setup = "return std::vector<int>{1,2,3};"
     expect_json = '''[{
@@ -59,8 +76,21 @@ definitions = '''
           "elementStaticSize":4
         }
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":44,
+      "staticSize":32,
+      "exclusiveSize":8,
+      "length":1,
+      "capacity":1,
+      "members":[
+        {"typeNames":["std::vector<int32_t, std::allocator<int32_t>>"], "size":36, "staticSize":24, "exclusiveSize":24, "members":[
+          {"typeNames":["int32_t"], "size":4, "staticSize":4, "exclusiveSize":4},
+          {"typeNames":["int32_t"], "size":4, "staticSize":4, "exclusiveSize":4},
+          {"typeNames":["int32_t"], "size":4, "staticSize":4, "exclusiveSize":4}
+        ]}
+      ]
+    }]'''
   [cases.vector_int_2]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<std::vector<int>, int>&"]
     setup = "return 123;"
     expect_json = '''[{
@@ -76,9 +106,18 @@ definitions = '''
           "dynamicSize":0
         }
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":32,
+      "staticSize":32,
+      "exclusiveSize":28,
+      "length":1,
+      "capacity":1,
+      "members":[
+        {"typeNames":["int32_t"], "size":4, "staticSize":4, "exclusiveSize":4}
+      ]
+    }]'''
 
   [cases.optional]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     # This test case ensures that the alignment of std::variant is set
     # correctly, as otherwise the size of the std::optional would be wrong
     param_types = ["const std::optional<std::variant<char, int64_t>>&"]
@@ -103,9 +142,25 @@ definitions = '''
           ]
         }
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":24,
+      "staticSize":24,
+      "exclusiveSize":8,
+      "length":1,
+      "capacity":1,
+      "members":[{
+        "size":16,
+        "staticSize":16,
+        "exclusiveSize":8,
+        "length":1,
+        "capacity":1,
+        "members":[
+          {"typeNames":["int64_t"], "size":8, "staticSize":8, "exclusiveSize":8}
+        ]
+      }]
+    }]'''
 
   [cases.empty]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     # https://en.cppreference.com/w/cpp/utility/variant/valueless_by_exception
     param_types = ["const std::variant<int, Thrower>&"]
     setup = '''
@@ -127,13 +182,20 @@ definitions = '''
       "elementStaticSize":4,
       "NOT":"members"
       }]'''
+    expect_json_v2 = '''[{
+      "size":8,
+      "staticSize":8,
+      "exclusiveSize":8,
+      "length":0,
+      "capacity":1,
+      "members":[]
+    }]'''
 
   # With less than 256 options, parameter indexes are stored in a uint8_t and
   # the invalid index value is 0xff. These tests check that we regonise that
   # 0xff can be a valid index if there are at least 256 parameters, and that
   # the invalid index value is raised to 0xffff.
   [cases.256_params_256]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<Thrower,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,char>&"]
     setup = "return 'a';"
     expect_json = '''[{
@@ -146,8 +208,17 @@ definitions = '''
       "members":[
         {"typeName":"int8_t", "staticSize":1, "exclusiveSize":1, "dynamicSize":0}
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":8,
+      "staticSize":8,
+      "exclusiveSize":7,
+      "length":1,
+      "capacity":1,
+      "members":[
+        {"typeNames":["int8_t"], "size":1, "staticSize":1, "exclusiveSize":1}
+      ]
+    }]'''
   [cases.256_params_empty]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<Thrower,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,char>&"]
     setup = '''
       std::variant<Thrower,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,char> var{'a'};
@@ -167,3 +238,11 @@ definitions = '''
       "elementStaticSize":4,
       "NOT":"members"
       }]'''
+    expect_json_v2 = '''[{
+      "size":8,
+      "staticSize":8,
+      "exclusiveSize":8,
+      "length":0,
+      "capacity":1,
+      "members":[]
+    }]'''
diff --git a/types/README.md b/types/README.md
index e2b70fd..30acf0b 100644
--- a/types/README.md
+++ b/types/README.md
@@ -55,6 +55,11 @@ This document describes the format of the container definition files contained i
   `static types::st::Unit<DB> getSizeType(const T& container, ST returnArg)`
   where `ST` defines the combined static type of each supplied processor.
 
+- `scoped_extra`
+
+  Extra C++ code to be included in the generated `TypeHandler` for this
+  container.
+
 - `extra`
 
   Any extra C++ code to be included directly. This code is not automatically
diff --git a/types/std_variant.toml b/types/std_variant.toml
index c7b9dcc..f8a7943 100644
--- a/types/std_variant.toml
+++ b/types/std_variant.toml
@@ -36,5 +36,54 @@ void getSizeType(const %1%<Types...> &container, size_t& returnArg)
 }
 """
 
-# TODO: Add tbv2 definitions. The removed intermediate handler is a good
-# template for this, find it in the git logs.
+scoped_extra = """
+template <size_t I = 0>
+static types::st::Unit<DB>
+getSizeTypeRecursive(
+    Ctx& ctx,
+    const container_type& container,
+    typename TypeHandler<Ctx, container_type>::type returnArg
+) {
+  if constexpr (I < sizeof...(Types)) {
+    if (I == container.index()) {
+      return returnArg.template delegate<I>([&ctx, &container](auto ret) {
+        return OIInternal::getSizeType<Ctx>(ctx, std::get<I>(container), ret);
+      });
+    } else {
+      return getSizeTypeRecursive<I+1>(ctx, container, returnArg);
+    }
+  } else {
+    return returnArg.template delegate<sizeof...(Types)>(std::identity());
+  }
+}
+"""
+
+handler_header = """
+template <typename Ctx, typename... Types>
+struct TypeHandler<Ctx, std::variant<Types...>>
+"""
+
+traversal_func = """
+return getSizeTypeRecursive(ctx, container, returnArg);
+"""
+
+[[codegen.processor]]
+type = "types::st::Sum<DB, typename TypeHandler<Ctx, Types>::type..., types::st::Unit<DB>>"
+func = """
+static constexpr std::array<inst::Field, sizeof...(Types)> children{
+  make_field<Ctx, Types>("*")...,
+};
+
+auto sum = std::get<ParsedData::Sum>(d.val);
+
+el.container_stats = result::Element::ContainerStats {
+  .capacity = 1,
+  .length = sum.index == sizeof...(Types) ? 0u : 1u,
+};
+
+if (el.container_stats->length == 0)
+  return;
+
+el.exclusive_size -= children[sum.index].static_size;
+stack_ins(children[sum.index]);
+"""

From 4d4dda231643460eba2417e70fbbea24d48bf2fc Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 23 Feb 2024 14:49:06 +0000
Subject: [PATCH 173/188] tbv2: update std::variant

`std::variant` is the last archetypal container missing in TreeBuilder-v2. The
code for it isn't hugely complicated and relies on pack expansion.

This change introduces a new field to the container specification:
`scoped_extra`. This field allows you to write extra code that will be included
within the TypeHandler in CodeGen. This means it will not have collisions with
other containers, unlike the existing `extra` field. It's used here to write
the recursive `getSizeType` function for `std::variant`.

Tech debt is introduced here by comparing the container name to `std::variant`
in CodeGen to conditionally generate some code. We've worked hard to remove
references to containers in code and move them to `.toml` files. On balance,
this is worth having to include the example of `std::variant`. It should be
moved into a container spec field at some point, the design of which is still
to be determined.

Test plan:
- Activated the OIL `std::variant` tests.
- CI
---
 oi/CodeGen.cpp                    | 68 ++++++++++++----------
 oi/ContainerInfo.cpp              |  4 ++
 oi/ContainerInfo.h                |  1 +
 test/integration/std_variant.toml | 95 ++++++++++++++++++++++++++++---
 types/README.md                   |  5 ++
 types/std_variant.toml            | 53 ++++++++++++++++-
 6 files changed, 187 insertions(+), 39 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 98bedf7..3cd6ec9 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -941,29 +941,45 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     containerWithTypes = "OICaptureKeys<" + containerWithTypes + ">";
   }
 
-  code += "template <typename Ctx";
-  types = 0, values = 0;
-  for (const auto& p : templateParams) {
-    if (p.value) {
-      code += ", ";
+  // TODO: This is tech debt. This should be moved to a field in the container
+  // spec/`.toml` called something like `codegen.handler_header` or have an
+  // explicit option for variable template parameters. However I'm landing it
+  // anyway to demonstrate how to handle tagged unions in TreeBuilder-v2.
+  if (c.typeName == "std::variant") {
+    code += R"(
+template <typename Ctx, typename... Types>
+struct TypeHandler<Ctx, std::variant<Types...>> {
+  using container_type = std::variant<Types...>;
+)";
+  } else {
+    code += "template <typename Ctx";
+    types = 0, values = 0;
+    for (const auto& p : templateParams) {
+      if (p.value) {
+        code += ", ";
 
-      // HACK: forward all enums directly. this might turn out to be a problem
-      // if there are enums we should be regenerating/use in the body.
-      if (const auto* e = dynamic_cast<const Enum*>(&p.type())) {
-        code += e->inputName();
+        // HACK: forward all enums directly. this might turn out to be a problem
+        // if there are enums we should be regenerating/use in the body.
+        if (const auto* e = dynamic_cast<const Enum*>(&p.type())) {
+          code += e->inputName();
+        } else {
+          code += p.type().name();
+        }
+
+        code += " N" + std::to_string(values++);
       } else {
-        code += p.type().name();
+        code += ", typename T" + std::to_string(types++);
       }
-
-      code += " N" + std::to_string(values++);
-    } else {
-      code += ", typename T" + std::to_string(types++);
     }
+    code += ">\n";
+    code += "struct TypeHandler<Ctx, ";
+    code += containerWithTypes;
+    code += "> {\n";
+    code += "  using container_type = ";
+    code += containerWithTypes;
+    code += ";\n";
   }
-  code += ">\n";
-  code += "struct TypeHandler<Ctx, ";
-  code += containerWithTypes;
-  code += "> {\n";
+
   code += "  using DB = typename Ctx::DataBuffer;\n";
 
   if (c.captureKeys) {
@@ -972,10 +988,6 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     code += "  static constexpr bool captureKeys = false;\n";
   }
 
-  code += "  using container_type = ";
-  code += containerWithTypes;
-  code += ";\n";
-
   code += "  using type = ";
   if (processors.empty()) {
     code += "types::st::Unit<DB>";
@@ -991,14 +1003,13 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   }
   code += ";\n";
 
+  code += c.codegen.scopedExtra;
+
   code += "  static types::st::Unit<DB> getSizeType(\n";
   code += "      Ctx& ctx,\n";
-  code += "      const ";
-  code += containerWithTypes;
-  code += "& container,\n";
-  code += "      typename TypeHandler<Ctx, ";
-  code += containerWithTypes;
-  code += ">::type returnArg) {\n";
+  code += "      const container_type& container,\n";
+  code +=
+      "      typename TypeHandler<Ctx, container_type>::type returnArg) {\n";
   code += func;  // has rubbish indentation
   code += "  }\n";
 
@@ -1029,7 +1040,6 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     code += "},\n";
   }
   code += "  };\n";
-
   code += "};\n\n";
 }
 
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 980ec61..a0d095e 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -298,6 +298,10 @@ ContainerInfo::ContainerInfo(const fs::path& path) {
           codegenToml["extra"].value<std::string>()) {
     codegen.extra = std::move(*str);
   }
+  if (std::optional<std::string> str =
+          codegenToml["scoped_extra"].value<std::string>()) {
+    codegen.scopedExtra = std::move(*str);
+  }
 
   if (toml::array* arr = codegenToml["processor"].as_array()) {
     codegen.processors.reserve(arr->size());
diff --git a/oi/ContainerInfo.h b/oi/ContainerInfo.h
index a43404c..faf7ed5 100644
--- a/oi/ContainerInfo.h
+++ b/oi/ContainerInfo.h
@@ -38,6 +38,7 @@ struct ContainerInfo {
     std::string func;
     std::string traversalFunc = "";
     std::string extra = "";
+    std::string scopedExtra = "";
     std::vector<Processor> processors{};
   };
 
diff --git a/test/integration/std_variant.toml b/test/integration/std_variant.toml
index 38ad343..c63a35c 100644
--- a/test/integration/std_variant.toml
+++ b/test/integration/std_variant.toml
@@ -9,7 +9,6 @@ definitions = '''
 
 [cases]
   [cases.char_int64_1]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<char, int64_t>&"]
     setup = "return 'a';"
     expect_json = '''[{
@@ -22,8 +21,17 @@ definitions = '''
       "members":[
         {"typeName":"int8_t", "staticSize":1, "exclusiveSize":1, "dynamicSize":0}
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":16,
+      "staticSize":16,
+      "exclusiveSize":15,
+      "length":1,
+      "capacity":1,
+      "members":[
+        {"typeNames":["int8_t"], "size":1, "staticSize":1, "exclusiveSize":1}
+      ]
+    }]'''
   [cases.char_int64_2]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<char, int64_t>&"]
     setup = "return 1234;"
     expect_json = '''[{
@@ -36,9 +44,18 @@ definitions = '''
       "members":[
         {"typeName":"int64_t", "staticSize":8, "exclusiveSize":8, "dynamicSize":0}
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":16,
+      "staticSize":16,
+      "exclusiveSize":8,
+      "length":1,
+      "capacity":1,
+      "members":[
+        {"typeNames":["int64_t"], "size":8, "staticSize":8, "exclusiveSize":8}
+      ]
+    }]'''
 
   [cases.vector_int_1]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<std::vector<int>, int>&"]
     setup = "return std::vector<int>{1,2,3};"
     expect_json = '''[{
@@ -59,8 +76,21 @@ definitions = '''
           "elementStaticSize":4
         }
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":44,
+      "staticSize":32,
+      "exclusiveSize":8,
+      "length":1,
+      "capacity":1,
+      "members":[
+        {"typeNames":["std::vector<int32_t, std::allocator<int32_t>>"], "size":36, "staticSize":24, "exclusiveSize":24, "members":[
+          {"typeNames":["int32_t"], "size":4, "staticSize":4, "exclusiveSize":4},
+          {"typeNames":["int32_t"], "size":4, "staticSize":4, "exclusiveSize":4},
+          {"typeNames":["int32_t"], "size":4, "staticSize":4, "exclusiveSize":4}
+        ]}
+      ]
+    }]'''
   [cases.vector_int_2]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<std::vector<int>, int>&"]
     setup = "return 123;"
     expect_json = '''[{
@@ -76,9 +106,18 @@ definitions = '''
           "dynamicSize":0
         }
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":32,
+      "staticSize":32,
+      "exclusiveSize":28,
+      "length":1,
+      "capacity":1,
+      "members":[
+        {"typeNames":["int32_t"], "size":4, "staticSize":4, "exclusiveSize":4}
+      ]
+    }]'''
 
   [cases.optional]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     # This test case ensures that the alignment of std::variant is set
     # correctly, as otherwise the size of the std::optional would be wrong
     param_types = ["const std::optional<std::variant<char, int64_t>>&"]
@@ -103,9 +142,25 @@ definitions = '''
           ]
         }
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":24,
+      "staticSize":24,
+      "exclusiveSize":8,
+      "length":1,
+      "capacity":1,
+      "members":[{
+        "size":16,
+        "staticSize":16,
+        "exclusiveSize":8,
+        "length":1,
+        "capacity":1,
+        "members":[
+          {"typeNames":["int64_t"], "size":8, "staticSize":8, "exclusiveSize":8}
+        ]
+      }]
+    }]'''
 
   [cases.empty]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     # https://en.cppreference.com/w/cpp/utility/variant/valueless_by_exception
     param_types = ["const std::variant<int, Thrower>&"]
     setup = '''
@@ -127,13 +182,20 @@ definitions = '''
       "elementStaticSize":4,
       "NOT":"members"
       }]'''
+    expect_json_v2 = '''[{
+      "size":8,
+      "staticSize":8,
+      "exclusiveSize":8,
+      "length":0,
+      "capacity":1,
+      "members":[]
+    }]'''
 
   # With less than 256 options, parameter indexes are stored in a uint8_t and
   # the invalid index value is 0xff. These tests check that we regonise that
   # 0xff can be a valid index if there are at least 256 parameters, and that
   # the invalid index value is raised to 0xffff.
   [cases.256_params_256]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<Thrower,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,char>&"]
     setup = "return 'a';"
     expect_json = '''[{
@@ -146,8 +208,17 @@ definitions = '''
       "members":[
         {"typeName":"int8_t", "staticSize":1, "exclusiveSize":1, "dynamicSize":0}
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":8,
+      "staticSize":8,
+      "exclusiveSize":7,
+      "length":1,
+      "capacity":1,
+      "members":[
+        {"typeNames":["int8_t"], "size":1, "staticSize":1, "exclusiveSize":1}
+      ]
+    }]'''
   [cases.256_params_empty]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<Thrower,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,char>&"]
     setup = '''
       std::variant<Thrower,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,char> var{'a'};
@@ -167,3 +238,11 @@ definitions = '''
       "elementStaticSize":4,
       "NOT":"members"
       }]'''
+    expect_json_v2 = '''[{
+      "size":8,
+      "staticSize":8,
+      "exclusiveSize":8,
+      "length":0,
+      "capacity":1,
+      "members":[]
+    }]'''
diff --git a/types/README.md b/types/README.md
index e2b70fd..30acf0b 100644
--- a/types/README.md
+++ b/types/README.md
@@ -55,6 +55,11 @@ This document describes the format of the container definition files contained i
   `static types::st::Unit<DB> getSizeType(const T& container, ST returnArg)`
   where `ST` defines the combined static type of each supplied processor.
 
+- `scoped_extra`
+
+  Extra C++ code to be included in the generated `TypeHandler` for this
+  container.
+
 - `extra`
 
   Any extra C++ code to be included directly. This code is not automatically
diff --git a/types/std_variant.toml b/types/std_variant.toml
index c7b9dcc..f8a7943 100644
--- a/types/std_variant.toml
+++ b/types/std_variant.toml
@@ -36,5 +36,54 @@ void getSizeType(const %1%<Types...> &container, size_t& returnArg)
 }
 """
 
-# TODO: Add tbv2 definitions. The removed intermediate handler is a good
-# template for this, find it in the git logs.
+scoped_extra = """
+template <size_t I = 0>
+static types::st::Unit<DB>
+getSizeTypeRecursive(
+    Ctx& ctx,
+    const container_type& container,
+    typename TypeHandler<Ctx, container_type>::type returnArg
+) {
+  if constexpr (I < sizeof...(Types)) {
+    if (I == container.index()) {
+      return returnArg.template delegate<I>([&ctx, &container](auto ret) {
+        return OIInternal::getSizeType<Ctx>(ctx, std::get<I>(container), ret);
+      });
+    } else {
+      return getSizeTypeRecursive<I+1>(ctx, container, returnArg);
+    }
+  } else {
+    return returnArg.template delegate<sizeof...(Types)>(std::identity());
+  }
+}
+"""
+
+handler_header = """
+template <typename Ctx, typename... Types>
+struct TypeHandler<Ctx, std::variant<Types...>>
+"""
+
+traversal_func = """
+return getSizeTypeRecursive(ctx, container, returnArg);
+"""
+
+[[codegen.processor]]
+type = "types::st::Sum<DB, typename TypeHandler<Ctx, Types>::type..., types::st::Unit<DB>>"
+func = """
+static constexpr std::array<inst::Field, sizeof...(Types)> children{
+  make_field<Ctx, Types>("*")...,
+};
+
+auto sum = std::get<ParsedData::Sum>(d.val);
+
+el.container_stats = result::Element::ContainerStats {
+  .capacity = 1,
+  .length = sum.index == sizeof...(Types) ? 0u : 1u,
+};
+
+if (el.container_stats->length == 0)
+  return;
+
+el.exclusive_size -= children[sum.index].static_size;
+stack_ins(children[sum.index]);
+"""

From 2e958fc9a40a4efa279a333da39eec3eb2419074 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 23 Feb 2024 15:30:25 +0000
Subject: [PATCH 174/188] tbv2: update std::variant

`std::variant` is the last archetypal container missing in TreeBuilder-v2. The
code for it isn't hugely complicated and relies on pack expansion.

This change introduces a new field to the container specification:
`scoped_extra`. This field allows you to write extra code that will be included
within the TypeHandler in CodeGen. This means it will not have collisions with
other containers, unlike the existing `extra` field. It's used here to write
the recursive `getSizeType` function for `std::variant`.

Tech debt is introduced here by comparing the container name to `std::variant`
in CodeGen to conditionally generate some code. We've worked hard to remove
references to containers in code and move them to `.toml` files. On balance,
this is worth having to include the example of `std::variant`. It should be
moved into a container spec field at some point, the design of which is still
to be determined.

Test plan:
- Activated the OIL `std::variant` tests.
- CI
---
 oi/CodeGen.cpp                    | 68 ++++++++++++----------
 oi/ContainerInfo.cpp              |  4 ++
 oi/ContainerInfo.h                |  1 +
 test/integration/std_variant.toml | 95 ++++++++++++++++++++++++++++---
 types/README.md                   |  5 ++
 types/std_variant.toml            | 53 ++++++++++++++++-
 6 files changed, 187 insertions(+), 39 deletions(-)

diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp
index 98bedf7..3cd6ec9 100644
--- a/oi/CodeGen.cpp
+++ b/oi/CodeGen.cpp
@@ -941,29 +941,45 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     containerWithTypes = "OICaptureKeys<" + containerWithTypes + ">";
   }
 
-  code += "template <typename Ctx";
-  types = 0, values = 0;
-  for (const auto& p : templateParams) {
-    if (p.value) {
-      code += ", ";
+  // TODO: This is tech debt. This should be moved to a field in the container
+  // spec/`.toml` called something like `codegen.handler_header` or have an
+  // explicit option for variable template parameters. However I'm landing it
+  // anyway to demonstrate how to handle tagged unions in TreeBuilder-v2.
+  if (c.typeName == "std::variant") {
+    code += R"(
+template <typename Ctx, typename... Types>
+struct TypeHandler<Ctx, std::variant<Types...>> {
+  using container_type = std::variant<Types...>;
+)";
+  } else {
+    code += "template <typename Ctx";
+    types = 0, values = 0;
+    for (const auto& p : templateParams) {
+      if (p.value) {
+        code += ", ";
 
-      // HACK: forward all enums directly. this might turn out to be a problem
-      // if there are enums we should be regenerating/use in the body.
-      if (const auto* e = dynamic_cast<const Enum*>(&p.type())) {
-        code += e->inputName();
+        // HACK: forward all enums directly. this might turn out to be a problem
+        // if there are enums we should be regenerating/use in the body.
+        if (const auto* e = dynamic_cast<const Enum*>(&p.type())) {
+          code += e->inputName();
+        } else {
+          code += p.type().name();
+        }
+
+        code += " N" + std::to_string(values++);
       } else {
-        code += p.type().name();
+        code += ", typename T" + std::to_string(types++);
       }
-
-      code += " N" + std::to_string(values++);
-    } else {
-      code += ", typename T" + std::to_string(types++);
     }
+    code += ">\n";
+    code += "struct TypeHandler<Ctx, ";
+    code += containerWithTypes;
+    code += "> {\n";
+    code += "  using container_type = ";
+    code += containerWithTypes;
+    code += ";\n";
   }
-  code += ">\n";
-  code += "struct TypeHandler<Ctx, ";
-  code += containerWithTypes;
-  code += "> {\n";
+
   code += "  using DB = typename Ctx::DataBuffer;\n";
 
   if (c.captureKeys) {
@@ -972,10 +988,6 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     code += "  static constexpr bool captureKeys = false;\n";
   }
 
-  code += "  using container_type = ";
-  code += containerWithTypes;
-  code += ";\n";
-
   code += "  using type = ";
   if (processors.empty()) {
     code += "types::st::Unit<DB>";
@@ -991,14 +1003,13 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
   }
   code += ";\n";
 
+  code += c.codegen.scopedExtra;
+
   code += "  static types::st::Unit<DB> getSizeType(\n";
   code += "      Ctx& ctx,\n";
-  code += "      const ";
-  code += containerWithTypes;
-  code += "& container,\n";
-  code += "      typename TypeHandler<Ctx, ";
-  code += containerWithTypes;
-  code += ">::type returnArg) {\n";
+  code += "      const container_type& container,\n";
+  code +=
+      "      typename TypeHandler<Ctx, container_type>::type returnArg) {\n";
   code += func;  // has rubbish indentation
   code += "  }\n";
 
@@ -1029,7 +1040,6 @@ void genContainerTypeHandler(std::unordered_set<const ContainerInfo*>& used,
     code += "},\n";
   }
   code += "  };\n";
-
   code += "};\n\n";
 }
 
diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp
index 980ec61..a0d095e 100644
--- a/oi/ContainerInfo.cpp
+++ b/oi/ContainerInfo.cpp
@@ -298,6 +298,10 @@ ContainerInfo::ContainerInfo(const fs::path& path) {
           codegenToml["extra"].value<std::string>()) {
     codegen.extra = std::move(*str);
   }
+  if (std::optional<std::string> str =
+          codegenToml["scoped_extra"].value<std::string>()) {
+    codegen.scopedExtra = std::move(*str);
+  }
 
   if (toml::array* arr = codegenToml["processor"].as_array()) {
     codegen.processors.reserve(arr->size());
diff --git a/oi/ContainerInfo.h b/oi/ContainerInfo.h
index a43404c..faf7ed5 100644
--- a/oi/ContainerInfo.h
+++ b/oi/ContainerInfo.h
@@ -38,6 +38,7 @@ struct ContainerInfo {
     std::string func;
     std::string traversalFunc = "";
     std::string extra = "";
+    std::string scopedExtra = "";
     std::vector<Processor> processors{};
   };
 
diff --git a/test/integration/std_variant.toml b/test/integration/std_variant.toml
index 38ad343..c63a35c 100644
--- a/test/integration/std_variant.toml
+++ b/test/integration/std_variant.toml
@@ -9,7 +9,6 @@ definitions = '''
 
 [cases]
   [cases.char_int64_1]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<char, int64_t>&"]
     setup = "return 'a';"
     expect_json = '''[{
@@ -22,8 +21,17 @@ definitions = '''
       "members":[
         {"typeName":"int8_t", "staticSize":1, "exclusiveSize":1, "dynamicSize":0}
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":16,
+      "staticSize":16,
+      "exclusiveSize":15,
+      "length":1,
+      "capacity":1,
+      "members":[
+        {"typeNames":["int8_t"], "size":1, "staticSize":1, "exclusiveSize":1}
+      ]
+    }]'''
   [cases.char_int64_2]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<char, int64_t>&"]
     setup = "return 1234;"
     expect_json = '''[{
@@ -36,9 +44,18 @@ definitions = '''
       "members":[
         {"typeName":"int64_t", "staticSize":8, "exclusiveSize":8, "dynamicSize":0}
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":16,
+      "staticSize":16,
+      "exclusiveSize":8,
+      "length":1,
+      "capacity":1,
+      "members":[
+        {"typeNames":["int64_t"], "size":8, "staticSize":8, "exclusiveSize":8}
+      ]
+    }]'''
 
   [cases.vector_int_1]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<std::vector<int>, int>&"]
     setup = "return std::vector<int>{1,2,3};"
     expect_json = '''[{
@@ -59,8 +76,21 @@ definitions = '''
           "elementStaticSize":4
         }
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":44,
+      "staticSize":32,
+      "exclusiveSize":8,
+      "length":1,
+      "capacity":1,
+      "members":[
+        {"typeNames":["std::vector<int32_t, std::allocator<int32_t>>"], "size":36, "staticSize":24, "exclusiveSize":24, "members":[
+          {"typeNames":["int32_t"], "size":4, "staticSize":4, "exclusiveSize":4},
+          {"typeNames":["int32_t"], "size":4, "staticSize":4, "exclusiveSize":4},
+          {"typeNames":["int32_t"], "size":4, "staticSize":4, "exclusiveSize":4}
+        ]}
+      ]
+    }]'''
   [cases.vector_int_2]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<std::vector<int>, int>&"]
     setup = "return 123;"
     expect_json = '''[{
@@ -76,9 +106,18 @@ definitions = '''
           "dynamicSize":0
         }
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":32,
+      "staticSize":32,
+      "exclusiveSize":28,
+      "length":1,
+      "capacity":1,
+      "members":[
+        {"typeNames":["int32_t"], "size":4, "staticSize":4, "exclusiveSize":4}
+      ]
+    }]'''
 
   [cases.optional]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     # This test case ensures that the alignment of std::variant is set
     # correctly, as otherwise the size of the std::optional would be wrong
     param_types = ["const std::optional<std::variant<char, int64_t>>&"]
@@ -103,9 +142,25 @@ definitions = '''
           ]
         }
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":24,
+      "staticSize":24,
+      "exclusiveSize":8,
+      "length":1,
+      "capacity":1,
+      "members":[{
+        "size":16,
+        "staticSize":16,
+        "exclusiveSize":8,
+        "length":1,
+        "capacity":1,
+        "members":[
+          {"typeNames":["int64_t"], "size":8, "staticSize":8, "exclusiveSize":8}
+        ]
+      }]
+    }]'''
 
   [cases.empty]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     # https://en.cppreference.com/w/cpp/utility/variant/valueless_by_exception
     param_types = ["const std::variant<int, Thrower>&"]
     setup = '''
@@ -127,13 +182,20 @@ definitions = '''
       "elementStaticSize":4,
       "NOT":"members"
       }]'''
+    expect_json_v2 = '''[{
+      "size":8,
+      "staticSize":8,
+      "exclusiveSize":8,
+      "length":0,
+      "capacity":1,
+      "members":[]
+    }]'''
 
   # With less than 256 options, parameter indexes are stored in a uint8_t and
   # the invalid index value is 0xff. These tests check that we regonise that
   # 0xff can be a valid index if there are at least 256 parameters, and that
   # the invalid index value is raised to 0xffff.
   [cases.256_params_256]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<Thrower,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,char>&"]
     setup = "return 'a';"
     expect_json = '''[{
@@ -146,8 +208,17 @@ definitions = '''
       "members":[
         {"typeName":"int8_t", "staticSize":1, "exclusiveSize":1, "dynamicSize":0}
       ]}]'''
+    expect_json_v2 = '''[{
+      "size":8,
+      "staticSize":8,
+      "exclusiveSize":7,
+      "length":1,
+      "capacity":1,
+      "members":[
+        {"typeNames":["int8_t"], "size":1, "staticSize":1, "exclusiveSize":1}
+      ]
+    }]'''
   [cases.256_params_empty]
-    oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298
     param_types = ["const std::variant<Thrower,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,char>&"]
     setup = '''
       std::variant<Thrower,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,char> var{'a'};
@@ -167,3 +238,11 @@ definitions = '''
       "elementStaticSize":4,
       "NOT":"members"
       }]'''
+    expect_json_v2 = '''[{
+      "size":8,
+      "staticSize":8,
+      "exclusiveSize":8,
+      "length":0,
+      "capacity":1,
+      "members":[]
+    }]'''
diff --git a/types/README.md b/types/README.md
index e2b70fd..30acf0b 100644
--- a/types/README.md
+++ b/types/README.md
@@ -55,6 +55,11 @@ This document describes the format of the container definition files contained i
   `static types::st::Unit<DB> getSizeType(const T& container, ST returnArg)`
   where `ST` defines the combined static type of each supplied processor.
 
+- `scoped_extra`
+
+  Extra C++ code to be included in the generated `TypeHandler` for this
+  container.
+
 - `extra`
 
   Any extra C++ code to be included directly. This code is not automatically
diff --git a/types/std_variant.toml b/types/std_variant.toml
index c7b9dcc..f8a7943 100644
--- a/types/std_variant.toml
+++ b/types/std_variant.toml
@@ -36,5 +36,54 @@ void getSizeType(const %1%<Types...> &container, size_t& returnArg)
 }
 """
 
-# TODO: Add tbv2 definitions. The removed intermediate handler is a good
-# template for this, find it in the git logs.
+scoped_extra = """
+template <size_t I = 0>
+static types::st::Unit<DB>
+getSizeTypeRecursive(
+    Ctx& ctx,
+    const container_type& container,
+    typename TypeHandler<Ctx, container_type>::type returnArg
+) {
+  if constexpr (I < sizeof...(Types)) {
+    if (I == container.index()) {
+      return returnArg.template delegate<I>([&ctx, &container](auto ret) {
+        return OIInternal::getSizeType<Ctx>(ctx, std::get<I>(container), ret);
+      });
+    } else {
+      return getSizeTypeRecursive<I+1>(ctx, container, returnArg);
+    }
+  } else {
+    return returnArg.template delegate<sizeof...(Types)>(std::identity());
+  }
+}
+"""
+
+handler_header = """
+template <typename Ctx, typename... Types>
+struct TypeHandler<Ctx, std::variant<Types...>>
+"""
+
+traversal_func = """
+return getSizeTypeRecursive(ctx, container, returnArg);
+"""
+
+[[codegen.processor]]
+type = "types::st::Sum<DB, typename TypeHandler<Ctx, Types>::type..., types::st::Unit<DB>>"
+func = """
+static constexpr std::array<inst::Field, sizeof...(Types)> children{
+  make_field<Ctx, Types>("*")...,
+};
+
+auto sum = std::get<ParsedData::Sum>(d.val);
+
+el.container_stats = result::Element::ContainerStats {
+  .capacity = 1,
+  .length = sum.index == sizeof...(Types) ? 0u : 1u,
+};
+
+if (el.container_stats->length == 0)
+  return;
+
+el.exclusive_size -= children[sum.index].static_size;
+stack_ins(children[sum.index]);
+"""

From 41e9f1c32bd9e9d2968f6532420433874d1bb933 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Fri, 23 Feb 2024 16:26:03 +0000
Subject: [PATCH 175/188] tests: add ClangTypeParserTest

Currently there is no testing for ClangTypeParser even though it's used in
production. This is because adding integration tests is very hard: they require
testing the build time behaviour at runtime, or else they'd be build failures
intead of test failures. There's a PR available for integration tests but it's
incomplete.

In contrast ClangTypeParser can be sort of unit tested. This follows the
structure of `test/test_drgn_parser.cpp` with some differences. There is a
tonne of boilerplate for setting up the Clang tool, and this set of testing
operates on type names instead of OID functions. The new tests are also
incredibly slow as they compile the entire `integration_test_target.cpp` (which
is huge) for every test case. I don't think this is avoidable without
compromising the separation of the tests somewhat due to the way Clang tooling
forces the code to be structured.

Currently I can't run these tests locally on a Meta devserver due to some
weirdness with the internal build and the `compile_commands.json` file. They
run in the CI and on any other open source machine though so I'm happy to merge
it - it's still useful. I'm going to close the PR to change the devserver build
given I'll be unable to follow up if it ends up being bad.

Test plan:
- CI
---
 .circleci/config.yml                  |   3 +-
 oi/type_graph/ClangTypeParserTest.cpp | 194 ++++++++++++++++++++++++++
 test/CMakeLists.txt                   |  25 ++++
 3 files changed, 221 insertions(+), 1 deletion(-)
 create mode 100644 oi/type_graph/ClangTypeParserTest.cpp

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 69a0807..589245a 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -12,7 +12,7 @@ workflows:
           name: test-gcc
           requires:
             - build-gcc
-          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
+          exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0|ClangTypeParserTest.*"
       - coverage:
           name: coverage
           requires:
@@ -104,6 +104,7 @@ jobs:
           paths:
             - build/*
             - extern/*
+            - include/*
             - types/*
 
   test:
diff --git a/oi/type_graph/ClangTypeParserTest.cpp b/oi/type_graph/ClangTypeParserTest.cpp
new file mode 100644
index 0000000..4671773
--- /dev/null
+++ b/oi/type_graph/ClangTypeParserTest.cpp
@@ -0,0 +1,194 @@
+#include <clang/AST/Mangle.h>
+#include <clang/AST/QualTypeNames.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Sema/Lookup.h>
+#include <clang/Sema/Sema.h>
+#include <clang/Tooling/CompilationDatabase.h>
+#include <clang/Tooling/Tooling.h>
+#include <gtest/gtest.h>
+
+#include <range/v3/core.hpp>
+#include <range/v3/view/drop.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/for_each.hpp>
+#include <range/v3/view/take.hpp>
+#include <range/v3/view/transform.hpp>
+#include <string>
+#include <string_view>
+
+#include "oi/type_graph/ClangTypeParser.h"
+#include "oi/type_graph/Printer.h"
+
+using namespace oi::detail;
+using namespace oi::detail::type_graph;
+
+class ClangTypeParserTest : public ::testing::Test {
+ public:
+  std::string run(std::string_view function, ClangTypeParserOptions opt);
+  void test(std::string_view function,
+            std::string_view expected,
+            ClangTypeParserOptions opts = {
+                .chaseRawPointers = true,
+                .readEnumValues = true,
+            });
+};
+
+// This stuff is a total mess. Set up factories as ClangTooling expects them.
+class ConsumerContext;
+class CreateTypeGraphConsumer;
+
+class CreateTypeGraphAction : public clang::ASTFrontendAction {
+ public:
+  CreateTypeGraphAction(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  void ExecuteAction() override;
+  std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+      clang::CompilerInstance& CI, clang::StringRef file) override;
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class CreateTypeGraphActionFactory
+    : public clang::tooling::FrontendActionFactory {
+ public:
+  CreateTypeGraphActionFactory(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  std::unique_ptr<clang::FrontendAction> create() override {
+    return std::make_unique<CreateTypeGraphAction>(ctx);
+  }
+
+ private:
+  ConsumerContext& ctx;
+};
+
+class ConsumerContext {
+ public:
+  ConsumerContext(
+      const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos_)
+      : containerInfos{containerInfos_} {
+  }
+  std::string_view fullyQualifiedName;
+  ClangTypeParserOptions opts;
+  const std::vector<std::unique_ptr<ContainerInfo>>& containerInfos;
+  TypeGraph typeGraph;
+  Type* result = nullptr;
+
+ private:
+  clang::Sema* sema = nullptr;
+  friend CreateTypeGraphConsumer;
+  friend CreateTypeGraphAction;
+};
+
+class CreateTypeGraphConsumer : public clang::ASTConsumer {
+ private:
+  ConsumerContext& ctx;
+
+ public:
+  CreateTypeGraphConsumer(ConsumerContext& ctx_) : ctx{ctx_} {
+  }
+
+  void HandleTranslationUnit(clang::ASTContext& Context) override {
+    const clang::Type* type = nullptr;
+    for (const clang::Type* ty : Context.getTypes()) {
+      std::string fqnWithoutTemplateParams;
+      switch (ty->getTypeClass()) {
+        case clang::Type::Record:
+          fqnWithoutTemplateParams = llvm::cast<const clang::RecordType>(ty)
+                                         ->getDecl()
+                                         ->getQualifiedNameAsString();
+          break;
+        default:
+          continue;
+      }
+      if (fqnWithoutTemplateParams != ctx.fullyQualifiedName)
+        continue;
+
+      type = ty;
+      break;
+    }
+    EXPECT_NE(type, nullptr);
+
+    ClangTypeParser parser{ctx.typeGraph, ctx.containerInfos, ctx.opts};
+    ctx.result = &parser.parse(Context, *ctx.sema, *type);
+  }
+};
+
+void CreateTypeGraphAction::ExecuteAction() {
+  clang::CompilerInstance& CI = getCompilerInstance();
+
+  if (!CI.hasSema())
+    CI.createSema(clang::TU_Complete, nullptr);
+  ctx.sema = &CI.getSema();
+
+  clang::ASTFrontendAction::ExecuteAction();
+}
+
+std::unique_ptr<clang::ASTConsumer> CreateTypeGraphAction::CreateASTConsumer(
+    [[maybe_unused]] clang::CompilerInstance& CI,
+    [[maybe_unused]] clang::StringRef file) {
+  return std::make_unique<CreateTypeGraphConsumer>(ctx);
+}
+
+std::string ClangTypeParserTest::run(std::string_view type,
+                                     ClangTypeParserOptions opts) {
+  std::string err{"failed to load compilation database"};
+  auto db =
+      clang::tooling::CompilationDatabase::loadFromDirectory(BUILD_DIR, err);
+  if (!db) {
+    throw std::runtime_error("failed to load compilation database");
+  }
+
+  std::vector<std::unique_ptr<ContainerInfo>> cis;
+  ConsumerContext ctx{cis};
+  ctx.fullyQualifiedName = type;
+  ctx.opts = std::move(opts);
+  CreateTypeGraphActionFactory factory{ctx};
+
+  std::vector<std::string> sourcePaths{TARGET_CPP_PATH};
+  clang::tooling::ClangTool tool{*db, sourcePaths};
+  if (auto retCode = tool.run(&factory); retCode != 0) {
+    throw std::runtime_error("clang type parsing failed");
+  }
+
+  std::stringstream out;
+  NodeTracker tracker;
+  Printer printer{out, tracker, ctx.typeGraph.size()};
+  printer.print(*ctx.result);
+
+  return std::move(out).str();
+}
+
+void ClangTypeParserTest::test(std::string_view type,
+                               std::string_view expected,
+                               ClangTypeParserOptions opts) {
+  std::string actual = run(type, std::move(opts));
+  expected.remove_prefix(1);  // Remove initial '\n'
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(ClangTypeParserTest, SimpleStruct) {
+  test("ns_simple::SimpleStruct", R"(
+[0] Struct: SimpleStruct [ns_simple::SimpleStruct] (size: 16, align: 8)
+      Member: a (offset: 0)
+        Primitive: int32_t
+      Member: b (offset: 4)
+        Primitive: int8_t
+      Member: c (offset: 8)
+        Primitive: int64_t
+)");
+}
+
+TEST_F(ClangTypeParserTest, MemberAlignment) {
+  test("ns_alignment::MemberAlignment", R"(
+[0] Struct: MemberAlignment [ns_alignment::MemberAlignment] (size: 64, align: 32)
+      Member: c (offset: 0)
+        Primitive: int8_t
+      Member: c32 (offset: 32, align: 32)
+        Primitive: int8_t
+)");
+}
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 287ebcd..ccdb606 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -69,6 +69,31 @@ target_link_libraries(test_type_graph
 include(GoogleTest)
 gtest_discover_tests(test_type_graph)
 
+add_executable(test_clang_type_parser
+  main.cpp
+  ../oi/type_graph/ClangTypeParserTest.cpp
+)
+add_dependencies(test_clang_type_parser integration_test_target)
+target_compile_definitions(test_clang_type_parser PRIVATE
+  TARGET_CPP_PATH="${CMAKE_CURRENT_BINARY_DIR}/integration/integration_test_target.cpp"
+  BUILD_DIR="${CMAKE_BINARY_DIR}"
+)
+target_link_libraries(test_clang_type_parser
+  codegen
+  container_info
+  type_graph
+
+  range-v3
+
+  GTest::gmock_main
+)
+if (FORCE_LLVM_STATIC)
+  target_link_libraries(test_clang_type_parser clangTooling ${llvm_libs})
+else()
+  target_link_libraries(test_clang_type_parser clang-cpp LLVM)
+endif()
+gtest_discover_tests(test_clang_type_parser)
+
 cpp_unittest(
   NAME test_parser
   SRCS test_parser.cpp

From 1419eb6879cf539c26e4c74570daf065ad441903 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Mon, 12 Aug 2024 14:03:15 +0100
Subject: [PATCH 176/188] build: remove dependency on modified folly repository

Currently we pull a modified folly repository to be able to use it header only.
The only difference in this repository is adding the `folly-config.h` file.
Pull this patch into this repo instead and use upstream folly.

Test plan:
- CI build. I have local build issues at the moment.
---
 CMakeLists.txt             |   5 +-
 extern/shim-folly-config.h | 454 +++++++++++++++++++++++++++++++++++++
 flake.nix                  |   5 +-
 3 files changed, 461 insertions(+), 3 deletions(-)
 create mode 100644 extern/shim-folly-config.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 183ba1f..def4bf8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -89,9 +89,10 @@ include_directories(SYSTEM "${rocksdb_SOURCE_DIR}/include")
 ### use folly as a header only library. some features won't be supported.
 FetchContent_Declare(
   folly
-  GIT_REPOSITORY https://github.com/JakeHillion/folly.git
-  GIT_TAG        8db54418e3ccdd97619ac8b69bb3702f82bb0f66
+  GIT_REPOSITORY https://github.com/facebook/folly.git
+  GIT_TAG        40631c52ceea0f376641374eb835e3bc49941c33
   GIT_PROGRESS   TRUE
+  PATCH_COMMAND  ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
 )
 FetchContent_Populate(folly)
 
diff --git a/extern/shim-folly-config.h b/extern/shim-folly-config.h
new file mode 100644
index 0000000..5307d07
--- /dev/null
+++ b/extern/shim-folly-config.h
@@ -0,0 +1,454 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _FOLLY_CONFIG_H
+#define _FOLLY_CONFIG_H 1
+
+#ifdef __APPLE__
+#include <TargetConditionals.h> // @manual
+#endif
+
+#include <time.h>
+
+#if !defined(FOLLY_MOBILE)
+#if defined(__ANDROID__) || \
+    (defined(__APPLE__) &&  \
+     (TARGET_IPHONE_SIMULATOR || TARGET_OS_SIMULATOR || TARGET_OS_IPHONE))
+#define FOLLY_MOBILE 1
+#else
+#define FOLLY_MOBILE 0
+#endif
+#endif // FOLLY_MOBILE
+
+/* define if the Boost library is available */
+#ifndef FOLLY_HAVE_BOOST
+#define FOLLY_HAVE_BOOST /**/
+#endif
+
+/* define if the Boost::Regex library is available */
+#ifndef FOLLY_HAVE_BOOST_REGEX
+#define FOLLY_HAVE_BOOST_REGEX /**/
+#endif
+
+/* define if the Boost::Thread library is available */
+#ifndef FOLLY_HAVE_BOOST_THREAD
+#define FOLLY_HAVE_BOOST_THREAD /**/
+#endif
+
+/* Define to 1 if we support clock_gettime(2) */
+#if !defined(FOLLY_HAVE_CLOCK_GETTIME) && \
+    (!defined(__APPLE__) || defined(CLOCK_REALTIME)) && !defined(_WIN32)
+#define FOLLY_HAVE_CLOCK_GETTIME 1
+#endif
+
+#if !defined(_WIN32)
+#define FOLLY_USE_SYMBOLIZER 1
+#endif
+
+#if !defined(FOLLY_HAVE_ELF) && defined(__linux__) && !defined(__ANDROID__)
+#define FOLLY_HAVE_ELF 1
+#endif
+
+#if !defined(FOLLY_HAVE_DWARF) && defined(__linux__) && !defined(__ANDROID__)
+#define FOLLY_HAVE_DWARF 1
+#endif
+
+#if !defined(FOLLY_HAVE_SWAPCONTEXT) && !defined(_WIN32) && \
+    !defined(__ANDROID__)
+#define FOLLY_HAVE_SWAPCONTEXT 1
+#endif
+
+#if (defined(__linux__) && !defined(__ANDROID__)) || defined(__APPLE__)
+#define FOLLY_HAVE_BACKTRACE 1
+#endif
+
+#if !defined(FOLLY_HAVE_LIBUNWIND) && !defined(_WIN32) && \
+    !(defined(__APPLE__) && defined(__arm__)) && !defined(__ANDROID__)
+#define FOLLY_HAVE_LIBUNWIND 1
+#endif
+
+#if __has_include(<features.h>)
+#include <features.h>
+#endif
+
+/* Define to 1 if you have the `double_conversion' library
+   (-ldouble_conversion). */
+#ifndef FOLLY_HAVE_LIBDOUBLE_CONVERSION
+#define FOLLY_HAVE_LIBDOUBLE_CONVERSION 1
+#endif
+
+/* Define to 1 if you have the `gflags' library (-lgflags). */
+#ifndef FOLLY_HAVE_LIBGFLAGS
+#define FOLLY_HAVE_LIBGFLAGS 1
+#endif
+
+/* Define to 1 if you have the `glog' library (-lglog). */
+#ifndef FOLLY_HAVE_LIBGLOG
+#define FOLLY_HAVE_LIBGLOG 1
+#endif
+
+/* Define to 1 if you have the `gtest' library (-lgtest). */
+#ifndef FOLLY_HAVE_LIBGTEST
+#define FOLLY_HAVE_LIBGTEST 1
+#endif
+
+/* Define to 1 if you have the `gtest_main' library (-lgtest_main). */
+#ifndef FOLLY_HAVE_LIBGTEST_MAIN
+#define FOLLY_HAVE_LIBGTEST_MAIN 1
+#endif
+
+/* Define to 1 if you have the `jemalloc' library (-ljemalloc). */
+#if !defined(FOLLY_HAVE_LIBJEMALLOC) && !defined(_WIN32)
+#define FOLLY_HAVE_LIBJEMALLOC 1
+#endif
+
+/* Define to 1 if you have the `tcmalloc' library (-ltcmalloc). */
+#if !defined(FOLLY_HAVE_LIBTCMALLOC) && !defined(_WIN32)
+#define FOLLY_HAVE_LIBTCMALLOC 1
+#endif
+
+/* Define to 1 if you have the `lz4' library (-llz4). */
+#if !defined(FOLLY_HAVE_LIBLZ4) && !FOLLY_MOBILE && !defined(__APPLE__)
+#define FOLLY_HAVE_LIBLZ4 1
+#endif
+
+/* Define to 1 if you have the `lzma' library (-llzma). */
+#if !defined(FOLLY_HAVE_LIBLZMA) && !FOLLY_MOBILE && !defined(__APPLE__)
+#define FOLLY_HAVE_LIBLZMA 1
+#endif
+
+/* Define to 1 if you have the `snappy' library (-lsnappy). */
+#if !defined(FOLLY_HAVE_LIBSNAPPY) && !FOLLY_MOBILE && !defined(__APPLE__)
+#define FOLLY_HAVE_LIBSNAPPY 1
+#endif
+
+/* Define to 1 if you have the `z' library (-lz). */
+#ifndef FOLLY_HAVE_LIBZ
+#define FOLLY_HAVE_LIBZ 1
+#endif
+
+/* Define to 1 if you have the `zstd' library (-lzstd). */
+#ifndef FOLLY_HAVE_LIBZSTD
+#define FOLLY_HAVE_LIBZSTD 1
+#endif
+
+/* Define to 1 if you have the `bz2' library (-lbz2). */
+#if !defined(FOLLY_HAVE_LIBBZ2) && !FOLLY_MOBILE && !defined(__APPLE__)
+#define FOLLY_HAVE_LIBBZ2 1
+#endif
+
+/* Defined to 1 if you have linux vdso */
+#if !defined(FOLLY_HAVE_LINUX_VDSO) && !defined(_MSC_VER) && !FOLLY_MOBILE && \
+    !defined(__APPLE__) && !defined(_WIN32)
+#define FOLLY_HAVE_LINUX_VDSO 1
+#endif
+
+/* Define to 1 if you have the `malloc_size' function. */
+/* #undef HAVE_MALLOC_SIZE */
+
+/* Define to 1 if you have the `malloc_usable_size' function. */
+#if !defined(FOLLY_HAVE_MALLOC_USABLE_SIZE) && !defined(__APPLE__) && \
+    !defined(_WIN32) && !(defined(ANDROID) && __ANDROID_API__ < 17)
+#define FOLLY_HAVE_MALLOC_USABLE_SIZE 1
+#endif
+
+// Clang doesn't support ifuncs. This also allows ifunc support to be explicitly
+// passed in as a compile flag.
+#ifndef FOLLY_HAVE_IFUNC
+#if defined(__clang__) || defined(__APPLE__) && !defined(_WIN32)
+#define FOLLY_HAVE_IFUNC 0
+#else
+#define FOLLY_HAVE_IFUNC 1
+#endif
+#endif
+
+/* Define to 1 if the system has the type `__int128'. */
+#if !defined(FOLLY_HAVE_INT128_T) && __SIZEOF_INT128__ >= 16 && !defined(_WIN32)
+#define FOLLY_HAVE_INT128_T 1
+#endif
+
+/* Define to 1 if the system needs the standard integer traits defined for the
+ * type `__int128'. */
+#if !defined(FOLLY_SUPPLY_MISSING_INT128_TRAITS) && FOLLY_HAVE_INT128_T && \
+    !defined(__APPLE__) && defined(__STRICT_ANSI__) &&                     \
+    (defined(__llvm__) && !defined(_LIBCPP_VERSION))
+#define FOLLY_SUPPLY_MISSING_INT128_TRAITS 1
+#endif
+
+/* Define if g++ supports C++0x features. */
+#ifndef FOLLY_HAVE_STDCXX_0X
+#define FOLLY_HAVE_STDCXX_0X /**/
+#endif
+
+// Define to 1 if you have the `preadv' and `pwritev' functions, respectively
+#if !defined(FOLLY_HAVE_PREADV) && !defined(FOLLY_HAVE_PWRITEV)
+#if defined(__GLIBC_PREREQ) && !defined(__APPLE__)
+#if __GLIBC_PREREQ(2, 10)
+#define FOLLY_HAVE_PREADV 1
+#define FOLLY_HAVE_PWRITEV 1
+#endif
+#endif
+#endif
+
+#ifndef FOLLY_HAVE_PREADV
+#define FOLLY_HAVE_PREADV 0
+#endif
+
+#ifndef FOLLY_HAVE_PWRITEV
+#define FOLLY_HAVE_PWRITEV 0
+#endif
+
+/* Define to 1 if your architecture can handle unaligned loads and stores. */
+#if !defined(FOLLY_HAVE_UNALIGNED_ACCESS) && !defined(__ANDROID__) && \
+    !defined(__arm__)
+#define FOLLY_HAVE_UNALIGNED_ACCESS 1
+#endif
+
+/* Define to 1 if the linker supports weak symbols. */
+#if !defined(FOLLY_HAVE_WEAK_SYMBOLS) && !defined(__APPLE__) && !defined(_WIN32)
+#define FOLLY_HAVE_WEAK_SYMBOLS 1
+#endif
+
+#if !defined(FOLLY_HAVE_WCHAR_SUPPORT) && !defined(__ANDROID__)
+#define FOLLY_HAVE_WCHAR_SUPPORT 1
+#endif
+
+/* Define to 1 if has <ext/random>. */
+#if !defined(FOLLY_HAVE_EXTRANDOM_SFMT19937) && defined(__has_include) && \
+    !defined(__ANDROID__)
+#if __has_include(<ext/random>)
+#define FOLLY_HAVE_EXTRANDOM_SFMT19937 1
+#endif
+#endif
+
+/* Define to 1 if the compiler has VLA (variable-length array) support,
+   otherwise define to 0 */
+#if !defined(FOLLY_HAVE_VLA) && !defined(_WIN32)
+#define FOLLY_HAVE_VLA 1
+#endif
+
+/* Define to 1 if the system has the type `_Bool'. */
+/* #undef HAVE__BOOL */
+
+/* Define to the sub-directory in which libtool stores uninstalled libraries.
+ */
+#ifndef FOLLY_LT_OBJDIR
+#define FOLLY_LT_OBJDIR ".libs/"
+#endif
+
+/* Name of package */
+#ifndef FOLLY_PACKAGE
+#define FOLLY_PACKAGE "folly"
+#endif
+
+/* Define to the address where bug reports for this package should be sent. */
+#ifndef FOLLY_PACKAGE_BUGREPORT
+#define FOLLY_PACKAGE_BUGREPORT "folly@fb.com"
+#endif
+
+/* Define to the full name of this package. */
+#ifndef FOLLY_PACKAGE_NAME
+#define FOLLY_PACKAGE_NAME "folly"
+#endif
+
+/* Define to the full name and version of this package. */
+#ifndef FOLLY_PACKAGE_STRING
+#define FOLLY_PACKAGE_STRING "folly 0.1"
+#endif
+
+/* Define to the one symbol short name of this package. */
+#ifndef FOLLY_PACKAGE_TARNAME
+#define FOLLY_PACKAGE_TARNAME "folly"
+#endif
+
+/* Define to the home page for this package. */
+#ifndef FOLLY_PACKAGE_URL
+#define FOLLY_PACKAGE_URL ""
+#endif
+
+/* Define to the version of this package. */
+#ifndef FOLLY_PACKAGE_VERSION
+#define FOLLY_PACKAGE_VERSION "0.1"
+#endif
+
+/* Define to 1 if you have the ANSI C header files. */
+#ifndef FOLLY_STDC_HEADERS
+#define FOLLY_STDC_HEADERS 1
+#endif
+
+/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */
+#ifndef FOLLY_TIME_WITH_SYS_TIME
+#define FOLLY_TIME_WITH_SYS_TIME 1
+#endif
+
+/* Define to 1 if openssl has the ASN1_TIME_diff() function */
+#ifndef FOLLY_HAVE_OPENSSL_ASN1_TIME_DIFF
+#define FOLLY_HAVE_OPENSSL_ASN1_TIME_DIFF 1
+#endif
+
+/* Define to 1 if this build supports use in shared libraries */
+#ifndef FOLLY_SUPPORT_SHARED_LIBRARY
+#define FOLLY_SUPPORT_SHARED_LIBRARY 1
+#endif
+
+/* Define to 1 if this build should use extern template for Future<Unit> and
+ * SemiFuture<Unit> */
+#ifndef FOLLY_USE_EXTERN_FUTURE_UNIT
+#if !defined(__ANDROID__) && !defined(__APPLE__)
+#define FOLLY_USE_EXTERN_FUTURE_UNIT 1
+#else
+#define FOLLY_USE_EXTERN_FUTURE_UNIT 0
+#endif
+#endif
+
+/* We always use libstdc++ in fbcode */
+#if !defined(FOLLY_USE_LIBSTDCPP) && !defined(FOLLY_USE_LIBCPP) && \
+    !defined(_WIN32) && !defined(__APPLE__)
+#define FOLLY_USE_LIBSTDCPP 1
+#endif
+
+/* Define to 1 if we're using libc++. */
+#if !defined(FOLLY_USE_LIBCPP) && defined(__APPLE__) && \
+    !defined(FOLLY_USE_LIBSTDCPP)
+#define FOLLY_USE_LIBCPP 1
+#endif
+
+#if defined(FOLLY_USE_LIBCPP) && defined(FOLLY_USE_LIBSTDCPP)
+#error "Both libstdc++ and libc++ cannot be used simultaneously"
+#endif
+
+/* Define to empty if `const' does not conform to ANSI C. */
+/* #undef const */
+
+/* Define to `__inline__' or `__inline' if that's what the C compiler
+   calls it, or to nothing if 'inline' is not supported under any name.  */
+#ifndef __cplusplus
+/* #undef inline */
+#endif
+
+/* Define to `unsigned int' if <sys/types.h> does not define. */
+/* #undef size_t */
+
+/* Define to empty if the keyword `volatile' does not work. Warning: valid
+   code using `volatile' can become incorrect without. Disable with care. */
+/* #undef volatile */
+
+/* Define to 1 if the gflags namespace is not "gflags" */
+#ifndef FOLLY_UNUSUAL_GFLAGS_NAMESPACE
+#define FOLLY_UNUSUAL_GFLAGS_NAMESPACE 1
+#endif
+
+/* Define to gflags namespace ("google" or "gflags") */
+#ifndef FOLLY_GFLAGS_NAMESPACE
+#define FOLLY_GFLAGS_NAMESPACE google
+#endif
+
+#if !defined(FOLLY_HAVE_PTHREAD_SPINLOCK_T) && !FOLLY_MOBILE && \
+    !defined(__APPLE__) && !defined(_WIN32)
+#define FOLLY_HAVE_PTHREAD_SPINLOCK_T 1
+#endif
+
+#if !defined(FOLLY_HAVE_PTHREAD_ATFORK) && !defined(__ANDROID__) && \
+    !defined(_WIN32)
+#define FOLLY_HAVE_PTHREAD_ATFORK 1
+#endif
+
+#define FOLLY_DEMANGLE_MAX_SYMBOL_SIZE 1024
+
+#if __GNUC__ && !__clang__
+#define FOLLY_HAVE_SHADOW_LOCAL_WARNINGS 1
+#endif
+
+#ifndef _WIN32
+#define FOLLY_HAVE_PTHREAD 1
+#endif
+
+/* For internal fbsource-based builds, ASAN is always enabled globally for the
+ * entire build or not.  FOLLY_LIBRARY_SANITIZE_ADDRESS is therefore the same as
+ * FOLLY_SANITIZE_ADDRESS.  We never have builds where folly is compiled with
+ * ASAN enabled but other libraries have ASAN disabled, or vice-versa.
+ */
+#if defined(__has_feature)
+#if __has_feature(address_sanitizer) || __SANITIZE_ADDRESS__
+#define FOLLY_LIBRARY_SANITIZE_ADDRESS 1
+#else
+#define FOLLY_LIBRARY_SANITIZE_ADDRESS 0
+#endif
+#else // defined(__has_feature)
+#if __SANITIZE_ADDRESS__
+#define FOLLY_LIBRARY_SANITIZE_ADDRESS 1
+#else
+#define FOLLY_LIBRARY_SANITIZE_ADDRESS 0
+#endif
+#endif // defined(__has_feature)
+
+// We depend on JEMalloc headers in fbcode, so use them (note that when using
+// sanitizers, `#ifdef` gates in the code will not use JEMalloc headers,
+// despite this setting).  This set of defines is a bit complex, but we try to
+// come up with a set of defines that will only match in fbcode, where JEMalloc
+// is available.
+#if !defined(FOLLY_USE_JEMALLOC) && !FOLLY_MOBILE && !defined(__APPLE__) && \
+    !defined(__ANDROID__) && !defined(_WIN32) && !defined(__wasm32__)
+#define FOLLY_USE_JEMALLOC 1
+#endif
+
+// We depend on range-v3 in fbcode. This set of defines is a bit complex, but we
+// try to come up with a condition that matches in fbcode, where range-v3 is
+// both available and desired.
+#if !defined(FOLLY_USE_RANGEV3) && !FOLLY_MOBILE && !defined(__APPLE__) && \
+    !defined(__ANDROID__) && !defined(_WIN32)
+#define FOLLY_USE_RANGEV3 1
+#endif
+
+// pipe2 is available on glibc 2.9 or later
+#ifdef __GLIBC__
+#if (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 9))
+#ifndef FOLLY_HAVE_PIPE2
+#define FOLLY_HAVE_PIPE2 1
+#endif
+#endif
+#endif
+
+// accept4, recvmmsg, sendmmsg
+#if defined(__linux__) && (!defined(__ANDROID__) || (__ANDROID_API__ >= 21))
+#define FOLLY_HAVE_ACCEPT4 1
+#define FOLLY_HAVE_RECVMMSG 1
+#define FOLLY_HAVE_SENDMMSG 1
+#endif
+
+#if defined(__GLIBC_PREREQ)
+#if __GLIBC_PREREQ(2, 25)
+#define FOLLY_HAVE_GETRANDOM 1
+#else
+#define FOLLY_HAVE_GETRANDOM 0
+#endif
+#else
+#define FOLLY_HAVE_GETRANDOM 0
+#endif
+
+#ifndef FOLLY_HAVE_LIBRT
+#if defined(__linux__) && !FOLLY_MOBILE
+#define FOLLY_HAVE_LIBRT 1
+#else
+#define FOLLY_HAVE_LIBRT 0
+#endif
+#endif
+
+// disable coroutines in our fork as we only need headers for containers
+#define FOLLY_CFG_NO_COROUTINES 1
+
+/* once: _FOLLY_CONFIG_H */
+#endif
diff --git a/flake.nix b/flake.nix
index b35d91e..d019785 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,5 +1,5 @@
 {
-  description = "A flake for building Object Introspection.";
+  description = "Object level memory profiler for C++";
 
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
@@ -11,6 +11,9 @@
   };
 
   outputs = { self, nixpkgs, flake-utils, treefmt-nix, ... }@inputs:
+    flake-utils.lib.eachSystem [ "x86_64-linux" ] (system:
+      {}
+    ) //
     flake-utils.lib.eachDefaultSystem (system:
       let
         pkgs = nixpkgs.legacyPackages.${system};

From 007695fe89daa4838d5a9e03f535d3d9c51746f4 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Mon, 12 Aug 2024 14:03:15 +0100
Subject: [PATCH 177/188] build: remove dependency on modified folly repository

Currently we pull a modified folly repository to be able to use it header only.
The only difference in this repository is adding the `folly-config.h` file.
Pull this patch into this repo instead and use upstream folly.

Test plan:
- CI build. I have local build issues at the moment.
---
 CMakeLists.txt             |   5 +-
 extern/shim-folly-config.h | 454 +++++++++++++++++++++++++++++++++++++
 2 files changed, 457 insertions(+), 2 deletions(-)
 create mode 100644 extern/shim-folly-config.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 183ba1f..def4bf8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -89,9 +89,10 @@ include_directories(SYSTEM "${rocksdb_SOURCE_DIR}/include")
 ### use folly as a header only library. some features won't be supported.
 FetchContent_Declare(
   folly
-  GIT_REPOSITORY https://github.com/JakeHillion/folly.git
-  GIT_TAG        8db54418e3ccdd97619ac8b69bb3702f82bb0f66
+  GIT_REPOSITORY https://github.com/facebook/folly.git
+  GIT_TAG        40631c52ceea0f376641374eb835e3bc49941c33
   GIT_PROGRESS   TRUE
+  PATCH_COMMAND  ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
 )
 FetchContent_Populate(folly)
 
diff --git a/extern/shim-folly-config.h b/extern/shim-folly-config.h
new file mode 100644
index 0000000..5307d07
--- /dev/null
+++ b/extern/shim-folly-config.h
@@ -0,0 +1,454 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _FOLLY_CONFIG_H
+#define _FOLLY_CONFIG_H 1
+
+#ifdef __APPLE__
+#include <TargetConditionals.h> // @manual
+#endif
+
+#include <time.h>
+
+#if !defined(FOLLY_MOBILE)
+#if defined(__ANDROID__) || \
+    (defined(__APPLE__) &&  \
+     (TARGET_IPHONE_SIMULATOR || TARGET_OS_SIMULATOR || TARGET_OS_IPHONE))
+#define FOLLY_MOBILE 1
+#else
+#define FOLLY_MOBILE 0
+#endif
+#endif // FOLLY_MOBILE
+
+/* define if the Boost library is available */
+#ifndef FOLLY_HAVE_BOOST
+#define FOLLY_HAVE_BOOST /**/
+#endif
+
+/* define if the Boost::Regex library is available */
+#ifndef FOLLY_HAVE_BOOST_REGEX
+#define FOLLY_HAVE_BOOST_REGEX /**/
+#endif
+
+/* define if the Boost::Thread library is available */
+#ifndef FOLLY_HAVE_BOOST_THREAD
+#define FOLLY_HAVE_BOOST_THREAD /**/
+#endif
+
+/* Define to 1 if we support clock_gettime(2) */
+#if !defined(FOLLY_HAVE_CLOCK_GETTIME) && \
+    (!defined(__APPLE__) || defined(CLOCK_REALTIME)) && !defined(_WIN32)
+#define FOLLY_HAVE_CLOCK_GETTIME 1
+#endif
+
+#if !defined(_WIN32)
+#define FOLLY_USE_SYMBOLIZER 1
+#endif
+
+#if !defined(FOLLY_HAVE_ELF) && defined(__linux__) && !defined(__ANDROID__)
+#define FOLLY_HAVE_ELF 1
+#endif
+
+#if !defined(FOLLY_HAVE_DWARF) && defined(__linux__) && !defined(__ANDROID__)
+#define FOLLY_HAVE_DWARF 1
+#endif
+
+#if !defined(FOLLY_HAVE_SWAPCONTEXT) && !defined(_WIN32) && \
+    !defined(__ANDROID__)
+#define FOLLY_HAVE_SWAPCONTEXT 1
+#endif
+
+#if (defined(__linux__) && !defined(__ANDROID__)) || defined(__APPLE__)
+#define FOLLY_HAVE_BACKTRACE 1
+#endif
+
+#if !defined(FOLLY_HAVE_LIBUNWIND) && !defined(_WIN32) && \
+    !(defined(__APPLE__) && defined(__arm__)) && !defined(__ANDROID__)
+#define FOLLY_HAVE_LIBUNWIND 1
+#endif
+
+#if __has_include(<features.h>)
+#include <features.h>
+#endif
+
+/* Define to 1 if you have the `double_conversion' library
+   (-ldouble_conversion). */
+#ifndef FOLLY_HAVE_LIBDOUBLE_CONVERSION
+#define FOLLY_HAVE_LIBDOUBLE_CONVERSION 1
+#endif
+
+/* Define to 1 if you have the `gflags' library (-lgflags). */
+#ifndef FOLLY_HAVE_LIBGFLAGS
+#define FOLLY_HAVE_LIBGFLAGS 1
+#endif
+
+/* Define to 1 if you have the `glog' library (-lglog). */
+#ifndef FOLLY_HAVE_LIBGLOG
+#define FOLLY_HAVE_LIBGLOG 1
+#endif
+
+/* Define to 1 if you have the `gtest' library (-lgtest). */
+#ifndef FOLLY_HAVE_LIBGTEST
+#define FOLLY_HAVE_LIBGTEST 1
+#endif
+
+/* Define to 1 if you have the `gtest_main' library (-lgtest_main). */
+#ifndef FOLLY_HAVE_LIBGTEST_MAIN
+#define FOLLY_HAVE_LIBGTEST_MAIN 1
+#endif
+
+/* Define to 1 if you have the `jemalloc' library (-ljemalloc). */
+#if !defined(FOLLY_HAVE_LIBJEMALLOC) && !defined(_WIN32)
+#define FOLLY_HAVE_LIBJEMALLOC 1
+#endif
+
+/* Define to 1 if you have the `tcmalloc' library (-ltcmalloc). */
+#if !defined(FOLLY_HAVE_LIBTCMALLOC) && !defined(_WIN32)
+#define FOLLY_HAVE_LIBTCMALLOC 1
+#endif
+
+/* Define to 1 if you have the `lz4' library (-llz4). */
+#if !defined(FOLLY_HAVE_LIBLZ4) && !FOLLY_MOBILE && !defined(__APPLE__)
+#define FOLLY_HAVE_LIBLZ4 1
+#endif
+
+/* Define to 1 if you have the `lzma' library (-llzma). */
+#if !defined(FOLLY_HAVE_LIBLZMA) && !FOLLY_MOBILE && !defined(__APPLE__)
+#define FOLLY_HAVE_LIBLZMA 1
+#endif
+
+/* Define to 1 if you have the `snappy' library (-lsnappy). */
+#if !defined(FOLLY_HAVE_LIBSNAPPY) && !FOLLY_MOBILE && !defined(__APPLE__)
+#define FOLLY_HAVE_LIBSNAPPY 1
+#endif
+
+/* Define to 1 if you have the `z' library (-lz). */
+#ifndef FOLLY_HAVE_LIBZ
+#define FOLLY_HAVE_LIBZ 1
+#endif
+
+/* Define to 1 if you have the `zstd' library (-lzstd). */
+#ifndef FOLLY_HAVE_LIBZSTD
+#define FOLLY_HAVE_LIBZSTD 1
+#endif
+
+/* Define to 1 if you have the `bz2' library (-lbz2). */
+#if !defined(FOLLY_HAVE_LIBBZ2) && !FOLLY_MOBILE && !defined(__APPLE__)
+#define FOLLY_HAVE_LIBBZ2 1
+#endif
+
+/* Defined to 1 if you have linux vdso */
+#if !defined(FOLLY_HAVE_LINUX_VDSO) && !defined(_MSC_VER) && !FOLLY_MOBILE && \
+    !defined(__APPLE__) && !defined(_WIN32)
+#define FOLLY_HAVE_LINUX_VDSO 1
+#endif
+
+/* Define to 1 if you have the `malloc_size' function. */
+/* #undef HAVE_MALLOC_SIZE */
+
+/* Define to 1 if you have the `malloc_usable_size' function. */
+#if !defined(FOLLY_HAVE_MALLOC_USABLE_SIZE) && !defined(__APPLE__) && \
+    !defined(_WIN32) && !(defined(ANDROID) && __ANDROID_API__ < 17)
+#define FOLLY_HAVE_MALLOC_USABLE_SIZE 1
+#endif
+
+// Clang doesn't support ifuncs. This also allows ifunc support to be explicitly
+// passed in as a compile flag.
+#ifndef FOLLY_HAVE_IFUNC
+#if defined(__clang__) || defined(__APPLE__) && !defined(_WIN32)
+#define FOLLY_HAVE_IFUNC 0
+#else
+#define FOLLY_HAVE_IFUNC 1
+#endif
+#endif
+
+/* Define to 1 if the system has the type `__int128'. */
+#if !defined(FOLLY_HAVE_INT128_T) && __SIZEOF_INT128__ >= 16 && !defined(_WIN32)
+#define FOLLY_HAVE_INT128_T 1
+#endif
+
+/* Define to 1 if the system needs the standard integer traits defined for the
+ * type `__int128'. */
+#if !defined(FOLLY_SUPPLY_MISSING_INT128_TRAITS) && FOLLY_HAVE_INT128_T && \
+    !defined(__APPLE__) && defined(__STRICT_ANSI__) &&                     \
+    (defined(__llvm__) && !defined(_LIBCPP_VERSION))
+#define FOLLY_SUPPLY_MISSING_INT128_TRAITS 1
+#endif
+
+/* Define if g++ supports C++0x features. */
+#ifndef FOLLY_HAVE_STDCXX_0X
+#define FOLLY_HAVE_STDCXX_0X /**/
+#endif
+
+// Define to 1 if you have the `preadv' and `pwritev' functions, respectively
+#if !defined(FOLLY_HAVE_PREADV) && !defined(FOLLY_HAVE_PWRITEV)
+#if defined(__GLIBC_PREREQ) && !defined(__APPLE__)
+#if __GLIBC_PREREQ(2, 10)
+#define FOLLY_HAVE_PREADV 1
+#define FOLLY_HAVE_PWRITEV 1
+#endif
+#endif
+#endif
+
+#ifndef FOLLY_HAVE_PREADV
+#define FOLLY_HAVE_PREADV 0
+#endif
+
+#ifndef FOLLY_HAVE_PWRITEV
+#define FOLLY_HAVE_PWRITEV 0
+#endif
+
+/* Define to 1 if your architecture can handle unaligned loads and stores. */
+#if !defined(FOLLY_HAVE_UNALIGNED_ACCESS) && !defined(__ANDROID__) && \
+    !defined(__arm__)
+#define FOLLY_HAVE_UNALIGNED_ACCESS 1
+#endif
+
+/* Define to 1 if the linker supports weak symbols. */
+#if !defined(FOLLY_HAVE_WEAK_SYMBOLS) && !defined(__APPLE__) && !defined(_WIN32)
+#define FOLLY_HAVE_WEAK_SYMBOLS 1
+#endif
+
+#if !defined(FOLLY_HAVE_WCHAR_SUPPORT) && !defined(__ANDROID__)
+#define FOLLY_HAVE_WCHAR_SUPPORT 1
+#endif
+
+/* Define to 1 if has <ext/random>. */
+#if !defined(FOLLY_HAVE_EXTRANDOM_SFMT19937) && defined(__has_include) && \
+    !defined(__ANDROID__)
+#if __has_include(<ext/random>)
+#define FOLLY_HAVE_EXTRANDOM_SFMT19937 1
+#endif
+#endif
+
+/* Define to 1 if the compiler has VLA (variable-length array) support,
+   otherwise define to 0 */
+#if !defined(FOLLY_HAVE_VLA) && !defined(_WIN32)
+#define FOLLY_HAVE_VLA 1
+#endif
+
+/* Define to 1 if the system has the type `_Bool'. */
+/* #undef HAVE__BOOL */
+
+/* Define to the sub-directory in which libtool stores uninstalled libraries.
+ */
+#ifndef FOLLY_LT_OBJDIR
+#define FOLLY_LT_OBJDIR ".libs/"
+#endif
+
+/* Name of package */
+#ifndef FOLLY_PACKAGE
+#define FOLLY_PACKAGE "folly"
+#endif
+
+/* Define to the address where bug reports for this package should be sent. */
+#ifndef FOLLY_PACKAGE_BUGREPORT
+#define FOLLY_PACKAGE_BUGREPORT "folly@fb.com"
+#endif
+
+/* Define to the full name of this package. */
+#ifndef FOLLY_PACKAGE_NAME
+#define FOLLY_PACKAGE_NAME "folly"
+#endif
+
+/* Define to the full name and version of this package. */
+#ifndef FOLLY_PACKAGE_STRING
+#define FOLLY_PACKAGE_STRING "folly 0.1"
+#endif
+
+/* Define to the one symbol short name of this package. */
+#ifndef FOLLY_PACKAGE_TARNAME
+#define FOLLY_PACKAGE_TARNAME "folly"
+#endif
+
+/* Define to the home page for this package. */
+#ifndef FOLLY_PACKAGE_URL
+#define FOLLY_PACKAGE_URL ""
+#endif
+
+/* Define to the version of this package. */
+#ifndef FOLLY_PACKAGE_VERSION
+#define FOLLY_PACKAGE_VERSION "0.1"
+#endif
+
+/* Define to 1 if you have the ANSI C header files. */
+#ifndef FOLLY_STDC_HEADERS
+#define FOLLY_STDC_HEADERS 1
+#endif
+
+/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */
+#ifndef FOLLY_TIME_WITH_SYS_TIME
+#define FOLLY_TIME_WITH_SYS_TIME 1
+#endif
+
+/* Define to 1 if openssl has the ASN1_TIME_diff() function */
+#ifndef FOLLY_HAVE_OPENSSL_ASN1_TIME_DIFF
+#define FOLLY_HAVE_OPENSSL_ASN1_TIME_DIFF 1
+#endif
+
+/* Define to 1 if this build supports use in shared libraries */
+#ifndef FOLLY_SUPPORT_SHARED_LIBRARY
+#define FOLLY_SUPPORT_SHARED_LIBRARY 1
+#endif
+
+/* Define to 1 if this build should use extern template for Future<Unit> and
+ * SemiFuture<Unit> */
+#ifndef FOLLY_USE_EXTERN_FUTURE_UNIT
+#if !defined(__ANDROID__) && !defined(__APPLE__)
+#define FOLLY_USE_EXTERN_FUTURE_UNIT 1
+#else
+#define FOLLY_USE_EXTERN_FUTURE_UNIT 0
+#endif
+#endif
+
+/* We always use libstdc++ in fbcode */
+#if !defined(FOLLY_USE_LIBSTDCPP) && !defined(FOLLY_USE_LIBCPP) && \
+    !defined(_WIN32) && !defined(__APPLE__)
+#define FOLLY_USE_LIBSTDCPP 1
+#endif
+
+/* Define to 1 if we're using libc++. */
+#if !defined(FOLLY_USE_LIBCPP) && defined(__APPLE__) && \
+    !defined(FOLLY_USE_LIBSTDCPP)
+#define FOLLY_USE_LIBCPP 1
+#endif
+
+#if defined(FOLLY_USE_LIBCPP) && defined(FOLLY_USE_LIBSTDCPP)
+#error "Both libstdc++ and libc++ cannot be used simultaneously"
+#endif
+
+/* Define to empty if `const' does not conform to ANSI C. */
+/* #undef const */
+
+/* Define to `__inline__' or `__inline' if that's what the C compiler
+   calls it, or to nothing if 'inline' is not supported under any name.  */
+#ifndef __cplusplus
+/* #undef inline */
+#endif
+
+/* Define to `unsigned int' if <sys/types.h> does not define. */
+/* #undef size_t */
+
+/* Define to empty if the keyword `volatile' does not work. Warning: valid
+   code using `volatile' can become incorrect without. Disable with care. */
+/* #undef volatile */
+
+/* Define to 1 if the gflags namespace is not "gflags" */
+#ifndef FOLLY_UNUSUAL_GFLAGS_NAMESPACE
+#define FOLLY_UNUSUAL_GFLAGS_NAMESPACE 1
+#endif
+
+/* Define to gflags namespace ("google" or "gflags") */
+#ifndef FOLLY_GFLAGS_NAMESPACE
+#define FOLLY_GFLAGS_NAMESPACE google
+#endif
+
+#if !defined(FOLLY_HAVE_PTHREAD_SPINLOCK_T) && !FOLLY_MOBILE && \
+    !defined(__APPLE__) && !defined(_WIN32)
+#define FOLLY_HAVE_PTHREAD_SPINLOCK_T 1
+#endif
+
+#if !defined(FOLLY_HAVE_PTHREAD_ATFORK) && !defined(__ANDROID__) && \
+    !defined(_WIN32)
+#define FOLLY_HAVE_PTHREAD_ATFORK 1
+#endif
+
+#define FOLLY_DEMANGLE_MAX_SYMBOL_SIZE 1024
+
+#if __GNUC__ && !__clang__
+#define FOLLY_HAVE_SHADOW_LOCAL_WARNINGS 1
+#endif
+
+#ifndef _WIN32
+#define FOLLY_HAVE_PTHREAD 1
+#endif
+
+/* For internal fbsource-based builds, ASAN is always enabled globally for the
+ * entire build or not.  FOLLY_LIBRARY_SANITIZE_ADDRESS is therefore the same as
+ * FOLLY_SANITIZE_ADDRESS.  We never have builds where folly is compiled with
+ * ASAN enabled but other libraries have ASAN disabled, or vice-versa.
+ */
+#if defined(__has_feature)
+#if __has_feature(address_sanitizer) || __SANITIZE_ADDRESS__
+#define FOLLY_LIBRARY_SANITIZE_ADDRESS 1
+#else
+#define FOLLY_LIBRARY_SANITIZE_ADDRESS 0
+#endif
+#else // defined(__has_feature)
+#if __SANITIZE_ADDRESS__
+#define FOLLY_LIBRARY_SANITIZE_ADDRESS 1
+#else
+#define FOLLY_LIBRARY_SANITIZE_ADDRESS 0
+#endif
+#endif // defined(__has_feature)
+
+// We depend on JEMalloc headers in fbcode, so use them (note that when using
+// sanitizers, `#ifdef` gates in the code will not use JEMalloc headers,
+// despite this setting).  This set of defines is a bit complex, but we try to
+// come up with a set of defines that will only match in fbcode, where JEMalloc
+// is available.
+#if !defined(FOLLY_USE_JEMALLOC) && !FOLLY_MOBILE && !defined(__APPLE__) && \
+    !defined(__ANDROID__) && !defined(_WIN32) && !defined(__wasm32__)
+#define FOLLY_USE_JEMALLOC 1
+#endif
+
+// We depend on range-v3 in fbcode. This set of defines is a bit complex, but we
+// try to come up with a condition that matches in fbcode, where range-v3 is
+// both available and desired.
+#if !defined(FOLLY_USE_RANGEV3) && !FOLLY_MOBILE && !defined(__APPLE__) && \
+    !defined(__ANDROID__) && !defined(_WIN32)
+#define FOLLY_USE_RANGEV3 1
+#endif
+
+// pipe2 is available on glibc 2.9 or later
+#ifdef __GLIBC__
+#if (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 9))
+#ifndef FOLLY_HAVE_PIPE2
+#define FOLLY_HAVE_PIPE2 1
+#endif
+#endif
+#endif
+
+// accept4, recvmmsg, sendmmsg
+#if defined(__linux__) && (!defined(__ANDROID__) || (__ANDROID_API__ >= 21))
+#define FOLLY_HAVE_ACCEPT4 1
+#define FOLLY_HAVE_RECVMMSG 1
+#define FOLLY_HAVE_SENDMMSG 1
+#endif
+
+#if defined(__GLIBC_PREREQ)
+#if __GLIBC_PREREQ(2, 25)
+#define FOLLY_HAVE_GETRANDOM 1
+#else
+#define FOLLY_HAVE_GETRANDOM 0
+#endif
+#else
+#define FOLLY_HAVE_GETRANDOM 0
+#endif
+
+#ifndef FOLLY_HAVE_LIBRT
+#if defined(__linux__) && !FOLLY_MOBILE
+#define FOLLY_HAVE_LIBRT 1
+#else
+#define FOLLY_HAVE_LIBRT 0
+#endif
+#endif
+
+// disable coroutines in our fork as we only need headers for containers
+#define FOLLY_CFG_NO_COROUTINES 1
+
+/* once: _FOLLY_CONFIG_H */
+#endif

From e4347679d8b89751c71061ab6ea68320e2c311f4 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Mon, 12 Aug 2024 14:12:15 +0100
Subject: [PATCH 178/188] build: remove dependency on modified folly repository

Currently we pull a modified folly repository to be able to use it header only.
The only difference in this repository is adding the `folly-config.h` file.
Pull this patch into this repo instead and use upstream folly.

Bump folly to latest main as well.

Test plan:
- CI build. I have local build issues at the moment.
---
 CMakeLists.txt             |   5 +-
 extern/shim-folly-config.h | 454 +++++++++++++++++++++++++++++++++++++
 2 files changed, 457 insertions(+), 2 deletions(-)
 create mode 100644 extern/shim-folly-config.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 183ba1f..8674d46 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -89,9 +89,10 @@ include_directories(SYSTEM "${rocksdb_SOURCE_DIR}/include")
 ### use folly as a header only library. some features won't be supported.
 FetchContent_Declare(
   folly
-  GIT_REPOSITORY https://github.com/JakeHillion/folly.git
-  GIT_TAG        8db54418e3ccdd97619ac8b69bb3702f82bb0f66
+  GIT_REPOSITORY https://github.com/facebook/folly.git
+  GIT_TAG        c5aa5c46291a27f69acc920894d43605ceb43eba
   GIT_PROGRESS   TRUE
+  PATCH_COMMAND  ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
 )
 FetchContent_Populate(folly)
 
diff --git a/extern/shim-folly-config.h b/extern/shim-folly-config.h
new file mode 100644
index 0000000..5307d07
--- /dev/null
+++ b/extern/shim-folly-config.h
@@ -0,0 +1,454 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _FOLLY_CONFIG_H
+#define _FOLLY_CONFIG_H 1
+
+#ifdef __APPLE__
+#include <TargetConditionals.h> // @manual
+#endif
+
+#include <time.h>
+
+#if !defined(FOLLY_MOBILE)
+#if defined(__ANDROID__) || \
+    (defined(__APPLE__) &&  \
+     (TARGET_IPHONE_SIMULATOR || TARGET_OS_SIMULATOR || TARGET_OS_IPHONE))
+#define FOLLY_MOBILE 1
+#else
+#define FOLLY_MOBILE 0
+#endif
+#endif // FOLLY_MOBILE
+
+/* define if the Boost library is available */
+#ifndef FOLLY_HAVE_BOOST
+#define FOLLY_HAVE_BOOST /**/
+#endif
+
+/* define if the Boost::Regex library is available */
+#ifndef FOLLY_HAVE_BOOST_REGEX
+#define FOLLY_HAVE_BOOST_REGEX /**/
+#endif
+
+/* define if the Boost::Thread library is available */
+#ifndef FOLLY_HAVE_BOOST_THREAD
+#define FOLLY_HAVE_BOOST_THREAD /**/
+#endif
+
+/* Define to 1 if we support clock_gettime(2) */
+#if !defined(FOLLY_HAVE_CLOCK_GETTIME) && \
+    (!defined(__APPLE__) || defined(CLOCK_REALTIME)) && !defined(_WIN32)
+#define FOLLY_HAVE_CLOCK_GETTIME 1
+#endif
+
+#if !defined(_WIN32)
+#define FOLLY_USE_SYMBOLIZER 1
+#endif
+
+#if !defined(FOLLY_HAVE_ELF) && defined(__linux__) && !defined(__ANDROID__)
+#define FOLLY_HAVE_ELF 1
+#endif
+
+#if !defined(FOLLY_HAVE_DWARF) && defined(__linux__) && !defined(__ANDROID__)
+#define FOLLY_HAVE_DWARF 1
+#endif
+
+#if !defined(FOLLY_HAVE_SWAPCONTEXT) && !defined(_WIN32) && \
+    !defined(__ANDROID__)
+#define FOLLY_HAVE_SWAPCONTEXT 1
+#endif
+
+#if (defined(__linux__) && !defined(__ANDROID__)) || defined(__APPLE__)
+#define FOLLY_HAVE_BACKTRACE 1
+#endif
+
+#if !defined(FOLLY_HAVE_LIBUNWIND) && !defined(_WIN32) && \
+    !(defined(__APPLE__) && defined(__arm__)) && !defined(__ANDROID__)
+#define FOLLY_HAVE_LIBUNWIND 1
+#endif
+
+#if __has_include(<features.h>)
+#include <features.h>
+#endif
+
+/* Define to 1 if you have the `double_conversion' library
+   (-ldouble_conversion). */
+#ifndef FOLLY_HAVE_LIBDOUBLE_CONVERSION
+#define FOLLY_HAVE_LIBDOUBLE_CONVERSION 1
+#endif
+
+/* Define to 1 if you have the `gflags' library (-lgflags). */
+#ifndef FOLLY_HAVE_LIBGFLAGS
+#define FOLLY_HAVE_LIBGFLAGS 1
+#endif
+
+/* Define to 1 if you have the `glog' library (-lglog). */
+#ifndef FOLLY_HAVE_LIBGLOG
+#define FOLLY_HAVE_LIBGLOG 1
+#endif
+
+/* Define to 1 if you have the `gtest' library (-lgtest). */
+#ifndef FOLLY_HAVE_LIBGTEST
+#define FOLLY_HAVE_LIBGTEST 1
+#endif
+
+/* Define to 1 if you have the `gtest_main' library (-lgtest_main). */
+#ifndef FOLLY_HAVE_LIBGTEST_MAIN
+#define FOLLY_HAVE_LIBGTEST_MAIN 1
+#endif
+
+/* Define to 1 if you have the `jemalloc' library (-ljemalloc). */
+#if !defined(FOLLY_HAVE_LIBJEMALLOC) && !defined(_WIN32)
+#define FOLLY_HAVE_LIBJEMALLOC 1
+#endif
+
+/* Define to 1 if you have the `tcmalloc' library (-ltcmalloc). */
+#if !defined(FOLLY_HAVE_LIBTCMALLOC) && !defined(_WIN32)
+#define FOLLY_HAVE_LIBTCMALLOC 1
+#endif
+
+/* Define to 1 if you have the `lz4' library (-llz4). */
+#if !defined(FOLLY_HAVE_LIBLZ4) && !FOLLY_MOBILE && !defined(__APPLE__)
+#define FOLLY_HAVE_LIBLZ4 1
+#endif
+
+/* Define to 1 if you have the `lzma' library (-llzma). */
+#if !defined(FOLLY_HAVE_LIBLZMA) && !FOLLY_MOBILE && !defined(__APPLE__)
+#define FOLLY_HAVE_LIBLZMA 1
+#endif
+
+/* Define to 1 if you have the `snappy' library (-lsnappy). */
+#if !defined(FOLLY_HAVE_LIBSNAPPY) && !FOLLY_MOBILE && !defined(__APPLE__)
+#define FOLLY_HAVE_LIBSNAPPY 1
+#endif
+
+/* Define to 1 if you have the `z' library (-lz). */
+#ifndef FOLLY_HAVE_LIBZ
+#define FOLLY_HAVE_LIBZ 1
+#endif
+
+/* Define to 1 if you have the `zstd' library (-lzstd). */
+#ifndef FOLLY_HAVE_LIBZSTD
+#define FOLLY_HAVE_LIBZSTD 1
+#endif
+
+/* Define to 1 if you have the `bz2' library (-lbz2). */
+#if !defined(FOLLY_HAVE_LIBBZ2) && !FOLLY_MOBILE && !defined(__APPLE__)
+#define FOLLY_HAVE_LIBBZ2 1
+#endif
+
+/* Defined to 1 if you have linux vdso */
+#if !defined(FOLLY_HAVE_LINUX_VDSO) && !defined(_MSC_VER) && !FOLLY_MOBILE && \
+    !defined(__APPLE__) && !defined(_WIN32)
+#define FOLLY_HAVE_LINUX_VDSO 1
+#endif
+
+/* Define to 1 if you have the `malloc_size' function. */
+/* #undef HAVE_MALLOC_SIZE */
+
+/* Define to 1 if you have the `malloc_usable_size' function. */
+#if !defined(FOLLY_HAVE_MALLOC_USABLE_SIZE) && !defined(__APPLE__) && \
+    !defined(_WIN32) && !(defined(ANDROID) && __ANDROID_API__ < 17)
+#define FOLLY_HAVE_MALLOC_USABLE_SIZE 1
+#endif
+
+// Clang doesn't support ifuncs. This also allows ifunc support to be explicitly
+// passed in as a compile flag.
+#ifndef FOLLY_HAVE_IFUNC
+#if defined(__clang__) || defined(__APPLE__) && !defined(_WIN32)
+#define FOLLY_HAVE_IFUNC 0
+#else
+#define FOLLY_HAVE_IFUNC 1
+#endif
+#endif
+
+/* Define to 1 if the system has the type `__int128'. */
+#if !defined(FOLLY_HAVE_INT128_T) && __SIZEOF_INT128__ >= 16 && !defined(_WIN32)
+#define FOLLY_HAVE_INT128_T 1
+#endif
+
+/* Define to 1 if the system needs the standard integer traits defined for the
+ * type `__int128'. */
+#if !defined(FOLLY_SUPPLY_MISSING_INT128_TRAITS) && FOLLY_HAVE_INT128_T && \
+    !defined(__APPLE__) && defined(__STRICT_ANSI__) &&                     \
+    (defined(__llvm__) && !defined(_LIBCPP_VERSION))
+#define FOLLY_SUPPLY_MISSING_INT128_TRAITS 1
+#endif
+
+/* Define if g++ supports C++0x features. */
+#ifndef FOLLY_HAVE_STDCXX_0X
+#define FOLLY_HAVE_STDCXX_0X /**/
+#endif
+
+// Define to 1 if you have the `preadv' and `pwritev' functions, respectively
+#if !defined(FOLLY_HAVE_PREADV) && !defined(FOLLY_HAVE_PWRITEV)
+#if defined(__GLIBC_PREREQ) && !defined(__APPLE__)
+#if __GLIBC_PREREQ(2, 10)
+#define FOLLY_HAVE_PREADV 1
+#define FOLLY_HAVE_PWRITEV 1
+#endif
+#endif
+#endif
+
+#ifndef FOLLY_HAVE_PREADV
+#define FOLLY_HAVE_PREADV 0
+#endif
+
+#ifndef FOLLY_HAVE_PWRITEV
+#define FOLLY_HAVE_PWRITEV 0
+#endif
+
+/* Define to 1 if your architecture can handle unaligned loads and stores. */
+#if !defined(FOLLY_HAVE_UNALIGNED_ACCESS) && !defined(__ANDROID__) && \
+    !defined(__arm__)
+#define FOLLY_HAVE_UNALIGNED_ACCESS 1
+#endif
+
+/* Define to 1 if the linker supports weak symbols. */
+#if !defined(FOLLY_HAVE_WEAK_SYMBOLS) && !defined(__APPLE__) && !defined(_WIN32)
+#define FOLLY_HAVE_WEAK_SYMBOLS 1
+#endif
+
+#if !defined(FOLLY_HAVE_WCHAR_SUPPORT) && !defined(__ANDROID__)
+#define FOLLY_HAVE_WCHAR_SUPPORT 1
+#endif
+
+/* Define to 1 if has <ext/random>. */
+#if !defined(FOLLY_HAVE_EXTRANDOM_SFMT19937) && defined(__has_include) && \
+    !defined(__ANDROID__)
+#if __has_include(<ext/random>)
+#define FOLLY_HAVE_EXTRANDOM_SFMT19937 1
+#endif
+#endif
+
+/* Define to 1 if the compiler has VLA (variable-length array) support,
+   otherwise define to 0 */
+#if !defined(FOLLY_HAVE_VLA) && !defined(_WIN32)
+#define FOLLY_HAVE_VLA 1
+#endif
+
+/* Define to 1 if the system has the type `_Bool'. */
+/* #undef HAVE__BOOL */
+
+/* Define to the sub-directory in which libtool stores uninstalled libraries.
+ */
+#ifndef FOLLY_LT_OBJDIR
+#define FOLLY_LT_OBJDIR ".libs/"
+#endif
+
+/* Name of package */
+#ifndef FOLLY_PACKAGE
+#define FOLLY_PACKAGE "folly"
+#endif
+
+/* Define to the address where bug reports for this package should be sent. */
+#ifndef FOLLY_PACKAGE_BUGREPORT
+#define FOLLY_PACKAGE_BUGREPORT "folly@fb.com"
+#endif
+
+/* Define to the full name of this package. */
+#ifndef FOLLY_PACKAGE_NAME
+#define FOLLY_PACKAGE_NAME "folly"
+#endif
+
+/* Define to the full name and version of this package. */
+#ifndef FOLLY_PACKAGE_STRING
+#define FOLLY_PACKAGE_STRING "folly 0.1"
+#endif
+
+/* Define to the one symbol short name of this package. */
+#ifndef FOLLY_PACKAGE_TARNAME
+#define FOLLY_PACKAGE_TARNAME "folly"
+#endif
+
+/* Define to the home page for this package. */
+#ifndef FOLLY_PACKAGE_URL
+#define FOLLY_PACKAGE_URL ""
+#endif
+
+/* Define to the version of this package. */
+#ifndef FOLLY_PACKAGE_VERSION
+#define FOLLY_PACKAGE_VERSION "0.1"
+#endif
+
+/* Define to 1 if you have the ANSI C header files. */
+#ifndef FOLLY_STDC_HEADERS
+#define FOLLY_STDC_HEADERS 1
+#endif
+
+/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */
+#ifndef FOLLY_TIME_WITH_SYS_TIME
+#define FOLLY_TIME_WITH_SYS_TIME 1
+#endif
+
+/* Define to 1 if openssl has the ASN1_TIME_diff() function */
+#ifndef FOLLY_HAVE_OPENSSL_ASN1_TIME_DIFF
+#define FOLLY_HAVE_OPENSSL_ASN1_TIME_DIFF 1
+#endif
+
+/* Define to 1 if this build supports use in shared libraries */
+#ifndef FOLLY_SUPPORT_SHARED_LIBRARY
+#define FOLLY_SUPPORT_SHARED_LIBRARY 1
+#endif
+
+/* Define to 1 if this build should use extern template for Future<Unit> and
+ * SemiFuture<Unit> */
+#ifndef FOLLY_USE_EXTERN_FUTURE_UNIT
+#if !defined(__ANDROID__) && !defined(__APPLE__)
+#define FOLLY_USE_EXTERN_FUTURE_UNIT 1
+#else
+#define FOLLY_USE_EXTERN_FUTURE_UNIT 0
+#endif
+#endif
+
+/* We always use libstdc++ in fbcode */
+#if !defined(FOLLY_USE_LIBSTDCPP) && !defined(FOLLY_USE_LIBCPP) && \
+    !defined(_WIN32) && !defined(__APPLE__)
+#define FOLLY_USE_LIBSTDCPP 1
+#endif
+
+/* Define to 1 if we're using libc++. */
+#if !defined(FOLLY_USE_LIBCPP) && defined(__APPLE__) && \
+    !defined(FOLLY_USE_LIBSTDCPP)
+#define FOLLY_USE_LIBCPP 1
+#endif
+
+#if defined(FOLLY_USE_LIBCPP) && defined(FOLLY_USE_LIBSTDCPP)
+#error "Both libstdc++ and libc++ cannot be used simultaneously"
+#endif
+
+/* Define to empty if `const' does not conform to ANSI C. */
+/* #undef const */
+
+/* Define to `__inline__' or `__inline' if that's what the C compiler
+   calls it, or to nothing if 'inline' is not supported under any name.  */
+#ifndef __cplusplus
+/* #undef inline */
+#endif
+
+/* Define to `unsigned int' if <sys/types.h> does not define. */
+/* #undef size_t */
+
+/* Define to empty if the keyword `volatile' does not work. Warning: valid
+   code using `volatile' can become incorrect without. Disable with care. */
+/* #undef volatile */
+
+/* Define to 1 if the gflags namespace is not "gflags" */
+#ifndef FOLLY_UNUSUAL_GFLAGS_NAMESPACE
+#define FOLLY_UNUSUAL_GFLAGS_NAMESPACE 1
+#endif
+
+/* Define to gflags namespace ("google" or "gflags") */
+#ifndef FOLLY_GFLAGS_NAMESPACE
+#define FOLLY_GFLAGS_NAMESPACE google
+#endif
+
+#if !defined(FOLLY_HAVE_PTHREAD_SPINLOCK_T) && !FOLLY_MOBILE && \
+    !defined(__APPLE__) && !defined(_WIN32)
+#define FOLLY_HAVE_PTHREAD_SPINLOCK_T 1
+#endif
+
+#if !defined(FOLLY_HAVE_PTHREAD_ATFORK) && !defined(__ANDROID__) && \
+    !defined(_WIN32)
+#define FOLLY_HAVE_PTHREAD_ATFORK 1
+#endif
+
+#define FOLLY_DEMANGLE_MAX_SYMBOL_SIZE 1024
+
+#if __GNUC__ && !__clang__
+#define FOLLY_HAVE_SHADOW_LOCAL_WARNINGS 1
+#endif
+
+#ifndef _WIN32
+#define FOLLY_HAVE_PTHREAD 1
+#endif
+
+/* For internal fbsource-based builds, ASAN is always enabled globally for the
+ * entire build or not.  FOLLY_LIBRARY_SANITIZE_ADDRESS is therefore the same as
+ * FOLLY_SANITIZE_ADDRESS.  We never have builds where folly is compiled with
+ * ASAN enabled but other libraries have ASAN disabled, or vice-versa.
+ */
+#if defined(__has_feature)
+#if __has_feature(address_sanitizer) || __SANITIZE_ADDRESS__
+#define FOLLY_LIBRARY_SANITIZE_ADDRESS 1
+#else
+#define FOLLY_LIBRARY_SANITIZE_ADDRESS 0
+#endif
+#else // defined(__has_feature)
+#if __SANITIZE_ADDRESS__
+#define FOLLY_LIBRARY_SANITIZE_ADDRESS 1
+#else
+#define FOLLY_LIBRARY_SANITIZE_ADDRESS 0
+#endif
+#endif // defined(__has_feature)
+
+// We depend on JEMalloc headers in fbcode, so use them (note that when using
+// sanitizers, `#ifdef` gates in the code will not use JEMalloc headers,
+// despite this setting).  This set of defines is a bit complex, but we try to
+// come up with a set of defines that will only match in fbcode, where JEMalloc
+// is available.
+#if !defined(FOLLY_USE_JEMALLOC) && !FOLLY_MOBILE && !defined(__APPLE__) && \
+    !defined(__ANDROID__) && !defined(_WIN32) && !defined(__wasm32__)
+#define FOLLY_USE_JEMALLOC 1
+#endif
+
+// We depend on range-v3 in fbcode. This set of defines is a bit complex, but we
+// try to come up with a condition that matches in fbcode, where range-v3 is
+// both available and desired.
+#if !defined(FOLLY_USE_RANGEV3) && !FOLLY_MOBILE && !defined(__APPLE__) && \
+    !defined(__ANDROID__) && !defined(_WIN32)
+#define FOLLY_USE_RANGEV3 1
+#endif
+
+// pipe2 is available on glibc 2.9 or later
+#ifdef __GLIBC__
+#if (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 9))
+#ifndef FOLLY_HAVE_PIPE2
+#define FOLLY_HAVE_PIPE2 1
+#endif
+#endif
+#endif
+
+// accept4, recvmmsg, sendmmsg
+#if defined(__linux__) && (!defined(__ANDROID__) || (__ANDROID_API__ >= 21))
+#define FOLLY_HAVE_ACCEPT4 1
+#define FOLLY_HAVE_RECVMMSG 1
+#define FOLLY_HAVE_SENDMMSG 1
+#endif
+
+#if defined(__GLIBC_PREREQ)
+#if __GLIBC_PREREQ(2, 25)
+#define FOLLY_HAVE_GETRANDOM 1
+#else
+#define FOLLY_HAVE_GETRANDOM 0
+#endif
+#else
+#define FOLLY_HAVE_GETRANDOM 0
+#endif
+
+#ifndef FOLLY_HAVE_LIBRT
+#if defined(__linux__) && !FOLLY_MOBILE
+#define FOLLY_HAVE_LIBRT 1
+#else
+#define FOLLY_HAVE_LIBRT 0
+#endif
+#endif
+
+// disable coroutines in our fork as we only need headers for containers
+#define FOLLY_CFG_NO_COROUTINES 1
+
+/* once: _FOLLY_CONFIG_H */
+#endif

From 179db036bf51b3cecb1a369845f42db4ef3b5469 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 13 Aug 2024 15:16:38 +0100
Subject: [PATCH 179/188] nix: update flake lock

This comes with some formatting changes due to a `clang-format` update.
---
 extern/shim-folly-config.h |   8 +--
 flake.lock                 |  18 +++---
 flake.nix                  |  18 ++++--
 oi/DrgnUtils.cpp           |   2 +-
 oi/DrgnUtils.h             |   4 +-
 oi/IntrospectionResult.cpp |   3 +-
 oi/OILexer.h               |   4 +-
 oi/Serialize.cpp           | 116 ++++++++++++++++++-------------------
 8 files changed, 91 insertions(+), 82 deletions(-)

diff --git a/extern/shim-folly-config.h b/extern/shim-folly-config.h
index 5307d07..737c3f6 100644
--- a/extern/shim-folly-config.h
+++ b/extern/shim-folly-config.h
@@ -18,7 +18,7 @@
 #define _FOLLY_CONFIG_H 1
 
 #ifdef __APPLE__
-#include <TargetConditionals.h> // @manual
+#include <TargetConditionals.h>  // @manual
 #endif
 
 #include <time.h>
@@ -31,7 +31,7 @@
 #else
 #define FOLLY_MOBILE 0
 #endif
-#endif // FOLLY_MOBILE
+#endif  // FOLLY_MOBILE
 
 /* define if the Boost library is available */
 #ifndef FOLLY_HAVE_BOOST
@@ -387,13 +387,13 @@
 #else
 #define FOLLY_LIBRARY_SANITIZE_ADDRESS 0
 #endif
-#else // defined(__has_feature)
+#else  // defined(__has_feature)
 #if __SANITIZE_ADDRESS__
 #define FOLLY_LIBRARY_SANITIZE_ADDRESS 1
 #else
 #define FOLLY_LIBRARY_SANITIZE_ADDRESS 0
 #endif
-#endif // defined(__has_feature)
+#endif  // defined(__has_feature)
 
 // We depend on JEMalloc headers in fbcode, so use them (note that when using
 // sanitizers, `#ifdef` gates in the code will not use JEMalloc headers,
diff --git a/flake.lock b/flake.lock
index 927748e..7e86812 100644
--- a/flake.lock
+++ b/flake.lock
@@ -5,11 +5,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1694529238,
-        "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=",
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "ff7b65b44d01cf9ba6a71320833626af21126384",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
         "type": "github"
       },
       "original": {
@@ -20,11 +20,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1696375444,
-        "narHash": "sha256-Sv0ICt/pXfpnFhTGYTsX6lUr1SljnuXWejYTI2ZqHa4=",
+        "lastModified": 1723362943,
+        "narHash": "sha256-dFZRVSgmJkyM0bkPpaYRtG/kRMRTorUIDj8BxoOt1T4=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "81e8f48ebdecf07aab321182011b067aafc78896",
+        "rev": "a58bc8ad779655e790115244571758e8de055e3d",
         "type": "github"
       },
       "original": {
@@ -63,11 +63,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1695822946,
-        "narHash": "sha256-IQU3fYo0H+oGlqX5YrgZU3VRhbt2Oqe6KmslQKUO4II=",
+        "lastModified": 1723454642,
+        "narHash": "sha256-S0Gvsenh0II7EAaoc9158ZB4vYyuycvMGKGxIbERNAM=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "720bd006d855b08e60664e4683ccddb7a9ff614a",
+        "rev": "349de7bc435bdff37785c2466f054ed1766173be",
         "type": "github"
       },
       "original": {
diff --git a/flake.nix b/flake.nix
index b35d91e..7a3382e 100644
--- a/flake.nix
+++ b/flake.nix
@@ -10,8 +10,16 @@
     treefmt-nix.inputs.nixpkgs.follows = "nixpkgs";
   };
 
-  outputs = { self, nixpkgs, flake-utils, treefmt-nix, ... }@inputs:
-    flake-utils.lib.eachDefaultSystem (system:
+  outputs =
+    {
+      self,
+      nixpkgs,
+      flake-utils,
+      treefmt-nix,
+      ...
+    }@inputs:
+    flake-utils.lib.eachDefaultSystem (
+      system:
       let
         pkgs = nixpkgs.legacyPackages.${system};
         treefmtEval = treefmt-nix.lib.evalModule pkgs (pkgs: {
@@ -23,8 +31,10 @@
           programs.black.enable = true;
           programs.isort.enable = true;
         });
-      in {
+      in
+      {
         formatter = treefmtEval.config.build.wrapper;
         checks.formatting = treefmtEval.config.build.check self;
-      });
+      }
+    );
 }
diff --git a/oi/DrgnUtils.cpp b/oi/DrgnUtils.cpp
index e7a4a0c..90a7b02 100644
--- a/oi/DrgnUtils.cpp
+++ b/oi/DrgnUtils.cpp
@@ -54,7 +54,7 @@ func_iterator::func_iterator(drgn_program* prog) {
   iter.reset(ret, Deleter());
 }
 
-func_iterator::func_iterator(program& prog) : func_iterator(prog.get()){};
+func_iterator::func_iterator(program& prog) : func_iterator(prog.get()) {};
 
 void func_iterator::Deleter::operator()(drgn_func_iterator* _iter) noexcept {
   drgn_func_iterator_destroy(_iter);
diff --git a/oi/DrgnUtils.h b/oi/DrgnUtils.h
index 2724f7a..6f4ed36 100644
--- a/oi/DrgnUtils.h
+++ b/oi/DrgnUtils.h
@@ -41,7 +41,7 @@ class error : public std::exception {
   struct Deleter {
     void operator()(drgn_error* err) noexcept;
   };
-  error(drgn_error* err) : ptr(err){};
+  error(drgn_error* err) : ptr(err) {};
 
   operator bool() const {
     return static_cast<bool>(ptr);
@@ -65,7 +65,7 @@ class program {
   };
 
   program();
-  program(drgn_program* prog) : ptr(prog){};
+  program(drgn_program* prog) : ptr(prog) {};
 
   symbols find_all_symbols();
 
diff --git a/oi/IntrospectionResult.cpp b/oi/IntrospectionResult.cpp
index 4b4839d..57fe63e 100644
--- a/oi/IntrospectionResult.cpp
+++ b/oi/IntrospectionResult.cpp
@@ -83,8 +83,7 @@ IntrospectionResult::const_iterator::operator++() {
 
             for (const auto& [dy, handler] : ty.processors) {
               auto parsed = exporters::ParsedData::parse(data_, dy);
-              handler(
-                  *next_, [this](auto i) { stack_.emplace(i); }, parsed);
+              handler(*next_, [this](auto i) { stack_.emplace(i); }, parsed);
             }
 
             if (auto new_name = genNameFromData(next_->data)) {
diff --git a/oi/OILexer.h b/oi/OILexer.h
index adcc216..12ed731 100644
--- a/oi/OILexer.h
+++ b/oi/OILexer.h
@@ -31,9 +31,9 @@ namespace oi::detail {
 
 class OIScanner : public yyFlexLexer {
  public:
-  OIScanner(std::istream* in) : yyFlexLexer(in){};
+  OIScanner(std::istream* in) : yyFlexLexer(in) {};
 
-  virtual ~OIScanner(){};
+  virtual ~OIScanner() {};
 
   // get rid of override virtual function warning
   using FlexLexer::yylex;
diff --git a/oi/Serialize.cpp b/oi/Serialize.cpp
index cd10f4f..3a60e2e 100644
--- a/oi/Serialize.cpp
+++ b/oi/Serialize.cpp
@@ -57,11 +57,11 @@ using oarchive = boost::archive::text_oarchive;
 template <class Archive>
 void serialize(Archive& ar, PaddingInfo& p, const unsigned int version) {
   verify_version<PaddingInfo>(version);
-  ar& p.structSize;
-  ar& p.paddingSize;
-  ar& p.definition;
-  ar& p.instancesCnt;
-  ar& p.savingSize;
+  ar & p.structSize;
+  ar & p.paddingSize;
+  ar & p.definition;
+  ar & p.instancesCnt;
+  ar & p.savingSize;
 }
 
 INSTANCIATE_SERIALIZE(PaddingInfo)
@@ -71,9 +71,9 @@ void serialize(Archive& ar,
                struct drgn_location_description& location,
                const unsigned int version) {
   verify_version<struct drgn_location_description>(version);
-  ar& location.start;
-  ar& location.end;
-  ar& location.expr_size;
+  ar & location.start;
+  ar & location.end;
+  ar & location.expr_size;
   if (Archive::is_loading::value) {
     // It is important to call `malloc` here instead of allocating with `new`
     // since these structs are usually allocated and deallocated directly by
@@ -91,11 +91,11 @@ void serialize(Archive& ar,
                struct drgn_object_locator& locator,
                const unsigned int version) {
   verify_version<struct drgn_object_locator>(version);
-  ar& locator.module_start;
-  ar& locator.module_end;
-  ar& locator.module_bias;
-  ar& locator.locations_size;
-  ar& locator.frame_base_locations_size;
+  ar & locator.module_start;
+  ar & locator.module_end;
+  ar & locator.module_bias;
+  ar & locator.locations_size;
+  ar & locator.frame_base_locations_size;
   if (Archive::is_loading::value) {
     // It is important to call `malloc` here instead of allocating with `new`
     // since these structs are usually allocated and deallocated directly by
@@ -110,7 +110,7 @@ void serialize(Archive& ar,
                                                    locator.locations_size);
   ar& make_array<struct drgn_location_description>(
       locator.frame_base_locations, locator.frame_base_locations_size);
-  ar& locator.qualified_type;
+  ar & locator.qualified_type;
 }
 
 INSTANCIATE_SERIALIZE(struct drgn_object_locator)
@@ -118,9 +118,9 @@ INSTANCIATE_SERIALIZE(struct drgn_object_locator)
 template <class Archive>
 void serialize(Archive& ar, FuncDesc::Arg& arg, const unsigned int version) {
   verify_version<FuncDesc::Arg>(version);
-  ar& arg.typeName;
-  ar& arg.valid;
-  ar& arg.locator;
+  ar & arg.typeName;
+  ar & arg.valid;
+  ar & arg.locator;
 }
 
 INSTANCIATE_SERIALIZE(FuncDesc::Arg)
@@ -130,8 +130,8 @@ void serialize(Archive& ar,
                FuncDesc::Retval& retval,
                const unsigned int version) {
   verify_version<FuncDesc::Retval>(version);
-  ar& retval.typeName;
-  ar& retval.valid;
+  ar & retval.typeName;
+  ar & retval.valid;
 }
 
 INSTANCIATE_SERIALIZE(FuncDesc::Retval)
@@ -141,8 +141,8 @@ void serialize(Archive& ar,
                FuncDesc::Range& range,
                const unsigned int version) {
   verify_version<FuncDesc::Range>(version);
-  ar& range.start;
-  ar& range.end;
+  ar & range.start;
+  ar & range.end;
 }
 
 INSTANCIATE_SERIALIZE(FuncDesc::Range)
@@ -150,11 +150,11 @@ INSTANCIATE_SERIALIZE(FuncDesc::Range)
 template <class Archive>
 void serialize(Archive& ar, FuncDesc& fd, const unsigned int version) {
   verify_version<FuncDesc>(version);
-  ar& fd.symName;
-  ar& fd.ranges;
-  ar& fd.isMethod;
-  ar& fd.arguments;
-  ar& fd.retval;
+  ar & fd.symName;
+  ar & fd.ranges;
+  ar & fd.isMethod;
+  ar & fd.arguments;
+  ar & fd.retval;
 }
 
 INSTANCIATE_SERIALIZE(FuncDesc)
@@ -162,9 +162,9 @@ INSTANCIATE_SERIALIZE(FuncDesc)
 template <class Archive>
 void serialize(Archive& ar, GlobalDesc& gd, const unsigned int version) {
   verify_version<GlobalDesc>(version);
-  ar& gd.symName;
-  ar& gd.typeName;
-  ar& gd.baseAddr;
+  ar & gd.symName;
+  ar & gd.typeName;
+  ar & gd.baseAddr;
 }
 
 INSTANCIATE_SERIALIZE(GlobalDesc)
@@ -175,7 +175,7 @@ static void serialize_c_string(Archive& ar, char** string) {
   if (Archive::is_saving::value) {
     length = *string ? strlen(*string) : 0;
   }
-  ar& length;
+  ar & length;
   if (Archive::is_loading::value) {
     *string = length ? (char*)malloc(sizeof(char) * (length + 1)) : NULL;
   }
@@ -218,10 +218,10 @@ void serialize(Archive& ar,
   // `.kind` MUST be serialized first, not just because it's declared first,
   // but because all of the `drgn_type_has_*` functions rely on the value of
   // `.kind`
-  ar& type._private.kind;
-  ar& type._private.is_complete;
-  ar& type._private.primitive;
-  ar& type._private.qualifiers;
+  ar & type._private.kind;
+  ar & type._private.is_complete;
+  ar & type._private.primitive;
+  ar & type._private.qualifiers;
   // `.program` is NULL, per the initial `memset`
 
   if (Archive::is_loading::value) {
@@ -247,13 +247,13 @@ void serialize(Archive& ar,
   assert_in_same_union(size, num_enumerators);
   assert_in_same_union(size, is_variadic);
   if (drgn_type_has_size(&type)) {
-    ar& type._private.size;
+    ar & type._private.size;
   } else if (drgn_type_has_length(&type)) {
-    ar& type._private.length;
+    ar & type._private.length;
   } else if (drgn_type_has_enumerators(&type)) {
-    ar& type._private.num_enumerators;
+    ar & type._private.num_enumerators;
   } else if (drgn_type_has_is_variadic(&type)) {
-    ar& type._private.is_variadic;
+    ar & type._private.is_variadic;
   }
 
   // Third union: `is_signed`, `num_members`, `num_paramters`
@@ -265,7 +265,7 @@ void serialize(Archive& ar,
   assert_in_same_union(little_endian, enumerators);
   assert_in_same_union(little_endian, parameters);
   if (drgn_type_has_little_endian(&type)) {
-    ar& type._private.little_endian;
+    ar & type._private.little_endian;
   } else if (drgn_type_has_members(&type)) {
     // Leave `members` set to NULL per the initial `memset`,
     // see "AVOIDING OVERSERIALIZATION" comment above
@@ -282,7 +282,7 @@ void serialize(Archive& ar,
   // "AVOIDING OVERSERIALIZATION" comment above
 
   uintptr_t die_addr = (uintptr_t)type._private.die_addr;
-  ar& die_addr;
+  ar & die_addr;
   type._private.die_addr = (void*)die_addr;
   // `.module` is NULL, per the initial `memset`
   if (Archive::is_saving::value) {
@@ -293,7 +293,7 @@ void serialize(Archive& ar,
           std::numeric_limits<decltype(type._private.oi_size)>::max();
     }
   }
-  ar& type._private.oi_size;
+  ar & type._private.oi_size;
 
   // It's important that `oi_name` is declared here and not inside the
   // if statement so that its data isn't freed when we call
@@ -306,7 +306,7 @@ void serialize(Archive& ar,
   serialize_c_string(ar, const_cast<char**>(&type._private.oi_name));
 
   if (drgn_type_kind(&type) == DRGN_TYPE_ARRAY) {
-    ar& type._private.type;
+    ar & type._private.type;
   }
 #undef assert_in_same_union
 }
@@ -318,10 +318,10 @@ void serialize(Archive& ar,
                struct DrgnClassMemberInfo& m,
                const unsigned int version) {
   verify_version<struct DrgnClassMemberInfo>(version);
-  ar& m.type;
-  ar& m.member_name;
-  ar& m.bit_offset;
-  ar& m.bit_field_size;
+  ar & m.type;
+  ar & m.member_name;
+  ar & m.bit_offset;
+  ar & m.bit_field_size;
 }
 
 INSTANCIATE_SERIALIZE(DrgnClassMemberInfo)
@@ -331,8 +331,8 @@ void serialize(Archive& ar,
                struct drgn_qualified_type& type,
                const unsigned int version) {
   verify_version<struct drgn_qualified_type>(version);
-  ar& type.type;
-  ar& type.qualifiers;
+  ar & type.type;
+  ar & type.qualifiers;
 }
 
 INSTANCIATE_SERIALIZE(struct drgn_qualified_type)
@@ -340,8 +340,8 @@ INSTANCIATE_SERIALIZE(struct drgn_qualified_type)
 template <class Archive>
 void serialize(Archive& ar, RootInfo& rootInfo, const unsigned int version) {
   verify_version<RootInfo>(version);
-  ar& rootInfo.varName;
-  ar& rootInfo.type;
+  ar & rootInfo.varName;
+  ar & rootInfo.type;
 }
 
 INSTANCIATE_SERIALIZE(RootInfo)
@@ -351,14 +351,14 @@ void serialize(Archive& ar,
                struct TypeHierarchy& th,
                const unsigned int version) {
   verify_version<TypeHierarchy>(version);
-  ar& th.classMembersMap;
-  ar& th.containerTypeMap;
-  ar& th.typedefMap;
-  ar& th.sizeMap;
-  ar& th.knownDummyTypeList;
-  ar& th.pointerToTypeMap;
-  ar& th.thriftIssetStructTypes;
-  ar& th.descendantClasses;
+  ar & th.classMembersMap;
+  ar & th.containerTypeMap;
+  ar & th.typedefMap;
+  ar & th.sizeMap;
+  ar & th.knownDummyTypeList;
+  ar & th.pointerToTypeMap;
+  ar & th.thriftIssetStructTypes;
+  ar & th.descendantClasses;
 }
 
 INSTANCIATE_SERIALIZE(struct TypeHierarchy)

From 6d94f2da819874dcc8d4556e0b27b8b349ad6f5b Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 14 Aug 2024 17:34:09 +0100
Subject: [PATCH 180/188] nix: add building oid to the flake

---
 .circleci/config.yml                       |  18 +-
 .github/workflows/object-introspection.yml |  17 +-
 CMakeLists.txt                             | 230 ++++++++++++---------
 README.md                                  |  58 ++++++
 cmake/Finduring.cmake                      |  27 +++
 flake.nix                                  |  96 ++++++++-
 oi/CMakeLists.txt                          |   3 -
 oi/type_graph/CMakeLists.txt               |   3 -
 tools/config_gen.py                        |   6 +-
 9 files changed, 344 insertions(+), 114 deletions(-)
 create mode 100644 cmake/Finduring.cmake

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 589245a..dba98d7 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -83,20 +83,26 @@ jobs:
               libgtest-dev \
               libjemalloc-dev \
               libmsgpack-dev \
+              libomp-12-dev \
+              liburing-dev \
               libzstd-dev \
               llvm-15-dev \
               ninja-build \
               pkg-config \
               python3-setuptools
             pip3 install toml
+
+            # Ubuntu 22.04 CMake is too old and we don't have a newer image yet
+            git clone --depth 1 --branch v3.30.2 https://github.com/Kitware/CMake.git /tmp/cmake
+            (cd /tmp/cmake && cmake -B build/ -G Ninja && cmake --build build/)
           environment:
             DEBIAN_FRONTEND: noninteractive
       - checkout
       - run:
           name: Build
           command: |
-            cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
-            cmake --build build/
+            /tmp/cmake/build/bin/cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
+            ninja -C build/
             # Testing rubbish:
             cp test/ci.oid.toml build/testing.oid.toml
       - persist_to_workspace:
@@ -131,10 +137,12 @@ jobs:
             sudo apt-get install -y \
               clang-15 \
               libboost-all-dev \
-              libgflags-dev \
-              llvm-15-dev \
               libfmt-dev \
-              libjemalloc-dev
+              libgflags-dev \
+              libgoogle-glog-dev \
+              libjemalloc-dev \
+              libomp-12-dev \
+              llvm-15-dev
           environment:
             DEBIAN_FRONTEND: noninteractive
       - run:
diff --git a/.github/workflows/object-introspection.yml b/.github/workflows/object-introspection.yml
index 70b41aa..9b26214 100644
--- a/.github/workflows/object-introspection.yml
+++ b/.github/workflows/object-introspection.yml
@@ -5,11 +5,24 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: cachix/install-nix-action@v25
+    - uses: actions/checkout@v4.1.7
+    - uses: cachix/install-nix-action@v27
       with:
         github_access_token: ${{ secrets.GITHUB_TOKEN }}
     - name: nix fmt
       run: |-
         nix --experimental-features 'nix-command flakes' fmt
         git diff --exit-code
+
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        llvm_version: [15]
+    steps:
+    - uses: actions/checkout@v4.1.7
+    - uses: cachix/install-nix-action@v27
+      with:
+        github_access_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: build (LLVM ${{ matrix.llvm_version }})
+      run: nix build -L .#oid-llvm${{ matrix.llvm_version }}
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8674d46..426b68c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,5 +1,5 @@
 # object-introspection
-cmake_minimum_required(VERSION 3.20)
+cmake_minimum_required(VERSION 3.24)
 project(object-introspection)
 
 # Lets find_program() locate SETUID binaries
@@ -33,75 +33,108 @@ find_package(gflags REQUIRED)
 ### tomlplusplus (for configuration files)
 FetchContent_Declare(
   tomlplusplus
-  GIT_REPOSITORY https://github.com/marzer/tomlplusplus.git
-  GIT_TAG        4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
-  GIT_PROGRESS   TRUE
+  GIT_REPOSITORY    https://github.com/marzer/tomlplusplus.git
+  GIT_TAG           4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
 FetchContent_MakeAvailable(tomlplusplus)
 
 ### glog
-FetchContent_Declare(
-  glog
-  GIT_REPOSITORY https://github.com/google/glog.git
-  GIT_TAG        96a2f23dca4cc7180821ca5f32e526314395d26a
-  GIT_PROGRESS   TRUE
-)
-FetchContent_MakeAvailable(glog)
+find_package(glog)
+if (NOT glog_FOUND)
+  FetchContent_Declare(
+    glog
+    GIT_REPOSITORY    https://github.com/google/glog.git
+    GIT_TAG           96a2f23dca4cc7180821ca5f32e526314395d26a
+    GIT_PROGRESS      TRUE
+    FIND_PACKAGE_ARGS
+  )
+  FetchContent_MakeAvailable(glog)
 
-# These glog executable targets still generate warnings - disable warnings for
-# them explicitly
-target_compile_options(demangle_unittest PRIVATE "-w")
-target_compile_options(logging_unittest PRIVATE "-w")
-target_compile_options(stl_logging_unittest PRIVATE "-w")
-target_compile_options(symbolize_unittest PRIVATE "-w")
-target_compile_options(utilities_unittest PRIVATE "-w")
+  # These glog executable targets still generate warnings - disable warnings for
+  # them explicitly
+  target_compile_options(demangle_unittest PRIVATE "-w")
+  target_compile_options(logging_unittest PRIVATE "-w")
+  target_compile_options(stl_logging_unittest PRIVATE "-w")
+  target_compile_options(symbolize_unittest PRIVATE "-w")
+  target_compile_options(utilities_unittest PRIVATE "-w")
+endif()
 
-### googletest
+### GTest
 # Do this in the main file so it can be fetched before setting project warnings.
 # After this is fixed with FetchContent, move to test/CMakeLists.txt.
 FetchContent_Declare(
-  googletest
-  GIT_REPOSITORY https://github.com/google/googletest.git
-  GIT_TAG        1ed6a8c67a0bd675149ece27bbec0ef1759854cf
-  GIT_PROGRESS   TRUE
+  GTest
+  GIT_REPOSITORY    https://github.com/google/googletest.git
+  GIT_TAG           1ed6a8c67a0bd675149ece27bbec0ef1759854cf
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
-FetchContent_MakeAvailable(googletest)
+FetchContent_MakeAvailable(GTest)
+
+### liburing (for RocksDB)
+find_package(uring REQUIRED)
 
 ### rocksdb
-FetchContent_Declare(
-  rocksdb
-  GIT_REPOSITORY https://github.com/facebook/rocksdb.git
-  GIT_TAG        f32521662acf3352397d438b732144c7813bbbec # v8.5.3
-  GIT_PROGRESS   TRUE
-)
-FetchContent_Populate(rocksdb)
-add_custom_target(librocksdb ALL
-  WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
-  COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
-  COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
-  BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
-  COMMENT "Building RocksDB"
-  USES_TERMINAL
-)
-include_directories(SYSTEM "${rocksdb_SOURCE_DIR}/include")
+find_package(RocksDB 8.11 CONFIG)
+if (NOT RocksDB_FOUND)
+  FetchContent_Declare(
+    rocksdb
+    GIT_REPOSITORY https://github.com/facebook/rocksdb.git
+    GIT_TAG        f32521662acf3352397d438b732144c7813bbbec # v8.5.3
+    GIT_PROGRESS   TRUE
+  )
+  FetchContent_Populate(rocksdb)
+
+  add_custom_target(librocksdb_build ALL
+    WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
+    COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
+    COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
+    BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
+    COMMENT "Building RocksDB"
+    USES_TERMINAL
+  )
+
+  ### zstd (for rocksdb)
+  find_package(zstd REQUIRED)
+
+  add_library(librocksdb INTERFACE)
+  add_dependencies(librocksdb librocksdb_build)
+  target_include_directories(librocksdb INTERFACE SYSTEM "${rocksdb_SOURCE_DIR}/include")
+  target_link_libraries(librocksdb INTERFACE ${rocksdb_BINARY_DIR}/librocksdb.a zstd::zstd)
+
+  add_library(RocksDB::rocksdb ALIAS librocksdb)
+endif()
 
 ### folly
 ### use folly as a header only library. some features won't be supported.
-FetchContent_Declare(
-  folly
-  GIT_REPOSITORY https://github.com/facebook/folly.git
-  GIT_TAG        c5aa5c46291a27f69acc920894d43605ceb43eba
-  GIT_PROGRESS   TRUE
-  PATCH_COMMAND  ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
-)
-FetchContent_Populate(folly)
+find_package(folly CONFIG)
+if (NOT folly_FOUND)
+  FetchContent_Declare(
+    folly
+    GIT_REPOSITORY    https://github.com/facebook/folly.git
+    GIT_TAG           c5aa5c46291a27f69acc920894d43605ceb43eba
+    GIT_PROGRESS      TRUE
+    PATCH_COMMAND     ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
+  )
+  FetchContent_Populate(folly)
+
+  add_library(folly_headers INTERFACE)
+  target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
+  target_link_libraries(folly_headers INTERFACE Boost::headers)
+else()
+  add_library(folly_headers INTERFACE)
+  target_include_directories(folly_headers SYSTEM INTERFACE ${FOLLY_INCLUDE_DIR})
+endif()
 
 ### range-v3
 FetchContent_Declare(
   range-v3
-  GIT_REPOSITORY https://github.com/ericniebler/range-v3.git
-  GIT_TAG        a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
-  GIT_PROGRESS   TRUE
+  GIT_REPOSITORY    https://github.com/ericniebler/range-v3.git
+  GIT_TAG           a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
 FetchContent_MakeAvailable(range-v3)
 
@@ -124,6 +157,18 @@ endif()
 ### (Re)download submodules
 find_package(Git QUIET)
 
+if (DEFINED drgn_SOURCE_DIR)
+    # drgn's autotools build requires source modification. in case we have a
+    # readonly source (read: nix) we must copy this. do this always to avoid
+    # polluting the source.
+    file(COPY "${drgn_SOURCE_DIR}/" DESTINATION "${FETCHCONTENT_BASE_DIR}/drgn-src" NO_SOURCE_PERMISSIONS)
+    SET(drgn_SOURCE_DIR "${FETCHCONTENT_BASE_DIR}/drgn-src")
+    SET(drgn_BINARY_DIR "${CMAKE_BINARY_DIR}/_deps/drgn-build")
+else()
+    SET(drgn_SOURCE_DIR "${PROJECT_SOURCE_DIR}/extern/drgn")
+    SET(drgn_BINARY_DIR "${drgn_SOURCE_DIR}/build")
+endif()
+
 # TODO: No idea if this huge block is required, just picked from an example. There may be a short-hand.
 if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
 # Update submodules as needed
@@ -145,20 +190,17 @@ if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
         if(NOT GIT_SUBMOD_RESULT EQUAL "0")
             message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules")
         endif()
+
+        if(NOT EXISTS "${drgn_SOURCE_DIR}")
+            message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
+        endif()
     endif()
 endif()
 
-if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn")
-    message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
-endif()
 
 ### Select Python version
 find_program(PYTHON NAMES python3.9 python3)
 
-add_library(folly_headers INTERFACE)
-target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
-target_link_libraries(folly_headers INTERFACE Boost::headers)
-
 ### bison & flex (for oid_parser)
 find_package(BISON 3.5 REQUIRED)
 find_package(FLEX)
@@ -194,8 +236,11 @@ find_package(msgpack REQUIRED CONFIG)
 get_target_property(MSGPACK_INCLUDE_DIRS msgpackc INTERFACE_INCLUDE_DIRECTORIES)
 include_directories(SYSTEM ${MSGPACK_INCLUDE_DIRS})
 
-### zstd (for rocksdb)
-find_package(zstd REQUIRED)
+### drgn/elfutils dependencies
+find_package(BZip2 REQUIRED)
+find_package(OpenMP REQUIRED)
+find_package(LibLZMA REQUIRED)
+find_package(ZLIB REQUIRED)
 
 ### drgn
 # The setup.py script in drgn is really meant to build drgn (python
@@ -214,46 +259,38 @@ find_package(zstd REQUIRED)
 # make
 #
 # Since setup.py has a single cmd to do this, just use it for now.
-#
-# Another extemely annoying point. drgn pretty much has to be compiled with gcc only
-# clang-12 does NOT work. clang fails with the following error :-
-# configure: error: gcc with GNU99 support required
 
 set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no")
 if (ASAN)
   list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes")
 endif()
-add_custom_target(libdrgn ALL
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn
-  COMMAND unset BISON_PKGDATADIR && CC=gcc CFLAGS="${DRGN_CFLAGS}" CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp build
-  BYPRODUCTS ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/.libs/libdrgnimpl.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdw/libdw.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libelf/libelf.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdwelf/libdwelf.a
+
+add_custom_target(drgn_build ALL
+  WORKING_DIRECTORY ${drgn_SOURCE_DIR}
+  COMMAND unset BISON_PKGDATADIR && CFLAGS="${DRGN_CFLAGS}"
+  CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp "${drgn_BINARY_DIR}"
+  BYPRODUCTS ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
+    ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
+    ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
+    ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
   COMMENT "Building drgn"
   USES_TERMINAL
 )
-set(DRGN_PATH "${PROJECT_SOURCE_DIR}/extern/drgn/build")
 
-# Ideally drgn stuff should be together at the end. But looks like rpath needs
-# to be set before add_executable() unfortunately. Maybe split libdrgn stuff
-# into a separate file later.
-set(CMAKE_SKIP_BUILD_RPATH FALSE)
-set(CMAKE_INSTALL_RPATH
-  "${DRGN_PATH}/.libs"
-  "${DRGN_PATH}/velfutils/libdw"
-  "${DRGN_PATH}/velfutils/libelf"
-  "${DRGN_PATH}/velfutils/libdwelf"
-)
-set(CMAKE_BUILD_RPATH
-  "${DRGN_PATH}/.libs"
-  "${DRGN_PATH}/velfutils/libdw"
-  "${DRGN_PATH}/velfutils/libelf"
-  "${DRGN_PATH}/velfutils/libdwelf"
-)
-set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+add_library(drgn INTERFACE)
+add_dependencies(drgn drgn_build)
+target_link_libraries(drgn INTERFACE
+  ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
+  ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
+  ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
+  ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
 
-include_directories(SYSTEM "${DRGN_PATH}")
+  BZip2::BZip2
+  LibLZMA::LibLZMA
+  OpenMP::OpenMP_CXX
+  ZLIB::ZLIB
+)
+target_include_directories(drgn SYSTEM INTERFACE "${drgn_SOURCE_DIR}" "${drgn_BINARY_DIR}")
 
 if (STATIC_LINK)
   # glog links against the `gflags` target, which is an alias for `gflags_shared`
@@ -289,7 +326,6 @@ add_library(oicore
   oi/PaddingHunter.cpp
   oi/Serialize.cpp
 )
-add_dependencies(oicore libdrgn)
 target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -323,7 +359,6 @@ else()
 endif()
 
 target_link_libraries(oicore
-  "-L${DRGN_PATH}/.libs"
   drgn
   dw
   pthread
@@ -334,11 +369,9 @@ add_library(treebuilder
   oi/TreeBuilder.cpp
   oi/exporters/TypeCheckingWalker.cpp
 )
-add_dependencies(treebuilder librocksdb)
 target_link_libraries(treebuilder
-  ${rocksdb_BINARY_DIR}/librocksdb.a
+  RocksDB::rocksdb
   oicore # overkill but it does need a lot of stuff
-  zstd::zstd
 )
 
 
@@ -375,10 +408,8 @@ target_link_libraries(oip oicore)
 
 ### Object Introspection RocksDB Printer (OIRP)
 add_executable(oirp tools/OIRP.cpp)
-add_dependencies(oirp librocksdb)
 target_link_libraries(oirp
-  ${rocksdb_BINARY_DIR}/librocksdb.a
-  zstd::zstd
+  RocksDB::rocksdb
   msgpackc
 )
 
@@ -417,3 +448,6 @@ endif()
 if (DEFINED ENV{CMAKE_HOOK})
   include($ENV{CMAKE_HOOK})
 endif()
+
+install(TARGETS oid DESTINATION ${CMAKE_INSTALL_BINDIR})
+
diff --git a/README.md b/README.md
index 9f95bf8..abce6f8 100644
--- a/README.md
+++ b/README.md
@@ -14,3 +14,61 @@ See the [CONTRIBUTING](CONTRIBUTING.md) file for how to help out.
 
 ## License
 Object Introspection is licensed under the [Apache 2.0 License](LICENSE).
+
+## Getting started with Nix
+
+Nix is the easiest way to get started with `oid` as it is non-trivial to build otherwise. Explicit Nix support for Object Introspection as a Library will come down the line, but Nix can currently provide you a reproducible development environment in which to build it.
+
+These examples expect you to have `nix` installed and available with no other dependencies required. Find the installation guide at https://nixos.org/download.html.
+
+We also required flake support. To enable flakes globally run:
+
+    $ mkdir -p ~/.config/nix
+    $ echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf
+
+Or suffix every `nix` command with `nix --extra-experimental-features 'nix-command flakes'`.
+
+### Run upstream OID without modifying the source
+
+    $ nix run github:facebookexperimental/object-introspection -- --help
+
+This will download the latest source into your Nix store along with all of its dependencies, running help afterwards.
+
+### Build OID locally
+
+    $ git clone https://github.com/facebookexperimental/object-introspection
+    $ nix build
+    $ ./result/bin/oid --help
+
+This will build OID from your local sources. Please note that this will NOT pick up changes to `extern/drgn` or `extern/drgn/libdrgn/velfutils`.
+
+### Get a development environment
+
+    $ nix develop
+    $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+    $ ninja -C build
+    $ build/oid --help
+
+This command provides a development shell with all the required dependencies. This is the most flexible option and will pick up source changes as CMake normally would.
+
+Sometimes this developer environment can be polluted by things installed on your normal system. If this is an issue, use:
+
+    $ nix develop -i
+
+This removes the environment from your host system and makes the build pure.
+
+### Run the tests
+
+    $ nix develop
+    $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+    $ ninja -C build
+    $ ./tools/config_gen.py -c clang++ build/testing.oid.toml
+    $ ctest -j --test-dir build/test
+
+Running tests under `nix` is new to the project and may take some time to mature. The CI is the source of truth for now.
+
+### Format source
+
+    $ nix fmt
+
+This formats the Nix, C++, and Python code in the repository.
diff --git a/cmake/Finduring.cmake b/cmake/Finduring.cmake
new file mode 100644
index 0000000..3b007fe
--- /dev/null
+++ b/cmake/Finduring.cmake
@@ -0,0 +1,27 @@
+# - Find liburing
+#
+# uring_INCLUDE_DIR - Where to find liburing.h
+# uring_LIBRARIES - List of libraries when using uring.
+# uring_FOUND - True if uring found.
+
+find_path(uring_INCLUDE_DIR
+  NAMES liburing.h)
+find_library(uring_LIBRARIES
+  NAMES liburing.a liburing.so)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(uring
+  DEFAULT_MSG uring_LIBRARIES uring_INCLUDE_DIR)
+
+mark_as_advanced(
+  uring_INCLUDE_DIR
+  uring_LIBRARIES)
+
+if(uring_FOUND AND NOT TARGET uring::uring)
+  add_library(uring::uring UNKNOWN IMPORTED)
+  set_target_properties(uring::uring PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${uring_INCLUDE_DIR}"
+    IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+    IMPORTED_LOCATION "${uring_LIBRARIES}")
+endif()
+
diff --git a/flake.nix b/flake.nix
index 7a3382e..1dffbe0 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,5 +1,5 @@
 {
-  description = "A flake for building Object Introspection.";
+  description = "Object level memory profiler for C++";
 
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
@@ -18,7 +18,99 @@
       treefmt-nix,
       ...
     }@inputs:
-    flake-utils.lib.eachDefaultSystem (
+    flake-utils.lib.eachSystem [ flake-utils.lib.system.x86_64-linux ] (
+      system:
+      let
+        pkgs = import nixpkgs { inherit system; };
+
+        drgnSrc = pkgs.fetchFromGitHub {
+          owner = "JakeHillion";
+          repo = "drgn";
+          rev = "b1f8c3e8526611b6720800250ba858a713dd9e4f";
+          hash = "sha256-5WhMHgx/RKtqjxGx4AyiqVKMot5xulr+6c8i2E9IxiA=";
+          fetchSubmodules = true;
+        };
+
+        mkOidPackage =
+          llvmPackages:
+          with pkgs;
+          pkgs.llvmPackages_15.stdenv.mkDerivation rec {
+            name = "oid";
+
+            src = self;
+
+            nativeBuildInputs = [
+              autoconf
+              automake
+              bison
+              cmake
+              flex
+              gettext
+              git
+              hexdump
+              libtool
+              ninja
+              pkgconf
+              python312
+              python312Packages.setuptools
+              python312Packages.toml
+              glibcLocales
+            ];
+
+            buildInputs = [
+              llvmPackages.libclang
+              llvmPackages.llvm
+
+              boost
+              bzip2
+              curl
+              double-conversion
+              elfutils
+              flex
+              folly
+              folly.fmt
+              gflags
+              glog
+              gtest
+              icu
+              jemalloc
+              libarchive
+              libmicrohttpd
+              liburing
+              libxml2
+              lzma
+              msgpack
+              range-v3
+              rocksdb_8_11
+              sqlite
+              tomlplusplus
+              zstd
+
+              llvmPackages_15.openmp # should match the stdenv clang version, see: https://github.com/NixOS/nixpkgs/issues/79818
+            ];
+
+            cmakeFlags = [
+              "-Ddrgn_SOURCE_DIR=${drgnSrc}"
+              "-DFORCE_BOOST_STATIC=Off"
+            ];
+
+            outputs = [ "out" ];
+          };
+      in
+      {
+        packages = rec {
+          default = oid-llvm15;
+
+          oid-llvm15 = mkOidPackage pkgs.llvmPackages_15;
+        };
+
+        apps.default = {
+          type = "app";
+          program = "${self.packages.${system}.default}/bin/oid";
+        };
+      }
+    )
+    // flake-utils.lib.eachDefaultSystem (
       system:
       let
         pkgs = nixpkgs.legacyPackages.${system};
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index bb73871..7d805e5 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus)
 add_library(drgn_utils DrgnUtils.cpp)
 target_link_libraries(drgn_utils
   glog::glog
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
-add_dependencies(drgn_utils libdrgn)
 
 add_library(symbol_service
   Descs.cpp
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 6c4da13..c878e08 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -20,12 +20,9 @@ add_library(type_graph
   TypeIdentifier.cpp
   Types.cpp
 )
-add_dependencies(type_graph libdrgn)
 target_link_libraries(type_graph
   container_info
   symbol_service
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
 target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/tools/config_gen.py b/tools/config_gen.py
index c20ca40..5ac56ff 100755
--- a/tools/config_gen.py
+++ b/tools/config_gen.py
@@ -16,6 +16,7 @@
 
 import argparse
 import getpass
+import os
 import pathlib
 import subprocess
 import typing
@@ -170,7 +171,10 @@ def pull_base_toml() -> typing.Dict:
 
     # Now, we need to replace any placeholders that might be present in the base toml file with the real verisons.
     user = getpass.getuser()
-    pwd = str(repo_path.resolve())
+    if "IN_NIX_SHELL" in os.environ and "src" in os.environ:
+        pwd = os.environ['src']
+    else:
+        pwd = str(repo_path.resolve())
 
     container_list = base.get("types", {}).get("containers")
     if container_list:

From b7cc5a0cfe41b0816b7ae54c78e5a2a0785d1dfa Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 13 Aug 2024 13:28:02 +0100
Subject: [PATCH 181/188] nix: add building oid to the flake

---
 .circleci/config.yml                       |  18 +-
 .github/workflows/object-introspection.yml |  22 +-
 CMakeLists.txt                             | 230 ++++++++++++---------
 README.md                                  |  58 ++++++
 cmake/Finduring.cmake                      |  27 +++
 flake.nix                                  |  96 ++++++++-
 oi/CMakeLists.txt                          |   3 -
 oi/type_graph/CMakeLists.txt               |   3 -
 tools/config_gen.py                        |   6 +-
 9 files changed, 349 insertions(+), 114 deletions(-)
 create mode 100644 cmake/Finduring.cmake

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 589245a..dba98d7 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -83,20 +83,26 @@ jobs:
               libgtest-dev \
               libjemalloc-dev \
               libmsgpack-dev \
+              libomp-12-dev \
+              liburing-dev \
               libzstd-dev \
               llvm-15-dev \
               ninja-build \
               pkg-config \
               python3-setuptools
             pip3 install toml
+
+            # Ubuntu 22.04 CMake is too old and we don't have a newer image yet
+            git clone --depth 1 --branch v3.30.2 https://github.com/Kitware/CMake.git /tmp/cmake
+            (cd /tmp/cmake && cmake -B build/ -G Ninja && cmake --build build/)
           environment:
             DEBIAN_FRONTEND: noninteractive
       - checkout
       - run:
           name: Build
           command: |
-            cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
-            cmake --build build/
+            /tmp/cmake/build/bin/cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
+            ninja -C build/
             # Testing rubbish:
             cp test/ci.oid.toml build/testing.oid.toml
       - persist_to_workspace:
@@ -131,10 +137,12 @@ jobs:
             sudo apt-get install -y \
               clang-15 \
               libboost-all-dev \
-              libgflags-dev \
-              llvm-15-dev \
               libfmt-dev \
-              libjemalloc-dev
+              libgflags-dev \
+              libgoogle-glog-dev \
+              libjemalloc-dev \
+              libomp-12-dev \
+              llvm-15-dev
           environment:
             DEBIAN_FRONTEND: noninteractive
       - run:
diff --git a/.github/workflows/object-introspection.yml b/.github/workflows/object-introspection.yml
index 70b41aa..bbb1a6b 100644
--- a/.github/workflows/object-introspection.yml
+++ b/.github/workflows/object-introspection.yml
@@ -5,11 +5,29 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: cachix/install-nix-action@v25
+    - uses: actions/checkout@v4.1.7
+    - uses: cachix/install-nix-action@v27
       with:
         github_access_token: ${{ secrets.GITHUB_TOKEN }}
     - name: nix fmt
       run: |-
         nix --experimental-features 'nix-command flakes' fmt
         git diff --exit-code
+
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        llvm_version: [15]
+    steps:
+    - uses: actions/checkout@v4.1.7
+    - uses: cachix/install-nix-action@v27
+      with:
+        github_access_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: build (LLVM ${{ matrix.llvm_version }})
+      # Run the build manually in `nix develop` to keep non-outputs around
+      run: nix develop .#oid-llvm${{ matrix.llvm_version }} --command genericBuild
+    - name: test (LLVM ${{ matrix.llvm_version }})
+      run: |
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ./tools/config_gen.py -c clang++ build/testing.oid.toml
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ctest -j --test-dir source/build/test/
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8674d46..426b68c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,5 +1,5 @@
 # object-introspection
-cmake_minimum_required(VERSION 3.20)
+cmake_minimum_required(VERSION 3.24)
 project(object-introspection)
 
 # Lets find_program() locate SETUID binaries
@@ -33,75 +33,108 @@ find_package(gflags REQUIRED)
 ### tomlplusplus (for configuration files)
 FetchContent_Declare(
   tomlplusplus
-  GIT_REPOSITORY https://github.com/marzer/tomlplusplus.git
-  GIT_TAG        4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
-  GIT_PROGRESS   TRUE
+  GIT_REPOSITORY    https://github.com/marzer/tomlplusplus.git
+  GIT_TAG           4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
 FetchContent_MakeAvailable(tomlplusplus)
 
 ### glog
-FetchContent_Declare(
-  glog
-  GIT_REPOSITORY https://github.com/google/glog.git
-  GIT_TAG        96a2f23dca4cc7180821ca5f32e526314395d26a
-  GIT_PROGRESS   TRUE
-)
-FetchContent_MakeAvailable(glog)
+find_package(glog)
+if (NOT glog_FOUND)
+  FetchContent_Declare(
+    glog
+    GIT_REPOSITORY    https://github.com/google/glog.git
+    GIT_TAG           96a2f23dca4cc7180821ca5f32e526314395d26a
+    GIT_PROGRESS      TRUE
+    FIND_PACKAGE_ARGS
+  )
+  FetchContent_MakeAvailable(glog)
 
-# These glog executable targets still generate warnings - disable warnings for
-# them explicitly
-target_compile_options(demangle_unittest PRIVATE "-w")
-target_compile_options(logging_unittest PRIVATE "-w")
-target_compile_options(stl_logging_unittest PRIVATE "-w")
-target_compile_options(symbolize_unittest PRIVATE "-w")
-target_compile_options(utilities_unittest PRIVATE "-w")
+  # These glog executable targets still generate warnings - disable warnings for
+  # them explicitly
+  target_compile_options(demangle_unittest PRIVATE "-w")
+  target_compile_options(logging_unittest PRIVATE "-w")
+  target_compile_options(stl_logging_unittest PRIVATE "-w")
+  target_compile_options(symbolize_unittest PRIVATE "-w")
+  target_compile_options(utilities_unittest PRIVATE "-w")
+endif()
 
-### googletest
+### GTest
 # Do this in the main file so it can be fetched before setting project warnings.
 # After this is fixed with FetchContent, move to test/CMakeLists.txt.
 FetchContent_Declare(
-  googletest
-  GIT_REPOSITORY https://github.com/google/googletest.git
-  GIT_TAG        1ed6a8c67a0bd675149ece27bbec0ef1759854cf
-  GIT_PROGRESS   TRUE
+  GTest
+  GIT_REPOSITORY    https://github.com/google/googletest.git
+  GIT_TAG           1ed6a8c67a0bd675149ece27bbec0ef1759854cf
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
-FetchContent_MakeAvailable(googletest)
+FetchContent_MakeAvailable(GTest)
+
+### liburing (for RocksDB)
+find_package(uring REQUIRED)
 
 ### rocksdb
-FetchContent_Declare(
-  rocksdb
-  GIT_REPOSITORY https://github.com/facebook/rocksdb.git
-  GIT_TAG        f32521662acf3352397d438b732144c7813bbbec # v8.5.3
-  GIT_PROGRESS   TRUE
-)
-FetchContent_Populate(rocksdb)
-add_custom_target(librocksdb ALL
-  WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
-  COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
-  COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
-  BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
-  COMMENT "Building RocksDB"
-  USES_TERMINAL
-)
-include_directories(SYSTEM "${rocksdb_SOURCE_DIR}/include")
+find_package(RocksDB 8.11 CONFIG)
+if (NOT RocksDB_FOUND)
+  FetchContent_Declare(
+    rocksdb
+    GIT_REPOSITORY https://github.com/facebook/rocksdb.git
+    GIT_TAG        f32521662acf3352397d438b732144c7813bbbec # v8.5.3
+    GIT_PROGRESS   TRUE
+  )
+  FetchContent_Populate(rocksdb)
+
+  add_custom_target(librocksdb_build ALL
+    WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
+    COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
+    COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
+    BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
+    COMMENT "Building RocksDB"
+    USES_TERMINAL
+  )
+
+  ### zstd (for rocksdb)
+  find_package(zstd REQUIRED)
+
+  add_library(librocksdb INTERFACE)
+  add_dependencies(librocksdb librocksdb_build)
+  target_include_directories(librocksdb INTERFACE SYSTEM "${rocksdb_SOURCE_DIR}/include")
+  target_link_libraries(librocksdb INTERFACE ${rocksdb_BINARY_DIR}/librocksdb.a zstd::zstd)
+
+  add_library(RocksDB::rocksdb ALIAS librocksdb)
+endif()
 
 ### folly
 ### use folly as a header only library. some features won't be supported.
-FetchContent_Declare(
-  folly
-  GIT_REPOSITORY https://github.com/facebook/folly.git
-  GIT_TAG        c5aa5c46291a27f69acc920894d43605ceb43eba
-  GIT_PROGRESS   TRUE
-  PATCH_COMMAND  ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
-)
-FetchContent_Populate(folly)
+find_package(folly CONFIG)
+if (NOT folly_FOUND)
+  FetchContent_Declare(
+    folly
+    GIT_REPOSITORY    https://github.com/facebook/folly.git
+    GIT_TAG           c5aa5c46291a27f69acc920894d43605ceb43eba
+    GIT_PROGRESS      TRUE
+    PATCH_COMMAND     ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
+  )
+  FetchContent_Populate(folly)
+
+  add_library(folly_headers INTERFACE)
+  target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
+  target_link_libraries(folly_headers INTERFACE Boost::headers)
+else()
+  add_library(folly_headers INTERFACE)
+  target_include_directories(folly_headers SYSTEM INTERFACE ${FOLLY_INCLUDE_DIR})
+endif()
 
 ### range-v3
 FetchContent_Declare(
   range-v3
-  GIT_REPOSITORY https://github.com/ericniebler/range-v3.git
-  GIT_TAG        a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
-  GIT_PROGRESS   TRUE
+  GIT_REPOSITORY    https://github.com/ericniebler/range-v3.git
+  GIT_TAG           a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
 FetchContent_MakeAvailable(range-v3)
 
@@ -124,6 +157,18 @@ endif()
 ### (Re)download submodules
 find_package(Git QUIET)
 
+if (DEFINED drgn_SOURCE_DIR)
+    # drgn's autotools build requires source modification. in case we have a
+    # readonly source (read: nix) we must copy this. do this always to avoid
+    # polluting the source.
+    file(COPY "${drgn_SOURCE_DIR}/" DESTINATION "${FETCHCONTENT_BASE_DIR}/drgn-src" NO_SOURCE_PERMISSIONS)
+    SET(drgn_SOURCE_DIR "${FETCHCONTENT_BASE_DIR}/drgn-src")
+    SET(drgn_BINARY_DIR "${CMAKE_BINARY_DIR}/_deps/drgn-build")
+else()
+    SET(drgn_SOURCE_DIR "${PROJECT_SOURCE_DIR}/extern/drgn")
+    SET(drgn_BINARY_DIR "${drgn_SOURCE_DIR}/build")
+endif()
+
 # TODO: No idea if this huge block is required, just picked from an example. There may be a short-hand.
 if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
 # Update submodules as needed
@@ -145,20 +190,17 @@ if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
         if(NOT GIT_SUBMOD_RESULT EQUAL "0")
             message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules")
         endif()
+
+        if(NOT EXISTS "${drgn_SOURCE_DIR}")
+            message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
+        endif()
     endif()
 endif()
 
-if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn")
-    message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
-endif()
 
 ### Select Python version
 find_program(PYTHON NAMES python3.9 python3)
 
-add_library(folly_headers INTERFACE)
-target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
-target_link_libraries(folly_headers INTERFACE Boost::headers)
-
 ### bison & flex (for oid_parser)
 find_package(BISON 3.5 REQUIRED)
 find_package(FLEX)
@@ -194,8 +236,11 @@ find_package(msgpack REQUIRED CONFIG)
 get_target_property(MSGPACK_INCLUDE_DIRS msgpackc INTERFACE_INCLUDE_DIRECTORIES)
 include_directories(SYSTEM ${MSGPACK_INCLUDE_DIRS})
 
-### zstd (for rocksdb)
-find_package(zstd REQUIRED)
+### drgn/elfutils dependencies
+find_package(BZip2 REQUIRED)
+find_package(OpenMP REQUIRED)
+find_package(LibLZMA REQUIRED)
+find_package(ZLIB REQUIRED)
 
 ### drgn
 # The setup.py script in drgn is really meant to build drgn (python
@@ -214,46 +259,38 @@ find_package(zstd REQUIRED)
 # make
 #
 # Since setup.py has a single cmd to do this, just use it for now.
-#
-# Another extemely annoying point. drgn pretty much has to be compiled with gcc only
-# clang-12 does NOT work. clang fails with the following error :-
-# configure: error: gcc with GNU99 support required
 
 set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no")
 if (ASAN)
   list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes")
 endif()
-add_custom_target(libdrgn ALL
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn
-  COMMAND unset BISON_PKGDATADIR && CC=gcc CFLAGS="${DRGN_CFLAGS}" CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp build
-  BYPRODUCTS ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/.libs/libdrgnimpl.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdw/libdw.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libelf/libelf.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdwelf/libdwelf.a
+
+add_custom_target(drgn_build ALL
+  WORKING_DIRECTORY ${drgn_SOURCE_DIR}
+  COMMAND unset BISON_PKGDATADIR && CFLAGS="${DRGN_CFLAGS}"
+  CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp "${drgn_BINARY_DIR}"
+  BYPRODUCTS ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
+    ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
+    ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
+    ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
   COMMENT "Building drgn"
   USES_TERMINAL
 )
-set(DRGN_PATH "${PROJECT_SOURCE_DIR}/extern/drgn/build")
 
-# Ideally drgn stuff should be together at the end. But looks like rpath needs
-# to be set before add_executable() unfortunately. Maybe split libdrgn stuff
-# into a separate file later.
-set(CMAKE_SKIP_BUILD_RPATH FALSE)
-set(CMAKE_INSTALL_RPATH
-  "${DRGN_PATH}/.libs"
-  "${DRGN_PATH}/velfutils/libdw"
-  "${DRGN_PATH}/velfutils/libelf"
-  "${DRGN_PATH}/velfutils/libdwelf"
-)
-set(CMAKE_BUILD_RPATH
-  "${DRGN_PATH}/.libs"
-  "${DRGN_PATH}/velfutils/libdw"
-  "${DRGN_PATH}/velfutils/libelf"
-  "${DRGN_PATH}/velfutils/libdwelf"
-)
-set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+add_library(drgn INTERFACE)
+add_dependencies(drgn drgn_build)
+target_link_libraries(drgn INTERFACE
+  ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
+  ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
+  ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
+  ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
 
-include_directories(SYSTEM "${DRGN_PATH}")
+  BZip2::BZip2
+  LibLZMA::LibLZMA
+  OpenMP::OpenMP_CXX
+  ZLIB::ZLIB
+)
+target_include_directories(drgn SYSTEM INTERFACE "${drgn_SOURCE_DIR}" "${drgn_BINARY_DIR}")
 
 if (STATIC_LINK)
   # glog links against the `gflags` target, which is an alias for `gflags_shared`
@@ -289,7 +326,6 @@ add_library(oicore
   oi/PaddingHunter.cpp
   oi/Serialize.cpp
 )
-add_dependencies(oicore libdrgn)
 target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -323,7 +359,6 @@ else()
 endif()
 
 target_link_libraries(oicore
-  "-L${DRGN_PATH}/.libs"
   drgn
   dw
   pthread
@@ -334,11 +369,9 @@ add_library(treebuilder
   oi/TreeBuilder.cpp
   oi/exporters/TypeCheckingWalker.cpp
 )
-add_dependencies(treebuilder librocksdb)
 target_link_libraries(treebuilder
-  ${rocksdb_BINARY_DIR}/librocksdb.a
+  RocksDB::rocksdb
   oicore # overkill but it does need a lot of stuff
-  zstd::zstd
 )
 
 
@@ -375,10 +408,8 @@ target_link_libraries(oip oicore)
 
 ### Object Introspection RocksDB Printer (OIRP)
 add_executable(oirp tools/OIRP.cpp)
-add_dependencies(oirp librocksdb)
 target_link_libraries(oirp
-  ${rocksdb_BINARY_DIR}/librocksdb.a
-  zstd::zstd
+  RocksDB::rocksdb
   msgpackc
 )
 
@@ -417,3 +448,6 @@ endif()
 if (DEFINED ENV{CMAKE_HOOK})
   include($ENV{CMAKE_HOOK})
 endif()
+
+install(TARGETS oid DESTINATION ${CMAKE_INSTALL_BINDIR})
+
diff --git a/README.md b/README.md
index 9f95bf8..abce6f8 100644
--- a/README.md
+++ b/README.md
@@ -14,3 +14,61 @@ See the [CONTRIBUTING](CONTRIBUTING.md) file for how to help out.
 
 ## License
 Object Introspection is licensed under the [Apache 2.0 License](LICENSE).
+
+## Getting started with Nix
+
+Nix is the easiest way to get started with `oid` as it is non-trivial to build otherwise. Explicit Nix support for Object Introspection as a Library will come down the line, but Nix can currently provide you a reproducible development environment in which to build it.
+
+These examples expect you to have `nix` installed and available with no other dependencies required. Find the installation guide at https://nixos.org/download.html.
+
+We also required flake support. To enable flakes globally run:
+
+    $ mkdir -p ~/.config/nix
+    $ echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf
+
+Or suffix every `nix` command with `nix --extra-experimental-features 'nix-command flakes'`.
+
+### Run upstream OID without modifying the source
+
+    $ nix run github:facebookexperimental/object-introspection -- --help
+
+This will download the latest source into your Nix store along with all of its dependencies, running help afterwards.
+
+### Build OID locally
+
+    $ git clone https://github.com/facebookexperimental/object-introspection
+    $ nix build
+    $ ./result/bin/oid --help
+
+This will build OID from your local sources. Please note that this will NOT pick up changes to `extern/drgn` or `extern/drgn/libdrgn/velfutils`.
+
+### Get a development environment
+
+    $ nix develop
+    $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+    $ ninja -C build
+    $ build/oid --help
+
+This command provides a development shell with all the required dependencies. This is the most flexible option and will pick up source changes as CMake normally would.
+
+Sometimes this developer environment can be polluted by things installed on your normal system. If this is an issue, use:
+
+    $ nix develop -i
+
+This removes the environment from your host system and makes the build pure.
+
+### Run the tests
+
+    $ nix develop
+    $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+    $ ninja -C build
+    $ ./tools/config_gen.py -c clang++ build/testing.oid.toml
+    $ ctest -j --test-dir build/test
+
+Running tests under `nix` is new to the project and may take some time to mature. The CI is the source of truth for now.
+
+### Format source
+
+    $ nix fmt
+
+This formats the Nix, C++, and Python code in the repository.
diff --git a/cmake/Finduring.cmake b/cmake/Finduring.cmake
new file mode 100644
index 0000000..3b007fe
--- /dev/null
+++ b/cmake/Finduring.cmake
@@ -0,0 +1,27 @@
+# - Find liburing
+#
+# uring_INCLUDE_DIR - Where to find liburing.h
+# uring_LIBRARIES - List of libraries when using uring.
+# uring_FOUND - True if uring found.
+
+find_path(uring_INCLUDE_DIR
+  NAMES liburing.h)
+find_library(uring_LIBRARIES
+  NAMES liburing.a liburing.so)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(uring
+  DEFAULT_MSG uring_LIBRARIES uring_INCLUDE_DIR)
+
+mark_as_advanced(
+  uring_INCLUDE_DIR
+  uring_LIBRARIES)
+
+if(uring_FOUND AND NOT TARGET uring::uring)
+  add_library(uring::uring UNKNOWN IMPORTED)
+  set_target_properties(uring::uring PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${uring_INCLUDE_DIR}"
+    IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+    IMPORTED_LOCATION "${uring_LIBRARIES}")
+endif()
+
diff --git a/flake.nix b/flake.nix
index 7a3382e..1dffbe0 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,5 +1,5 @@
 {
-  description = "A flake for building Object Introspection.";
+  description = "Object level memory profiler for C++";
 
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
@@ -18,7 +18,99 @@
       treefmt-nix,
       ...
     }@inputs:
-    flake-utils.lib.eachDefaultSystem (
+    flake-utils.lib.eachSystem [ flake-utils.lib.system.x86_64-linux ] (
+      system:
+      let
+        pkgs = import nixpkgs { inherit system; };
+
+        drgnSrc = pkgs.fetchFromGitHub {
+          owner = "JakeHillion";
+          repo = "drgn";
+          rev = "b1f8c3e8526611b6720800250ba858a713dd9e4f";
+          hash = "sha256-5WhMHgx/RKtqjxGx4AyiqVKMot5xulr+6c8i2E9IxiA=";
+          fetchSubmodules = true;
+        };
+
+        mkOidPackage =
+          llvmPackages:
+          with pkgs;
+          pkgs.llvmPackages_15.stdenv.mkDerivation rec {
+            name = "oid";
+
+            src = self;
+
+            nativeBuildInputs = [
+              autoconf
+              automake
+              bison
+              cmake
+              flex
+              gettext
+              git
+              hexdump
+              libtool
+              ninja
+              pkgconf
+              python312
+              python312Packages.setuptools
+              python312Packages.toml
+              glibcLocales
+            ];
+
+            buildInputs = [
+              llvmPackages.libclang
+              llvmPackages.llvm
+
+              boost
+              bzip2
+              curl
+              double-conversion
+              elfutils
+              flex
+              folly
+              folly.fmt
+              gflags
+              glog
+              gtest
+              icu
+              jemalloc
+              libarchive
+              libmicrohttpd
+              liburing
+              libxml2
+              lzma
+              msgpack
+              range-v3
+              rocksdb_8_11
+              sqlite
+              tomlplusplus
+              zstd
+
+              llvmPackages_15.openmp # should match the stdenv clang version, see: https://github.com/NixOS/nixpkgs/issues/79818
+            ];
+
+            cmakeFlags = [
+              "-Ddrgn_SOURCE_DIR=${drgnSrc}"
+              "-DFORCE_BOOST_STATIC=Off"
+            ];
+
+            outputs = [ "out" ];
+          };
+      in
+      {
+        packages = rec {
+          default = oid-llvm15;
+
+          oid-llvm15 = mkOidPackage pkgs.llvmPackages_15;
+        };
+
+        apps.default = {
+          type = "app";
+          program = "${self.packages.${system}.default}/bin/oid";
+        };
+      }
+    )
+    // flake-utils.lib.eachDefaultSystem (
       system:
       let
         pkgs = nixpkgs.legacyPackages.${system};
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index bb73871..7d805e5 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus)
 add_library(drgn_utils DrgnUtils.cpp)
 target_link_libraries(drgn_utils
   glog::glog
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
-add_dependencies(drgn_utils libdrgn)
 
 add_library(symbol_service
   Descs.cpp
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 6c4da13..c878e08 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -20,12 +20,9 @@ add_library(type_graph
   TypeIdentifier.cpp
   Types.cpp
 )
-add_dependencies(type_graph libdrgn)
 target_link_libraries(type_graph
   container_info
   symbol_service
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
 target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/tools/config_gen.py b/tools/config_gen.py
index c20ca40..5ac56ff 100755
--- a/tools/config_gen.py
+++ b/tools/config_gen.py
@@ -16,6 +16,7 @@
 
 import argparse
 import getpass
+import os
 import pathlib
 import subprocess
 import typing
@@ -170,7 +171,10 @@ def pull_base_toml() -> typing.Dict:
 
     # Now, we need to replace any placeholders that might be present in the base toml file with the real verisons.
     user = getpass.getuser()
-    pwd = str(repo_path.resolve())
+    if "IN_NIX_SHELL" in os.environ and "src" in os.environ:
+        pwd = os.environ['src']
+    else:
+        pwd = str(repo_path.resolve())
 
     container_list = base.get("types", {}).get("containers")
     if container_list:

From e02df2c6dc14a52de4926fb1fef9c841d1f7c7b8 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 13 Aug 2024 13:28:02 +0100
Subject: [PATCH 182/188] nix: add building oid to the flake

---
 .circleci/config.yml                       |  18 +-
 .github/workflows/object-introspection.yml |  24 ++-
 CMakeLists.txt                             | 230 ++++++++++++---------
 README.md                                  |  58 ++++++
 cmake/Finduring.cmake                      |  27 +++
 flake.nix                                  |  96 ++++++++-
 oi/CMakeLists.txt                          |   3 -
 oi/type_graph/CMakeLists.txt               |   3 -
 tools/config_gen.py                        |   6 +-
 9 files changed, 351 insertions(+), 114 deletions(-)
 create mode 100644 cmake/Finduring.cmake

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 589245a..dba98d7 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -83,20 +83,26 @@ jobs:
               libgtest-dev \
               libjemalloc-dev \
               libmsgpack-dev \
+              libomp-12-dev \
+              liburing-dev \
               libzstd-dev \
               llvm-15-dev \
               ninja-build \
               pkg-config \
               python3-setuptools
             pip3 install toml
+
+            # Ubuntu 22.04 CMake is too old and we don't have a newer image yet
+            git clone --depth 1 --branch v3.30.2 https://github.com/Kitware/CMake.git /tmp/cmake
+            (cd /tmp/cmake && cmake -B build/ -G Ninja && cmake --build build/)
           environment:
             DEBIAN_FRONTEND: noninteractive
       - checkout
       - run:
           name: Build
           command: |
-            cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
-            cmake --build build/
+            /tmp/cmake/build/bin/cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
+            ninja -C build/
             # Testing rubbish:
             cp test/ci.oid.toml build/testing.oid.toml
       - persist_to_workspace:
@@ -131,10 +137,12 @@ jobs:
             sudo apt-get install -y \
               clang-15 \
               libboost-all-dev \
-              libgflags-dev \
-              llvm-15-dev \
               libfmt-dev \
-              libjemalloc-dev
+              libgflags-dev \
+              libgoogle-glog-dev \
+              libjemalloc-dev \
+              libomp-12-dev \
+              llvm-15-dev
           environment:
             DEBIAN_FRONTEND: noninteractive
       - run:
diff --git a/.github/workflows/object-introspection.yml b/.github/workflows/object-introspection.yml
index 70b41aa..9b3f371 100644
--- a/.github/workflows/object-introspection.yml
+++ b/.github/workflows/object-introspection.yml
@@ -5,11 +5,31 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: cachix/install-nix-action@v25
+    - uses: actions/checkout@v4.1.7
+    - uses: cachix/install-nix-action@v27
       with:
         github_access_token: ${{ secrets.GITHUB_TOKEN }}
     - name: nix fmt
       run: |-
         nix --experimental-features 'nix-command flakes' fmt
         git diff --exit-code
+
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        llvm_version: [15]
+    steps:
+    - uses: actions/checkout@v4.1.7
+    - uses: cachix/install-nix-action@v27
+      with:
+        github_access_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: build (LLVM ${{ matrix.llvm_version }})
+      # Run the build manually in `nix develop` to keep non-outputs around
+      run: |
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ninja -C build
+    - name: test (LLVM ${{ matrix.llvm_version }})
+      run: |
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ./tools/config_gen.py -c clang++ build/testing.oid.toml
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ctest -j --test-dir source/build/test/
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8674d46..426b68c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,5 +1,5 @@
 # object-introspection
-cmake_minimum_required(VERSION 3.20)
+cmake_minimum_required(VERSION 3.24)
 project(object-introspection)
 
 # Lets find_program() locate SETUID binaries
@@ -33,75 +33,108 @@ find_package(gflags REQUIRED)
 ### tomlplusplus (for configuration files)
 FetchContent_Declare(
   tomlplusplus
-  GIT_REPOSITORY https://github.com/marzer/tomlplusplus.git
-  GIT_TAG        4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
-  GIT_PROGRESS   TRUE
+  GIT_REPOSITORY    https://github.com/marzer/tomlplusplus.git
+  GIT_TAG           4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
 FetchContent_MakeAvailable(tomlplusplus)
 
 ### glog
-FetchContent_Declare(
-  glog
-  GIT_REPOSITORY https://github.com/google/glog.git
-  GIT_TAG        96a2f23dca4cc7180821ca5f32e526314395d26a
-  GIT_PROGRESS   TRUE
-)
-FetchContent_MakeAvailable(glog)
+find_package(glog)
+if (NOT glog_FOUND)
+  FetchContent_Declare(
+    glog
+    GIT_REPOSITORY    https://github.com/google/glog.git
+    GIT_TAG           96a2f23dca4cc7180821ca5f32e526314395d26a
+    GIT_PROGRESS      TRUE
+    FIND_PACKAGE_ARGS
+  )
+  FetchContent_MakeAvailable(glog)
 
-# These glog executable targets still generate warnings - disable warnings for
-# them explicitly
-target_compile_options(demangle_unittest PRIVATE "-w")
-target_compile_options(logging_unittest PRIVATE "-w")
-target_compile_options(stl_logging_unittest PRIVATE "-w")
-target_compile_options(symbolize_unittest PRIVATE "-w")
-target_compile_options(utilities_unittest PRIVATE "-w")
+  # These glog executable targets still generate warnings - disable warnings for
+  # them explicitly
+  target_compile_options(demangle_unittest PRIVATE "-w")
+  target_compile_options(logging_unittest PRIVATE "-w")
+  target_compile_options(stl_logging_unittest PRIVATE "-w")
+  target_compile_options(symbolize_unittest PRIVATE "-w")
+  target_compile_options(utilities_unittest PRIVATE "-w")
+endif()
 
-### googletest
+### GTest
 # Do this in the main file so it can be fetched before setting project warnings.
 # After this is fixed with FetchContent, move to test/CMakeLists.txt.
 FetchContent_Declare(
-  googletest
-  GIT_REPOSITORY https://github.com/google/googletest.git
-  GIT_TAG        1ed6a8c67a0bd675149ece27bbec0ef1759854cf
-  GIT_PROGRESS   TRUE
+  GTest
+  GIT_REPOSITORY    https://github.com/google/googletest.git
+  GIT_TAG           1ed6a8c67a0bd675149ece27bbec0ef1759854cf
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
-FetchContent_MakeAvailable(googletest)
+FetchContent_MakeAvailable(GTest)
+
+### liburing (for RocksDB)
+find_package(uring REQUIRED)
 
 ### rocksdb
-FetchContent_Declare(
-  rocksdb
-  GIT_REPOSITORY https://github.com/facebook/rocksdb.git
-  GIT_TAG        f32521662acf3352397d438b732144c7813bbbec # v8.5.3
-  GIT_PROGRESS   TRUE
-)
-FetchContent_Populate(rocksdb)
-add_custom_target(librocksdb ALL
-  WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
-  COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
-  COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
-  BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
-  COMMENT "Building RocksDB"
-  USES_TERMINAL
-)
-include_directories(SYSTEM "${rocksdb_SOURCE_DIR}/include")
+find_package(RocksDB 8.11 CONFIG)
+if (NOT RocksDB_FOUND)
+  FetchContent_Declare(
+    rocksdb
+    GIT_REPOSITORY https://github.com/facebook/rocksdb.git
+    GIT_TAG        f32521662acf3352397d438b732144c7813bbbec # v8.5.3
+    GIT_PROGRESS   TRUE
+  )
+  FetchContent_Populate(rocksdb)
+
+  add_custom_target(librocksdb_build ALL
+    WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
+    COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
+    COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
+    BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
+    COMMENT "Building RocksDB"
+    USES_TERMINAL
+  )
+
+  ### zstd (for rocksdb)
+  find_package(zstd REQUIRED)
+
+  add_library(librocksdb INTERFACE)
+  add_dependencies(librocksdb librocksdb_build)
+  target_include_directories(librocksdb INTERFACE SYSTEM "${rocksdb_SOURCE_DIR}/include")
+  target_link_libraries(librocksdb INTERFACE ${rocksdb_BINARY_DIR}/librocksdb.a zstd::zstd)
+
+  add_library(RocksDB::rocksdb ALIAS librocksdb)
+endif()
 
 ### folly
 ### use folly as a header only library. some features won't be supported.
-FetchContent_Declare(
-  folly
-  GIT_REPOSITORY https://github.com/facebook/folly.git
-  GIT_TAG        c5aa5c46291a27f69acc920894d43605ceb43eba
-  GIT_PROGRESS   TRUE
-  PATCH_COMMAND  ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
-)
-FetchContent_Populate(folly)
+find_package(folly CONFIG)
+if (NOT folly_FOUND)
+  FetchContent_Declare(
+    folly
+    GIT_REPOSITORY    https://github.com/facebook/folly.git
+    GIT_TAG           c5aa5c46291a27f69acc920894d43605ceb43eba
+    GIT_PROGRESS      TRUE
+    PATCH_COMMAND     ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
+  )
+  FetchContent_Populate(folly)
+
+  add_library(folly_headers INTERFACE)
+  target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
+  target_link_libraries(folly_headers INTERFACE Boost::headers)
+else()
+  add_library(folly_headers INTERFACE)
+  target_include_directories(folly_headers SYSTEM INTERFACE ${FOLLY_INCLUDE_DIR})
+endif()
 
 ### range-v3
 FetchContent_Declare(
   range-v3
-  GIT_REPOSITORY https://github.com/ericniebler/range-v3.git
-  GIT_TAG        a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
-  GIT_PROGRESS   TRUE
+  GIT_REPOSITORY    https://github.com/ericniebler/range-v3.git
+  GIT_TAG           a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
 FetchContent_MakeAvailable(range-v3)
 
@@ -124,6 +157,18 @@ endif()
 ### (Re)download submodules
 find_package(Git QUIET)
 
+if (DEFINED drgn_SOURCE_DIR)
+    # drgn's autotools build requires source modification. in case we have a
+    # readonly source (read: nix) we must copy this. do this always to avoid
+    # polluting the source.
+    file(COPY "${drgn_SOURCE_DIR}/" DESTINATION "${FETCHCONTENT_BASE_DIR}/drgn-src" NO_SOURCE_PERMISSIONS)
+    SET(drgn_SOURCE_DIR "${FETCHCONTENT_BASE_DIR}/drgn-src")
+    SET(drgn_BINARY_DIR "${CMAKE_BINARY_DIR}/_deps/drgn-build")
+else()
+    SET(drgn_SOURCE_DIR "${PROJECT_SOURCE_DIR}/extern/drgn")
+    SET(drgn_BINARY_DIR "${drgn_SOURCE_DIR}/build")
+endif()
+
 # TODO: No idea if this huge block is required, just picked from an example. There may be a short-hand.
 if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
 # Update submodules as needed
@@ -145,20 +190,17 @@ if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
         if(NOT GIT_SUBMOD_RESULT EQUAL "0")
             message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules")
         endif()
+
+        if(NOT EXISTS "${drgn_SOURCE_DIR}")
+            message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
+        endif()
     endif()
 endif()
 
-if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn")
-    message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
-endif()
 
 ### Select Python version
 find_program(PYTHON NAMES python3.9 python3)
 
-add_library(folly_headers INTERFACE)
-target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
-target_link_libraries(folly_headers INTERFACE Boost::headers)
-
 ### bison & flex (for oid_parser)
 find_package(BISON 3.5 REQUIRED)
 find_package(FLEX)
@@ -194,8 +236,11 @@ find_package(msgpack REQUIRED CONFIG)
 get_target_property(MSGPACK_INCLUDE_DIRS msgpackc INTERFACE_INCLUDE_DIRECTORIES)
 include_directories(SYSTEM ${MSGPACK_INCLUDE_DIRS})
 
-### zstd (for rocksdb)
-find_package(zstd REQUIRED)
+### drgn/elfutils dependencies
+find_package(BZip2 REQUIRED)
+find_package(OpenMP REQUIRED)
+find_package(LibLZMA REQUIRED)
+find_package(ZLIB REQUIRED)
 
 ### drgn
 # The setup.py script in drgn is really meant to build drgn (python
@@ -214,46 +259,38 @@ find_package(zstd REQUIRED)
 # make
 #
 # Since setup.py has a single cmd to do this, just use it for now.
-#
-# Another extemely annoying point. drgn pretty much has to be compiled with gcc only
-# clang-12 does NOT work. clang fails with the following error :-
-# configure: error: gcc with GNU99 support required
 
 set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no")
 if (ASAN)
   list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes")
 endif()
-add_custom_target(libdrgn ALL
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn
-  COMMAND unset BISON_PKGDATADIR && CC=gcc CFLAGS="${DRGN_CFLAGS}" CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp build
-  BYPRODUCTS ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/.libs/libdrgnimpl.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdw/libdw.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libelf/libelf.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdwelf/libdwelf.a
+
+add_custom_target(drgn_build ALL
+  WORKING_DIRECTORY ${drgn_SOURCE_DIR}
+  COMMAND unset BISON_PKGDATADIR && CFLAGS="${DRGN_CFLAGS}"
+  CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp "${drgn_BINARY_DIR}"
+  BYPRODUCTS ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
+    ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
+    ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
+    ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
   COMMENT "Building drgn"
   USES_TERMINAL
 )
-set(DRGN_PATH "${PROJECT_SOURCE_DIR}/extern/drgn/build")
 
-# Ideally drgn stuff should be together at the end. But looks like rpath needs
-# to be set before add_executable() unfortunately. Maybe split libdrgn stuff
-# into a separate file later.
-set(CMAKE_SKIP_BUILD_RPATH FALSE)
-set(CMAKE_INSTALL_RPATH
-  "${DRGN_PATH}/.libs"
-  "${DRGN_PATH}/velfutils/libdw"
-  "${DRGN_PATH}/velfutils/libelf"
-  "${DRGN_PATH}/velfutils/libdwelf"
-)
-set(CMAKE_BUILD_RPATH
-  "${DRGN_PATH}/.libs"
-  "${DRGN_PATH}/velfutils/libdw"
-  "${DRGN_PATH}/velfutils/libelf"
-  "${DRGN_PATH}/velfutils/libdwelf"
-)
-set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+add_library(drgn INTERFACE)
+add_dependencies(drgn drgn_build)
+target_link_libraries(drgn INTERFACE
+  ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
+  ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
+  ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
+  ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
 
-include_directories(SYSTEM "${DRGN_PATH}")
+  BZip2::BZip2
+  LibLZMA::LibLZMA
+  OpenMP::OpenMP_CXX
+  ZLIB::ZLIB
+)
+target_include_directories(drgn SYSTEM INTERFACE "${drgn_SOURCE_DIR}" "${drgn_BINARY_DIR}")
 
 if (STATIC_LINK)
   # glog links against the `gflags` target, which is an alias for `gflags_shared`
@@ -289,7 +326,6 @@ add_library(oicore
   oi/PaddingHunter.cpp
   oi/Serialize.cpp
 )
-add_dependencies(oicore libdrgn)
 target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -323,7 +359,6 @@ else()
 endif()
 
 target_link_libraries(oicore
-  "-L${DRGN_PATH}/.libs"
   drgn
   dw
   pthread
@@ -334,11 +369,9 @@ add_library(treebuilder
   oi/TreeBuilder.cpp
   oi/exporters/TypeCheckingWalker.cpp
 )
-add_dependencies(treebuilder librocksdb)
 target_link_libraries(treebuilder
-  ${rocksdb_BINARY_DIR}/librocksdb.a
+  RocksDB::rocksdb
   oicore # overkill but it does need a lot of stuff
-  zstd::zstd
 )
 
 
@@ -375,10 +408,8 @@ target_link_libraries(oip oicore)
 
 ### Object Introspection RocksDB Printer (OIRP)
 add_executable(oirp tools/OIRP.cpp)
-add_dependencies(oirp librocksdb)
 target_link_libraries(oirp
-  ${rocksdb_BINARY_DIR}/librocksdb.a
-  zstd::zstd
+  RocksDB::rocksdb
   msgpackc
 )
 
@@ -417,3 +448,6 @@ endif()
 if (DEFINED ENV{CMAKE_HOOK})
   include($ENV{CMAKE_HOOK})
 endif()
+
+install(TARGETS oid DESTINATION ${CMAKE_INSTALL_BINDIR})
+
diff --git a/README.md b/README.md
index 9f95bf8..abce6f8 100644
--- a/README.md
+++ b/README.md
@@ -14,3 +14,61 @@ See the [CONTRIBUTING](CONTRIBUTING.md) file for how to help out.
 
 ## License
 Object Introspection is licensed under the [Apache 2.0 License](LICENSE).
+
+## Getting started with Nix
+
+Nix is the easiest way to get started with `oid` as it is non-trivial to build otherwise. Explicit Nix support for Object Introspection as a Library will come down the line, but Nix can currently provide you a reproducible development environment in which to build it.
+
+These examples expect you to have `nix` installed and available with no other dependencies required. Find the installation guide at https://nixos.org/download.html.
+
+We also required flake support. To enable flakes globally run:
+
+    $ mkdir -p ~/.config/nix
+    $ echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf
+
+Or suffix every `nix` command with `nix --extra-experimental-features 'nix-command flakes'`.
+
+### Run upstream OID without modifying the source
+
+    $ nix run github:facebookexperimental/object-introspection -- --help
+
+This will download the latest source into your Nix store along with all of its dependencies, running help afterwards.
+
+### Build OID locally
+
+    $ git clone https://github.com/facebookexperimental/object-introspection
+    $ nix build
+    $ ./result/bin/oid --help
+
+This will build OID from your local sources. Please note that this will NOT pick up changes to `extern/drgn` or `extern/drgn/libdrgn/velfutils`.
+
+### Get a development environment
+
+    $ nix develop
+    $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+    $ ninja -C build
+    $ build/oid --help
+
+This command provides a development shell with all the required dependencies. This is the most flexible option and will pick up source changes as CMake normally would.
+
+Sometimes this developer environment can be polluted by things installed on your normal system. If this is an issue, use:
+
+    $ nix develop -i
+
+This removes the environment from your host system and makes the build pure.
+
+### Run the tests
+
+    $ nix develop
+    $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+    $ ninja -C build
+    $ ./tools/config_gen.py -c clang++ build/testing.oid.toml
+    $ ctest -j --test-dir build/test
+
+Running tests under `nix` is new to the project and may take some time to mature. The CI is the source of truth for now.
+
+### Format source
+
+    $ nix fmt
+
+This formats the Nix, C++, and Python code in the repository.
diff --git a/cmake/Finduring.cmake b/cmake/Finduring.cmake
new file mode 100644
index 0000000..3b007fe
--- /dev/null
+++ b/cmake/Finduring.cmake
@@ -0,0 +1,27 @@
+# - Find liburing
+#
+# uring_INCLUDE_DIR - Where to find liburing.h
+# uring_LIBRARIES - List of libraries when using uring.
+# uring_FOUND - True if uring found.
+
+find_path(uring_INCLUDE_DIR
+  NAMES liburing.h)
+find_library(uring_LIBRARIES
+  NAMES liburing.a liburing.so)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(uring
+  DEFAULT_MSG uring_LIBRARIES uring_INCLUDE_DIR)
+
+mark_as_advanced(
+  uring_INCLUDE_DIR
+  uring_LIBRARIES)
+
+if(uring_FOUND AND NOT TARGET uring::uring)
+  add_library(uring::uring UNKNOWN IMPORTED)
+  set_target_properties(uring::uring PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${uring_INCLUDE_DIR}"
+    IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+    IMPORTED_LOCATION "${uring_LIBRARIES}")
+endif()
+
diff --git a/flake.nix b/flake.nix
index 7a3382e..1dffbe0 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,5 +1,5 @@
 {
-  description = "A flake for building Object Introspection.";
+  description = "Object level memory profiler for C++";
 
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
@@ -18,7 +18,99 @@
       treefmt-nix,
       ...
     }@inputs:
-    flake-utils.lib.eachDefaultSystem (
+    flake-utils.lib.eachSystem [ flake-utils.lib.system.x86_64-linux ] (
+      system:
+      let
+        pkgs = import nixpkgs { inherit system; };
+
+        drgnSrc = pkgs.fetchFromGitHub {
+          owner = "JakeHillion";
+          repo = "drgn";
+          rev = "b1f8c3e8526611b6720800250ba858a713dd9e4f";
+          hash = "sha256-5WhMHgx/RKtqjxGx4AyiqVKMot5xulr+6c8i2E9IxiA=";
+          fetchSubmodules = true;
+        };
+
+        mkOidPackage =
+          llvmPackages:
+          with pkgs;
+          pkgs.llvmPackages_15.stdenv.mkDerivation rec {
+            name = "oid";
+
+            src = self;
+
+            nativeBuildInputs = [
+              autoconf
+              automake
+              bison
+              cmake
+              flex
+              gettext
+              git
+              hexdump
+              libtool
+              ninja
+              pkgconf
+              python312
+              python312Packages.setuptools
+              python312Packages.toml
+              glibcLocales
+            ];
+
+            buildInputs = [
+              llvmPackages.libclang
+              llvmPackages.llvm
+
+              boost
+              bzip2
+              curl
+              double-conversion
+              elfutils
+              flex
+              folly
+              folly.fmt
+              gflags
+              glog
+              gtest
+              icu
+              jemalloc
+              libarchive
+              libmicrohttpd
+              liburing
+              libxml2
+              lzma
+              msgpack
+              range-v3
+              rocksdb_8_11
+              sqlite
+              tomlplusplus
+              zstd
+
+              llvmPackages_15.openmp # should match the stdenv clang version, see: https://github.com/NixOS/nixpkgs/issues/79818
+            ];
+
+            cmakeFlags = [
+              "-Ddrgn_SOURCE_DIR=${drgnSrc}"
+              "-DFORCE_BOOST_STATIC=Off"
+            ];
+
+            outputs = [ "out" ];
+          };
+      in
+      {
+        packages = rec {
+          default = oid-llvm15;
+
+          oid-llvm15 = mkOidPackage pkgs.llvmPackages_15;
+        };
+
+        apps.default = {
+          type = "app";
+          program = "${self.packages.${system}.default}/bin/oid";
+        };
+      }
+    )
+    // flake-utils.lib.eachDefaultSystem (
       system:
       let
         pkgs = nixpkgs.legacyPackages.${system};
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index bb73871..7d805e5 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus)
 add_library(drgn_utils DrgnUtils.cpp)
 target_link_libraries(drgn_utils
   glog::glog
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
-add_dependencies(drgn_utils libdrgn)
 
 add_library(symbol_service
   Descs.cpp
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 6c4da13..c878e08 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -20,12 +20,9 @@ add_library(type_graph
   TypeIdentifier.cpp
   Types.cpp
 )
-add_dependencies(type_graph libdrgn)
 target_link_libraries(type_graph
   container_info
   symbol_service
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
 target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/tools/config_gen.py b/tools/config_gen.py
index c20ca40..5ac56ff 100755
--- a/tools/config_gen.py
+++ b/tools/config_gen.py
@@ -16,6 +16,7 @@
 
 import argparse
 import getpass
+import os
 import pathlib
 import subprocess
 import typing
@@ -170,7 +171,10 @@ def pull_base_toml() -> typing.Dict:
 
     # Now, we need to replace any placeholders that might be present in the base toml file with the real verisons.
     user = getpass.getuser()
-    pwd = str(repo_path.resolve())
+    if "IN_NIX_SHELL" in os.environ and "src" in os.environ:
+        pwd = os.environ['src']
+    else:
+        pwd = str(repo_path.resolve())
 
     container_list = base.get("types", {}).get("containers")
     if container_list:

From cc6cfcf11fc3c2c538eea8972a257672d02ba791 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 13 Aug 2024 13:28:02 +0100
Subject: [PATCH 183/188] nix: add building oid to the flake

---
 .circleci/config.yml                       |  18 +-
 .github/workflows/object-introspection.yml |  27 ++-
 CMakeLists.txt                             | 230 ++++++++++++---------
 README.md                                  |  58 ++++++
 cmake/Finduring.cmake                      |  27 +++
 flake.nix                                  |  96 ++++++++-
 oi/CMakeLists.txt                          |   3 -
 oi/type_graph/CMakeLists.txt               |   3 -
 tools/config_gen.py                        |   6 +-
 9 files changed, 354 insertions(+), 114 deletions(-)
 create mode 100644 cmake/Finduring.cmake

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 589245a..dba98d7 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -83,20 +83,26 @@ jobs:
               libgtest-dev \
               libjemalloc-dev \
               libmsgpack-dev \
+              libomp-12-dev \
+              liburing-dev \
               libzstd-dev \
               llvm-15-dev \
               ninja-build \
               pkg-config \
               python3-setuptools
             pip3 install toml
+
+            # Ubuntu 22.04 CMake is too old and we don't have a newer image yet
+            git clone --depth 1 --branch v3.30.2 https://github.com/Kitware/CMake.git /tmp/cmake
+            (cd /tmp/cmake && cmake -B build/ -G Ninja && cmake --build build/)
           environment:
             DEBIAN_FRONTEND: noninteractive
       - checkout
       - run:
           name: Build
           command: |
-            cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
-            cmake --build build/
+            /tmp/cmake/build/bin/cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
+            ninja -C build/
             # Testing rubbish:
             cp test/ci.oid.toml build/testing.oid.toml
       - persist_to_workspace:
@@ -131,10 +137,12 @@ jobs:
             sudo apt-get install -y \
               clang-15 \
               libboost-all-dev \
-              libgflags-dev \
-              llvm-15-dev \
               libfmt-dev \
-              libjemalloc-dev
+              libgflags-dev \
+              libgoogle-glog-dev \
+              libjemalloc-dev \
+              libomp-12-dev \
+              llvm-15-dev
           environment:
             DEBIAN_FRONTEND: noninteractive
       - run:
diff --git a/.github/workflows/object-introspection.yml b/.github/workflows/object-introspection.yml
index 70b41aa..feef0b8 100644
--- a/.github/workflows/object-introspection.yml
+++ b/.github/workflows/object-introspection.yml
@@ -5,11 +5,34 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: cachix/install-nix-action@v25
+    - uses: actions/checkout@v4.1.7
+    - uses: cachix/install-nix-action@v27
       with:
         github_access_token: ${{ secrets.GITHUB_TOKEN }}
     - name: nix fmt
       run: |-
         nix --experimental-features 'nix-command flakes' fmt
         git diff --exit-code
+
+  build-test:
+    runs-on: 16-core-ubuntu
+    strategy:
+      matrix:
+        llvm_version: [15]
+    steps:
+    - uses: actions/checkout@v4.1.7
+    - uses: cachix/install-nix-action@v27
+      with:
+        github_access_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: build (LLVM ${{ matrix.llvm_version }})
+      # Run the build manually in `nix develop` to keep non-outputs around
+      run: |
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ninja -C build
+    - name: test (LLVM ${{ matrix.llvm_version }})
+      env:
+        # disable drgn multithreading as tests are already run in parallel
+        OMP_NUM_THREADS: 1
+      run: |
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ./tools/config_gen.py -c clang++ build/testing.oid.toml
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ctest -j --test-dir build/test/
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8674d46..426b68c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,5 +1,5 @@
 # object-introspection
-cmake_minimum_required(VERSION 3.20)
+cmake_minimum_required(VERSION 3.24)
 project(object-introspection)
 
 # Lets find_program() locate SETUID binaries
@@ -33,75 +33,108 @@ find_package(gflags REQUIRED)
 ### tomlplusplus (for configuration files)
 FetchContent_Declare(
   tomlplusplus
-  GIT_REPOSITORY https://github.com/marzer/tomlplusplus.git
-  GIT_TAG        4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
-  GIT_PROGRESS   TRUE
+  GIT_REPOSITORY    https://github.com/marzer/tomlplusplus.git
+  GIT_TAG           4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
 FetchContent_MakeAvailable(tomlplusplus)
 
 ### glog
-FetchContent_Declare(
-  glog
-  GIT_REPOSITORY https://github.com/google/glog.git
-  GIT_TAG        96a2f23dca4cc7180821ca5f32e526314395d26a
-  GIT_PROGRESS   TRUE
-)
-FetchContent_MakeAvailable(glog)
+find_package(glog)
+if (NOT glog_FOUND)
+  FetchContent_Declare(
+    glog
+    GIT_REPOSITORY    https://github.com/google/glog.git
+    GIT_TAG           96a2f23dca4cc7180821ca5f32e526314395d26a
+    GIT_PROGRESS      TRUE
+    FIND_PACKAGE_ARGS
+  )
+  FetchContent_MakeAvailable(glog)
 
-# These glog executable targets still generate warnings - disable warnings for
-# them explicitly
-target_compile_options(demangle_unittest PRIVATE "-w")
-target_compile_options(logging_unittest PRIVATE "-w")
-target_compile_options(stl_logging_unittest PRIVATE "-w")
-target_compile_options(symbolize_unittest PRIVATE "-w")
-target_compile_options(utilities_unittest PRIVATE "-w")
+  # These glog executable targets still generate warnings - disable warnings for
+  # them explicitly
+  target_compile_options(demangle_unittest PRIVATE "-w")
+  target_compile_options(logging_unittest PRIVATE "-w")
+  target_compile_options(stl_logging_unittest PRIVATE "-w")
+  target_compile_options(symbolize_unittest PRIVATE "-w")
+  target_compile_options(utilities_unittest PRIVATE "-w")
+endif()
 
-### googletest
+### GTest
 # Do this in the main file so it can be fetched before setting project warnings.
 # After this is fixed with FetchContent, move to test/CMakeLists.txt.
 FetchContent_Declare(
-  googletest
-  GIT_REPOSITORY https://github.com/google/googletest.git
-  GIT_TAG        1ed6a8c67a0bd675149ece27bbec0ef1759854cf
-  GIT_PROGRESS   TRUE
+  GTest
+  GIT_REPOSITORY    https://github.com/google/googletest.git
+  GIT_TAG           1ed6a8c67a0bd675149ece27bbec0ef1759854cf
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
-FetchContent_MakeAvailable(googletest)
+FetchContent_MakeAvailable(GTest)
+
+### liburing (for RocksDB)
+find_package(uring REQUIRED)
 
 ### rocksdb
-FetchContent_Declare(
-  rocksdb
-  GIT_REPOSITORY https://github.com/facebook/rocksdb.git
-  GIT_TAG        f32521662acf3352397d438b732144c7813bbbec # v8.5.3
-  GIT_PROGRESS   TRUE
-)
-FetchContent_Populate(rocksdb)
-add_custom_target(librocksdb ALL
-  WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
-  COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
-  COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
-  BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
-  COMMENT "Building RocksDB"
-  USES_TERMINAL
-)
-include_directories(SYSTEM "${rocksdb_SOURCE_DIR}/include")
+find_package(RocksDB 8.11 CONFIG)
+if (NOT RocksDB_FOUND)
+  FetchContent_Declare(
+    rocksdb
+    GIT_REPOSITORY https://github.com/facebook/rocksdb.git
+    GIT_TAG        f32521662acf3352397d438b732144c7813bbbec # v8.5.3
+    GIT_PROGRESS   TRUE
+  )
+  FetchContent_Populate(rocksdb)
+
+  add_custom_target(librocksdb_build ALL
+    WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
+    COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
+    COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
+    BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
+    COMMENT "Building RocksDB"
+    USES_TERMINAL
+  )
+
+  ### zstd (for rocksdb)
+  find_package(zstd REQUIRED)
+
+  add_library(librocksdb INTERFACE)
+  add_dependencies(librocksdb librocksdb_build)
+  target_include_directories(librocksdb INTERFACE SYSTEM "${rocksdb_SOURCE_DIR}/include")
+  target_link_libraries(librocksdb INTERFACE ${rocksdb_BINARY_DIR}/librocksdb.a zstd::zstd)
+
+  add_library(RocksDB::rocksdb ALIAS librocksdb)
+endif()
 
 ### folly
 ### use folly as a header only library. some features won't be supported.
-FetchContent_Declare(
-  folly
-  GIT_REPOSITORY https://github.com/facebook/folly.git
-  GIT_TAG        c5aa5c46291a27f69acc920894d43605ceb43eba
-  GIT_PROGRESS   TRUE
-  PATCH_COMMAND  ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
-)
-FetchContent_Populate(folly)
+find_package(folly CONFIG)
+if (NOT folly_FOUND)
+  FetchContent_Declare(
+    folly
+    GIT_REPOSITORY    https://github.com/facebook/folly.git
+    GIT_TAG           c5aa5c46291a27f69acc920894d43605ceb43eba
+    GIT_PROGRESS      TRUE
+    PATCH_COMMAND     ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
+  )
+  FetchContent_Populate(folly)
+
+  add_library(folly_headers INTERFACE)
+  target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
+  target_link_libraries(folly_headers INTERFACE Boost::headers)
+else()
+  add_library(folly_headers INTERFACE)
+  target_include_directories(folly_headers SYSTEM INTERFACE ${FOLLY_INCLUDE_DIR})
+endif()
 
 ### range-v3
 FetchContent_Declare(
   range-v3
-  GIT_REPOSITORY https://github.com/ericniebler/range-v3.git
-  GIT_TAG        a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
-  GIT_PROGRESS   TRUE
+  GIT_REPOSITORY    https://github.com/ericniebler/range-v3.git
+  GIT_TAG           a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
 FetchContent_MakeAvailable(range-v3)
 
@@ -124,6 +157,18 @@ endif()
 ### (Re)download submodules
 find_package(Git QUIET)
 
+if (DEFINED drgn_SOURCE_DIR)
+    # drgn's autotools build requires source modification. in case we have a
+    # readonly source (read: nix) we must copy this. do this always to avoid
+    # polluting the source.
+    file(COPY "${drgn_SOURCE_DIR}/" DESTINATION "${FETCHCONTENT_BASE_DIR}/drgn-src" NO_SOURCE_PERMISSIONS)
+    SET(drgn_SOURCE_DIR "${FETCHCONTENT_BASE_DIR}/drgn-src")
+    SET(drgn_BINARY_DIR "${CMAKE_BINARY_DIR}/_deps/drgn-build")
+else()
+    SET(drgn_SOURCE_DIR "${PROJECT_SOURCE_DIR}/extern/drgn")
+    SET(drgn_BINARY_DIR "${drgn_SOURCE_DIR}/build")
+endif()
+
 # TODO: No idea if this huge block is required, just picked from an example. There may be a short-hand.
 if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
 # Update submodules as needed
@@ -145,20 +190,17 @@ if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
         if(NOT GIT_SUBMOD_RESULT EQUAL "0")
             message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules")
         endif()
+
+        if(NOT EXISTS "${drgn_SOURCE_DIR}")
+            message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
+        endif()
     endif()
 endif()
 
-if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn")
-    message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
-endif()
 
 ### Select Python version
 find_program(PYTHON NAMES python3.9 python3)
 
-add_library(folly_headers INTERFACE)
-target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
-target_link_libraries(folly_headers INTERFACE Boost::headers)
-
 ### bison & flex (for oid_parser)
 find_package(BISON 3.5 REQUIRED)
 find_package(FLEX)
@@ -194,8 +236,11 @@ find_package(msgpack REQUIRED CONFIG)
 get_target_property(MSGPACK_INCLUDE_DIRS msgpackc INTERFACE_INCLUDE_DIRECTORIES)
 include_directories(SYSTEM ${MSGPACK_INCLUDE_DIRS})
 
-### zstd (for rocksdb)
-find_package(zstd REQUIRED)
+### drgn/elfutils dependencies
+find_package(BZip2 REQUIRED)
+find_package(OpenMP REQUIRED)
+find_package(LibLZMA REQUIRED)
+find_package(ZLIB REQUIRED)
 
 ### drgn
 # The setup.py script in drgn is really meant to build drgn (python
@@ -214,46 +259,38 @@ find_package(zstd REQUIRED)
 # make
 #
 # Since setup.py has a single cmd to do this, just use it for now.
-#
-# Another extemely annoying point. drgn pretty much has to be compiled with gcc only
-# clang-12 does NOT work. clang fails with the following error :-
-# configure: error: gcc with GNU99 support required
 
 set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no")
 if (ASAN)
   list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes")
 endif()
-add_custom_target(libdrgn ALL
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn
-  COMMAND unset BISON_PKGDATADIR && CC=gcc CFLAGS="${DRGN_CFLAGS}" CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp build
-  BYPRODUCTS ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/.libs/libdrgnimpl.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdw/libdw.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libelf/libelf.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdwelf/libdwelf.a
+
+add_custom_target(drgn_build ALL
+  WORKING_DIRECTORY ${drgn_SOURCE_DIR}
+  COMMAND unset BISON_PKGDATADIR && CFLAGS="${DRGN_CFLAGS}"
+  CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp "${drgn_BINARY_DIR}"
+  BYPRODUCTS ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
+    ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
+    ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
+    ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
   COMMENT "Building drgn"
   USES_TERMINAL
 )
-set(DRGN_PATH "${PROJECT_SOURCE_DIR}/extern/drgn/build")
 
-# Ideally drgn stuff should be together at the end. But looks like rpath needs
-# to be set before add_executable() unfortunately. Maybe split libdrgn stuff
-# into a separate file later.
-set(CMAKE_SKIP_BUILD_RPATH FALSE)
-set(CMAKE_INSTALL_RPATH
-  "${DRGN_PATH}/.libs"
-  "${DRGN_PATH}/velfutils/libdw"
-  "${DRGN_PATH}/velfutils/libelf"
-  "${DRGN_PATH}/velfutils/libdwelf"
-)
-set(CMAKE_BUILD_RPATH
-  "${DRGN_PATH}/.libs"
-  "${DRGN_PATH}/velfutils/libdw"
-  "${DRGN_PATH}/velfutils/libelf"
-  "${DRGN_PATH}/velfutils/libdwelf"
-)
-set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+add_library(drgn INTERFACE)
+add_dependencies(drgn drgn_build)
+target_link_libraries(drgn INTERFACE
+  ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
+  ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
+  ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
+  ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
 
-include_directories(SYSTEM "${DRGN_PATH}")
+  BZip2::BZip2
+  LibLZMA::LibLZMA
+  OpenMP::OpenMP_CXX
+  ZLIB::ZLIB
+)
+target_include_directories(drgn SYSTEM INTERFACE "${drgn_SOURCE_DIR}" "${drgn_BINARY_DIR}")
 
 if (STATIC_LINK)
   # glog links against the `gflags` target, which is an alias for `gflags_shared`
@@ -289,7 +326,6 @@ add_library(oicore
   oi/PaddingHunter.cpp
   oi/Serialize.cpp
 )
-add_dependencies(oicore libdrgn)
 target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -323,7 +359,6 @@ else()
 endif()
 
 target_link_libraries(oicore
-  "-L${DRGN_PATH}/.libs"
   drgn
   dw
   pthread
@@ -334,11 +369,9 @@ add_library(treebuilder
   oi/TreeBuilder.cpp
   oi/exporters/TypeCheckingWalker.cpp
 )
-add_dependencies(treebuilder librocksdb)
 target_link_libraries(treebuilder
-  ${rocksdb_BINARY_DIR}/librocksdb.a
+  RocksDB::rocksdb
   oicore # overkill but it does need a lot of stuff
-  zstd::zstd
 )
 
 
@@ -375,10 +408,8 @@ target_link_libraries(oip oicore)
 
 ### Object Introspection RocksDB Printer (OIRP)
 add_executable(oirp tools/OIRP.cpp)
-add_dependencies(oirp librocksdb)
 target_link_libraries(oirp
-  ${rocksdb_BINARY_DIR}/librocksdb.a
-  zstd::zstd
+  RocksDB::rocksdb
   msgpackc
 )
 
@@ -417,3 +448,6 @@ endif()
 if (DEFINED ENV{CMAKE_HOOK})
   include($ENV{CMAKE_HOOK})
 endif()
+
+install(TARGETS oid DESTINATION ${CMAKE_INSTALL_BINDIR})
+
diff --git a/README.md b/README.md
index 9f95bf8..abce6f8 100644
--- a/README.md
+++ b/README.md
@@ -14,3 +14,61 @@ See the [CONTRIBUTING](CONTRIBUTING.md) file for how to help out.
 
 ## License
 Object Introspection is licensed under the [Apache 2.0 License](LICENSE).
+
+## Getting started with Nix
+
+Nix is the easiest way to get started with `oid` as it is non-trivial to build otherwise. Explicit Nix support for Object Introspection as a Library will come down the line, but Nix can currently provide you a reproducible development environment in which to build it.
+
+These examples expect you to have `nix` installed and available with no other dependencies required. Find the installation guide at https://nixos.org/download.html.
+
+We also required flake support. To enable flakes globally run:
+
+    $ mkdir -p ~/.config/nix
+    $ echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf
+
+Or suffix every `nix` command with `nix --extra-experimental-features 'nix-command flakes'`.
+
+### Run upstream OID without modifying the source
+
+    $ nix run github:facebookexperimental/object-introspection -- --help
+
+This will download the latest source into your Nix store along with all of its dependencies, running help afterwards.
+
+### Build OID locally
+
+    $ git clone https://github.com/facebookexperimental/object-introspection
+    $ nix build
+    $ ./result/bin/oid --help
+
+This will build OID from your local sources. Please note that this will NOT pick up changes to `extern/drgn` or `extern/drgn/libdrgn/velfutils`.
+
+### Get a development environment
+
+    $ nix develop
+    $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+    $ ninja -C build
+    $ build/oid --help
+
+This command provides a development shell with all the required dependencies. This is the most flexible option and will pick up source changes as CMake normally would.
+
+Sometimes this developer environment can be polluted by things installed on your normal system. If this is an issue, use:
+
+    $ nix develop -i
+
+This removes the environment from your host system and makes the build pure.
+
+### Run the tests
+
+    $ nix develop
+    $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+    $ ninja -C build
+    $ ./tools/config_gen.py -c clang++ build/testing.oid.toml
+    $ ctest -j --test-dir build/test
+
+Running tests under `nix` is new to the project and may take some time to mature. The CI is the source of truth for now.
+
+### Format source
+
+    $ nix fmt
+
+This formats the Nix, C++, and Python code in the repository.
diff --git a/cmake/Finduring.cmake b/cmake/Finduring.cmake
new file mode 100644
index 0000000..3b007fe
--- /dev/null
+++ b/cmake/Finduring.cmake
@@ -0,0 +1,27 @@
+# - Find liburing
+#
+# uring_INCLUDE_DIR - Where to find liburing.h
+# uring_LIBRARIES - List of libraries when using uring.
+# uring_FOUND - True if uring found.
+
+find_path(uring_INCLUDE_DIR
+  NAMES liburing.h)
+find_library(uring_LIBRARIES
+  NAMES liburing.a liburing.so)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(uring
+  DEFAULT_MSG uring_LIBRARIES uring_INCLUDE_DIR)
+
+mark_as_advanced(
+  uring_INCLUDE_DIR
+  uring_LIBRARIES)
+
+if(uring_FOUND AND NOT TARGET uring::uring)
+  add_library(uring::uring UNKNOWN IMPORTED)
+  set_target_properties(uring::uring PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${uring_INCLUDE_DIR}"
+    IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+    IMPORTED_LOCATION "${uring_LIBRARIES}")
+endif()
+
diff --git a/flake.nix b/flake.nix
index 7a3382e..1dffbe0 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,5 +1,5 @@
 {
-  description = "A flake for building Object Introspection.";
+  description = "Object level memory profiler for C++";
 
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
@@ -18,7 +18,99 @@
       treefmt-nix,
       ...
     }@inputs:
-    flake-utils.lib.eachDefaultSystem (
+    flake-utils.lib.eachSystem [ flake-utils.lib.system.x86_64-linux ] (
+      system:
+      let
+        pkgs = import nixpkgs { inherit system; };
+
+        drgnSrc = pkgs.fetchFromGitHub {
+          owner = "JakeHillion";
+          repo = "drgn";
+          rev = "b1f8c3e8526611b6720800250ba858a713dd9e4f";
+          hash = "sha256-5WhMHgx/RKtqjxGx4AyiqVKMot5xulr+6c8i2E9IxiA=";
+          fetchSubmodules = true;
+        };
+
+        mkOidPackage =
+          llvmPackages:
+          with pkgs;
+          pkgs.llvmPackages_15.stdenv.mkDerivation rec {
+            name = "oid";
+
+            src = self;
+
+            nativeBuildInputs = [
+              autoconf
+              automake
+              bison
+              cmake
+              flex
+              gettext
+              git
+              hexdump
+              libtool
+              ninja
+              pkgconf
+              python312
+              python312Packages.setuptools
+              python312Packages.toml
+              glibcLocales
+            ];
+
+            buildInputs = [
+              llvmPackages.libclang
+              llvmPackages.llvm
+
+              boost
+              bzip2
+              curl
+              double-conversion
+              elfutils
+              flex
+              folly
+              folly.fmt
+              gflags
+              glog
+              gtest
+              icu
+              jemalloc
+              libarchive
+              libmicrohttpd
+              liburing
+              libxml2
+              lzma
+              msgpack
+              range-v3
+              rocksdb_8_11
+              sqlite
+              tomlplusplus
+              zstd
+
+              llvmPackages_15.openmp # should match the stdenv clang version, see: https://github.com/NixOS/nixpkgs/issues/79818
+            ];
+
+            cmakeFlags = [
+              "-Ddrgn_SOURCE_DIR=${drgnSrc}"
+              "-DFORCE_BOOST_STATIC=Off"
+            ];
+
+            outputs = [ "out" ];
+          };
+      in
+      {
+        packages = rec {
+          default = oid-llvm15;
+
+          oid-llvm15 = mkOidPackage pkgs.llvmPackages_15;
+        };
+
+        apps.default = {
+          type = "app";
+          program = "${self.packages.${system}.default}/bin/oid";
+        };
+      }
+    )
+    // flake-utils.lib.eachDefaultSystem (
       system:
       let
         pkgs = nixpkgs.legacyPackages.${system};
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index bb73871..7d805e5 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus)
 add_library(drgn_utils DrgnUtils.cpp)
 target_link_libraries(drgn_utils
   glog::glog
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
-add_dependencies(drgn_utils libdrgn)
 
 add_library(symbol_service
   Descs.cpp
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 6c4da13..c878e08 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -20,12 +20,9 @@ add_library(type_graph
   TypeIdentifier.cpp
   Types.cpp
 )
-add_dependencies(type_graph libdrgn)
 target_link_libraries(type_graph
   container_info
   symbol_service
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
 target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/tools/config_gen.py b/tools/config_gen.py
index c20ca40..5ac56ff 100755
--- a/tools/config_gen.py
+++ b/tools/config_gen.py
@@ -16,6 +16,7 @@
 
 import argparse
 import getpass
+import os
 import pathlib
 import subprocess
 import typing
@@ -170,7 +171,10 @@ def pull_base_toml() -> typing.Dict:
 
     # Now, we need to replace any placeholders that might be present in the base toml file with the real verisons.
     user = getpass.getuser()
-    pwd = str(repo_path.resolve())
+    if "IN_NIX_SHELL" in os.environ and "src" in os.environ:
+        pwd = os.environ['src']
+    else:
+        pwd = str(repo_path.resolve())
 
     container_list = base.get("types", {}).get("containers")
     if container_list:

From f4cdd68d0f4084665f2a025f327aa41d7c986550 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Tue, 13 Aug 2024 13:28:02 +0100
Subject: [PATCH 184/188] nix: add building oid to the flake

---
 .circleci/config.yml                       |  18 +-
 .github/workflows/ci.yml                   |  53 +++++
 .github/workflows/object-introspection.yml |  15 --
 .github/workflows/test-report.xml          |  20 ++
 CMakeLists.txt                             | 230 ++++++++++++---------
 README.md                                  |  58 ++++++
 cmake/Finduring.cmake                      |  27 +++
 flake.nix                                  |  96 ++++++++-
 oi/CMakeLists.txt                          |   3 -
 oi/type_graph/CMakeLists.txt               |   3 -
 tools/config_gen.py                        |   6 +-
 11 files changed, 402 insertions(+), 127 deletions(-)
 create mode 100644 .github/workflows/ci.yml
 delete mode 100644 .github/workflows/object-introspection.yml
 create mode 100644 .github/workflows/test-report.xml
 create mode 100644 cmake/Finduring.cmake

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 589245a..dba98d7 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -83,20 +83,26 @@ jobs:
               libgtest-dev \
               libjemalloc-dev \
               libmsgpack-dev \
+              libomp-12-dev \
+              liburing-dev \
               libzstd-dev \
               llvm-15-dev \
               ninja-build \
               pkg-config \
               python3-setuptools
             pip3 install toml
+
+            # Ubuntu 22.04 CMake is too old and we don't have a newer image yet
+            git clone --depth 1 --branch v3.30.2 https://github.com/Kitware/CMake.git /tmp/cmake
+            (cd /tmp/cmake && cmake -B build/ -G Ninja && cmake --build build/)
           environment:
             DEBIAN_FRONTEND: noninteractive
       - checkout
       - run:
           name: Build
           command: |
-            cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
-            cmake --build build/
+            /tmp/cmake/build/bin/cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
+            ninja -C build/
             # Testing rubbish:
             cp test/ci.oid.toml build/testing.oid.toml
       - persist_to_workspace:
@@ -131,10 +137,12 @@ jobs:
             sudo apt-get install -y \
               clang-15 \
               libboost-all-dev \
-              libgflags-dev \
-              llvm-15-dev \
               libfmt-dev \
-              libjemalloc-dev
+              libgflags-dev \
+              libgoogle-glog-dev \
+              libjemalloc-dev \
+              libomp-12-dev \
+              llvm-15-dev
           environment:
             DEBIAN_FRONTEND: noninteractive
       - run:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..0aa8bbf
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,53 @@
+name: CI
+on:
+  pull_request:
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.7
+    - uses: cachix/install-nix-action@v27
+      with:
+        github_access_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: nix fmt
+      run: |-
+        nix --experimental-features 'nix-command flakes' fmt
+        git diff --exit-code
+
+  build-test:
+    runs-on: 16-core-ubuntu
+    strategy:
+      matrix:
+        llvm_version: [15]
+    steps:
+    - uses: actions/checkout@v4.1.7
+    - uses: cachix/install-nix-action@v27
+      with:
+        github_access_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: build (LLVM ${{ matrix.llvm_version }})
+      # Run the build manually in `nix develop` to keep non-outputs around
+      run: |
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ninja -C build
+    - name: test (LLVM ${{ matrix.llvm_version }})
+      env:
+        # disable drgn multithreading as tests are already run in parallel
+        OMP_NUM_THREADS: 1
+      run: |
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ./tools/config_gen.py -c clang++ build/testing.oid.toml
+      nix develop .#oid-llvm${{ matrix.llvm_version }} --command ctest \
+          --test-dir build/test/ \
+          --test-action Test \
+          --parallel \
+          --no-compress-output \
+          --output-on-failure \
+          --schedule-random \
+          --timeout 60 \
+          --repeat until-pass:2 \
+          --output-junit results.xml
+    - name: upload results
+      uses: actions/upload-artifact@v4
+      if: success() || failure()
+      with:
+        name: test-results
+        path: build/test/results.xml
diff --git a/.github/workflows/object-introspection.yml b/.github/workflows/object-introspection.yml
deleted file mode 100644
index 70b41aa..0000000
--- a/.github/workflows/object-introspection.yml
+++ /dev/null
@@ -1,15 +0,0 @@
-name: facebookexperimental/object-introspection
-on:
-  pull_request:
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: cachix/install-nix-action@v25
-      with:
-        github_access_token: ${{ secrets.GITHUB_TOKEN }}
-    - name: nix fmt
-      run: |-
-        nix --experimental-features 'nix-command flakes' fmt
-        git diff --exit-code
diff --git a/.github/workflows/test-report.xml b/.github/workflows/test-report.xml
new file mode 100644
index 0000000..f8b62b9
--- /dev/null
+++ b/.github/workflows/test-report.xml
@@ -0,0 +1,20 @@
+name: 'Test Report'
+on:
+  workflow_run:
+    workflows: ['CI']
+    types:
+      - completed
+permissions:
+  contents: read
+  actions: read
+  checks: write
+jobs:
+  report:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: dorny/test-reporter@v1
+      with:
+        artifact: test-results
+        name: CTest Tests
+        path: 'build/test/*.xml'
+        reporter: jest-junit
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8674d46..426b68c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,5 +1,5 @@
 # object-introspection
-cmake_minimum_required(VERSION 3.20)
+cmake_minimum_required(VERSION 3.24)
 project(object-introspection)
 
 # Lets find_program() locate SETUID binaries
@@ -33,75 +33,108 @@ find_package(gflags REQUIRED)
 ### tomlplusplus (for configuration files)
 FetchContent_Declare(
   tomlplusplus
-  GIT_REPOSITORY https://github.com/marzer/tomlplusplus.git
-  GIT_TAG        4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
-  GIT_PROGRESS   TRUE
+  GIT_REPOSITORY    https://github.com/marzer/tomlplusplus.git
+  GIT_TAG           4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
 FetchContent_MakeAvailable(tomlplusplus)
 
 ### glog
-FetchContent_Declare(
-  glog
-  GIT_REPOSITORY https://github.com/google/glog.git
-  GIT_TAG        96a2f23dca4cc7180821ca5f32e526314395d26a
-  GIT_PROGRESS   TRUE
-)
-FetchContent_MakeAvailable(glog)
+find_package(glog)
+if (NOT glog_FOUND)
+  FetchContent_Declare(
+    glog
+    GIT_REPOSITORY    https://github.com/google/glog.git
+    GIT_TAG           96a2f23dca4cc7180821ca5f32e526314395d26a
+    GIT_PROGRESS      TRUE
+    FIND_PACKAGE_ARGS
+  )
+  FetchContent_MakeAvailable(glog)
 
-# These glog executable targets still generate warnings - disable warnings for
-# them explicitly
-target_compile_options(demangle_unittest PRIVATE "-w")
-target_compile_options(logging_unittest PRIVATE "-w")
-target_compile_options(stl_logging_unittest PRIVATE "-w")
-target_compile_options(symbolize_unittest PRIVATE "-w")
-target_compile_options(utilities_unittest PRIVATE "-w")
+  # These glog executable targets still generate warnings - disable warnings for
+  # them explicitly
+  target_compile_options(demangle_unittest PRIVATE "-w")
+  target_compile_options(logging_unittest PRIVATE "-w")
+  target_compile_options(stl_logging_unittest PRIVATE "-w")
+  target_compile_options(symbolize_unittest PRIVATE "-w")
+  target_compile_options(utilities_unittest PRIVATE "-w")
+endif()
 
-### googletest
+### GTest
 # Do this in the main file so it can be fetched before setting project warnings.
 # After this is fixed with FetchContent, move to test/CMakeLists.txt.
 FetchContent_Declare(
-  googletest
-  GIT_REPOSITORY https://github.com/google/googletest.git
-  GIT_TAG        1ed6a8c67a0bd675149ece27bbec0ef1759854cf
-  GIT_PROGRESS   TRUE
+  GTest
+  GIT_REPOSITORY    https://github.com/google/googletest.git
+  GIT_TAG           1ed6a8c67a0bd675149ece27bbec0ef1759854cf
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
-FetchContent_MakeAvailable(googletest)
+FetchContent_MakeAvailable(GTest)
+
+### liburing (for RocksDB)
+find_package(uring REQUIRED)
 
 ### rocksdb
-FetchContent_Declare(
-  rocksdb
-  GIT_REPOSITORY https://github.com/facebook/rocksdb.git
-  GIT_TAG        f32521662acf3352397d438b732144c7813bbbec # v8.5.3
-  GIT_PROGRESS   TRUE
-)
-FetchContent_Populate(rocksdb)
-add_custom_target(librocksdb ALL
-  WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
-  COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
-  COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
-  BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
-  COMMENT "Building RocksDB"
-  USES_TERMINAL
-)
-include_directories(SYSTEM "${rocksdb_SOURCE_DIR}/include")
+find_package(RocksDB 8.11 CONFIG)
+if (NOT RocksDB_FOUND)
+  FetchContent_Declare(
+    rocksdb
+    GIT_REPOSITORY https://github.com/facebook/rocksdb.git
+    GIT_TAG        f32521662acf3352397d438b732144c7813bbbec # v8.5.3
+    GIT_PROGRESS   TRUE
+  )
+  FetchContent_Populate(rocksdb)
+
+  add_custom_target(librocksdb_build ALL
+    WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
+    COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
+    COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
+    BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
+    COMMENT "Building RocksDB"
+    USES_TERMINAL
+  )
+
+  ### zstd (for rocksdb)
+  find_package(zstd REQUIRED)
+
+  add_library(librocksdb INTERFACE)
+  add_dependencies(librocksdb librocksdb_build)
+  target_include_directories(librocksdb INTERFACE SYSTEM "${rocksdb_SOURCE_DIR}/include")
+  target_link_libraries(librocksdb INTERFACE ${rocksdb_BINARY_DIR}/librocksdb.a zstd::zstd)
+
+  add_library(RocksDB::rocksdb ALIAS librocksdb)
+endif()
 
 ### folly
 ### use folly as a header only library. some features won't be supported.
-FetchContent_Declare(
-  folly
-  GIT_REPOSITORY https://github.com/facebook/folly.git
-  GIT_TAG        c5aa5c46291a27f69acc920894d43605ceb43eba
-  GIT_PROGRESS   TRUE
-  PATCH_COMMAND  ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
-)
-FetchContent_Populate(folly)
+find_package(folly CONFIG)
+if (NOT folly_FOUND)
+  FetchContent_Declare(
+    folly
+    GIT_REPOSITORY    https://github.com/facebook/folly.git
+    GIT_TAG           c5aa5c46291a27f69acc920894d43605ceb43eba
+    GIT_PROGRESS      TRUE
+    PATCH_COMMAND     ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
+  )
+  FetchContent_Populate(folly)
+
+  add_library(folly_headers INTERFACE)
+  target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
+  target_link_libraries(folly_headers INTERFACE Boost::headers)
+else()
+  add_library(folly_headers INTERFACE)
+  target_include_directories(folly_headers SYSTEM INTERFACE ${FOLLY_INCLUDE_DIR})
+endif()
 
 ### range-v3
 FetchContent_Declare(
   range-v3
-  GIT_REPOSITORY https://github.com/ericniebler/range-v3.git
-  GIT_TAG        a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
-  GIT_PROGRESS   TRUE
+  GIT_REPOSITORY    https://github.com/ericniebler/range-v3.git
+  GIT_TAG           a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
 FetchContent_MakeAvailable(range-v3)
 
@@ -124,6 +157,18 @@ endif()
 ### (Re)download submodules
 find_package(Git QUIET)
 
+if (DEFINED drgn_SOURCE_DIR)
+    # drgn's autotools build requires source modification. in case we have a
+    # readonly source (read: nix) we must copy this. do this always to avoid
+    # polluting the source.
+    file(COPY "${drgn_SOURCE_DIR}/" DESTINATION "${FETCHCONTENT_BASE_DIR}/drgn-src" NO_SOURCE_PERMISSIONS)
+    SET(drgn_SOURCE_DIR "${FETCHCONTENT_BASE_DIR}/drgn-src")
+    SET(drgn_BINARY_DIR "${CMAKE_BINARY_DIR}/_deps/drgn-build")
+else()
+    SET(drgn_SOURCE_DIR "${PROJECT_SOURCE_DIR}/extern/drgn")
+    SET(drgn_BINARY_DIR "${drgn_SOURCE_DIR}/build")
+endif()
+
 # TODO: No idea if this huge block is required, just picked from an example. There may be a short-hand.
 if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
 # Update submodules as needed
@@ -145,20 +190,17 @@ if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
         if(NOT GIT_SUBMOD_RESULT EQUAL "0")
             message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules")
         endif()
+
+        if(NOT EXISTS "${drgn_SOURCE_DIR}")
+            message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
+        endif()
     endif()
 endif()
 
-if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn")
-    message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
-endif()
 
 ### Select Python version
 find_program(PYTHON NAMES python3.9 python3)
 
-add_library(folly_headers INTERFACE)
-target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
-target_link_libraries(folly_headers INTERFACE Boost::headers)
-
 ### bison & flex (for oid_parser)
 find_package(BISON 3.5 REQUIRED)
 find_package(FLEX)
@@ -194,8 +236,11 @@ find_package(msgpack REQUIRED CONFIG)
 get_target_property(MSGPACK_INCLUDE_DIRS msgpackc INTERFACE_INCLUDE_DIRECTORIES)
 include_directories(SYSTEM ${MSGPACK_INCLUDE_DIRS})
 
-### zstd (for rocksdb)
-find_package(zstd REQUIRED)
+### drgn/elfutils dependencies
+find_package(BZip2 REQUIRED)
+find_package(OpenMP REQUIRED)
+find_package(LibLZMA REQUIRED)
+find_package(ZLIB REQUIRED)
 
 ### drgn
 # The setup.py script in drgn is really meant to build drgn (python
@@ -214,46 +259,38 @@ find_package(zstd REQUIRED)
 # make
 #
 # Since setup.py has a single cmd to do this, just use it for now.
-#
-# Another extemely annoying point. drgn pretty much has to be compiled with gcc only
-# clang-12 does NOT work. clang fails with the following error :-
-# configure: error: gcc with GNU99 support required
 
 set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no")
 if (ASAN)
   list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes")
 endif()
-add_custom_target(libdrgn ALL
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn
-  COMMAND unset BISON_PKGDATADIR && CC=gcc CFLAGS="${DRGN_CFLAGS}" CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp build
-  BYPRODUCTS ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/.libs/libdrgnimpl.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdw/libdw.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libelf/libelf.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdwelf/libdwelf.a
+
+add_custom_target(drgn_build ALL
+  WORKING_DIRECTORY ${drgn_SOURCE_DIR}
+  COMMAND unset BISON_PKGDATADIR && CFLAGS="${DRGN_CFLAGS}"
+  CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp "${drgn_BINARY_DIR}"
+  BYPRODUCTS ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
+    ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
+    ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
+    ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
   COMMENT "Building drgn"
   USES_TERMINAL
 )
-set(DRGN_PATH "${PROJECT_SOURCE_DIR}/extern/drgn/build")
 
-# Ideally drgn stuff should be together at the end. But looks like rpath needs
-# to be set before add_executable() unfortunately. Maybe split libdrgn stuff
-# into a separate file later.
-set(CMAKE_SKIP_BUILD_RPATH FALSE)
-set(CMAKE_INSTALL_RPATH
-  "${DRGN_PATH}/.libs"
-  "${DRGN_PATH}/velfutils/libdw"
-  "${DRGN_PATH}/velfutils/libelf"
-  "${DRGN_PATH}/velfutils/libdwelf"
-)
-set(CMAKE_BUILD_RPATH
-  "${DRGN_PATH}/.libs"
-  "${DRGN_PATH}/velfutils/libdw"
-  "${DRGN_PATH}/velfutils/libelf"
-  "${DRGN_PATH}/velfutils/libdwelf"
-)
-set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+add_library(drgn INTERFACE)
+add_dependencies(drgn drgn_build)
+target_link_libraries(drgn INTERFACE
+  ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
+  ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
+  ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
+  ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
 
-include_directories(SYSTEM "${DRGN_PATH}")
+  BZip2::BZip2
+  LibLZMA::LibLZMA
+  OpenMP::OpenMP_CXX
+  ZLIB::ZLIB
+)
+target_include_directories(drgn SYSTEM INTERFACE "${drgn_SOURCE_DIR}" "${drgn_BINARY_DIR}")
 
 if (STATIC_LINK)
   # glog links against the `gflags` target, which is an alias for `gflags_shared`
@@ -289,7 +326,6 @@ add_library(oicore
   oi/PaddingHunter.cpp
   oi/Serialize.cpp
 )
-add_dependencies(oicore libdrgn)
 target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -323,7 +359,6 @@ else()
 endif()
 
 target_link_libraries(oicore
-  "-L${DRGN_PATH}/.libs"
   drgn
   dw
   pthread
@@ -334,11 +369,9 @@ add_library(treebuilder
   oi/TreeBuilder.cpp
   oi/exporters/TypeCheckingWalker.cpp
 )
-add_dependencies(treebuilder librocksdb)
 target_link_libraries(treebuilder
-  ${rocksdb_BINARY_DIR}/librocksdb.a
+  RocksDB::rocksdb
   oicore # overkill but it does need a lot of stuff
-  zstd::zstd
 )
 
 
@@ -375,10 +408,8 @@ target_link_libraries(oip oicore)
 
 ### Object Introspection RocksDB Printer (OIRP)
 add_executable(oirp tools/OIRP.cpp)
-add_dependencies(oirp librocksdb)
 target_link_libraries(oirp
-  ${rocksdb_BINARY_DIR}/librocksdb.a
-  zstd::zstd
+  RocksDB::rocksdb
   msgpackc
 )
 
@@ -417,3 +448,6 @@ endif()
 if (DEFINED ENV{CMAKE_HOOK})
   include($ENV{CMAKE_HOOK})
 endif()
+
+install(TARGETS oid DESTINATION ${CMAKE_INSTALL_BINDIR})
+
diff --git a/README.md b/README.md
index 9f95bf8..abce6f8 100644
--- a/README.md
+++ b/README.md
@@ -14,3 +14,61 @@ See the [CONTRIBUTING](CONTRIBUTING.md) file for how to help out.
 
 ## License
 Object Introspection is licensed under the [Apache 2.0 License](LICENSE).
+
+## Getting started with Nix
+
+Nix is the easiest way to get started with `oid` as it is non-trivial to build otherwise. Explicit Nix support for Object Introspection as a Library will come down the line, but Nix can currently provide you a reproducible development environment in which to build it.
+
+These examples expect you to have `nix` installed and available with no other dependencies required. Find the installation guide at https://nixos.org/download.html.
+
+We also required flake support. To enable flakes globally run:
+
+    $ mkdir -p ~/.config/nix
+    $ echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf
+
+Or suffix every `nix` command with `nix --extra-experimental-features 'nix-command flakes'`.
+
+### Run upstream OID without modifying the source
+
+    $ nix run github:facebookexperimental/object-introspection -- --help
+
+This will download the latest source into your Nix store along with all of its dependencies, running help afterwards.
+
+### Build OID locally
+
+    $ git clone https://github.com/facebookexperimental/object-introspection
+    $ nix build
+    $ ./result/bin/oid --help
+
+This will build OID from your local sources. Please note that this will NOT pick up changes to `extern/drgn` or `extern/drgn/libdrgn/velfutils`.
+
+### Get a development environment
+
+    $ nix develop
+    $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+    $ ninja -C build
+    $ build/oid --help
+
+This command provides a development shell with all the required dependencies. This is the most flexible option and will pick up source changes as CMake normally would.
+
+Sometimes this developer environment can be polluted by things installed on your normal system. If this is an issue, use:
+
+    $ nix develop -i
+
+This removes the environment from your host system and makes the build pure.
+
+### Run the tests
+
+    $ nix develop
+    $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+    $ ninja -C build
+    $ ./tools/config_gen.py -c clang++ build/testing.oid.toml
+    $ ctest -j --test-dir build/test
+
+Running tests under `nix` is new to the project and may take some time to mature. The CI is the source of truth for now.
+
+### Format source
+
+    $ nix fmt
+
+This formats the Nix, C++, and Python code in the repository.
diff --git a/cmake/Finduring.cmake b/cmake/Finduring.cmake
new file mode 100644
index 0000000..3b007fe
--- /dev/null
+++ b/cmake/Finduring.cmake
@@ -0,0 +1,27 @@
+# - Find liburing
+#
+# uring_INCLUDE_DIR - Where to find liburing.h
+# uring_LIBRARIES - List of libraries when using uring.
+# uring_FOUND - True if uring found.
+
+find_path(uring_INCLUDE_DIR
+  NAMES liburing.h)
+find_library(uring_LIBRARIES
+  NAMES liburing.a liburing.so)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(uring
+  DEFAULT_MSG uring_LIBRARIES uring_INCLUDE_DIR)
+
+mark_as_advanced(
+  uring_INCLUDE_DIR
+  uring_LIBRARIES)
+
+if(uring_FOUND AND NOT TARGET uring::uring)
+  add_library(uring::uring UNKNOWN IMPORTED)
+  set_target_properties(uring::uring PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${uring_INCLUDE_DIR}"
+    IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+    IMPORTED_LOCATION "${uring_LIBRARIES}")
+endif()
+
diff --git a/flake.nix b/flake.nix
index 7a3382e..1dffbe0 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,5 +1,5 @@
 {
-  description = "A flake for building Object Introspection.";
+  description = "Object level memory profiler for C++";
 
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
@@ -18,7 +18,99 @@
       treefmt-nix,
       ...
     }@inputs:
-    flake-utils.lib.eachDefaultSystem (
+    flake-utils.lib.eachSystem [ flake-utils.lib.system.x86_64-linux ] (
+      system:
+      let
+        pkgs = import nixpkgs { inherit system; };
+
+        drgnSrc = pkgs.fetchFromGitHub {
+          owner = "JakeHillion";
+          repo = "drgn";
+          rev = "b1f8c3e8526611b6720800250ba858a713dd9e4f";
+          hash = "sha256-5WhMHgx/RKtqjxGx4AyiqVKMot5xulr+6c8i2E9IxiA=";
+          fetchSubmodules = true;
+        };
+
+        mkOidPackage =
+          llvmPackages:
+          with pkgs;
+          pkgs.llvmPackages_15.stdenv.mkDerivation rec {
+            name = "oid";
+
+            src = self;
+
+            nativeBuildInputs = [
+              autoconf
+              automake
+              bison
+              cmake
+              flex
+              gettext
+              git
+              hexdump
+              libtool
+              ninja
+              pkgconf
+              python312
+              python312Packages.setuptools
+              python312Packages.toml
+              glibcLocales
+            ];
+
+            buildInputs = [
+              llvmPackages.libclang
+              llvmPackages.llvm
+
+              boost
+              bzip2
+              curl
+              double-conversion
+              elfutils
+              flex
+              folly
+              folly.fmt
+              gflags
+              glog
+              gtest
+              icu
+              jemalloc
+              libarchive
+              libmicrohttpd
+              liburing
+              libxml2
+              lzma
+              msgpack
+              range-v3
+              rocksdb_8_11
+              sqlite
+              tomlplusplus
+              zstd
+
+              llvmPackages_15.openmp # should match the stdenv clang version, see: https://github.com/NixOS/nixpkgs/issues/79818
+            ];
+
+            cmakeFlags = [
+              "-Ddrgn_SOURCE_DIR=${drgnSrc}"
+              "-DFORCE_BOOST_STATIC=Off"
+            ];
+
+            outputs = [ "out" ];
+          };
+      in
+      {
+        packages = rec {
+          default = oid-llvm15;
+
+          oid-llvm15 = mkOidPackage pkgs.llvmPackages_15;
+        };
+
+        apps.default = {
+          type = "app";
+          program = "${self.packages.${system}.default}/bin/oid";
+        };
+      }
+    )
+    // flake-utils.lib.eachDefaultSystem (
       system:
       let
         pkgs = nixpkgs.legacyPackages.${system};
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index bb73871..7d805e5 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus)
 add_library(drgn_utils DrgnUtils.cpp)
 target_link_libraries(drgn_utils
   glog::glog
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
-add_dependencies(drgn_utils libdrgn)
 
 add_library(symbol_service
   Descs.cpp
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 6c4da13..c878e08 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -20,12 +20,9 @@ add_library(type_graph
   TypeIdentifier.cpp
   Types.cpp
 )
-add_dependencies(type_graph libdrgn)
 target_link_libraries(type_graph
   container_info
   symbol_service
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
 target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/tools/config_gen.py b/tools/config_gen.py
index c20ca40..5ac56ff 100755
--- a/tools/config_gen.py
+++ b/tools/config_gen.py
@@ -16,6 +16,7 @@
 
 import argparse
 import getpass
+import os
 import pathlib
 import subprocess
 import typing
@@ -170,7 +171,10 @@ def pull_base_toml() -> typing.Dict:
 
     # Now, we need to replace any placeholders that might be present in the base toml file with the real verisons.
     user = getpass.getuser()
-    pwd = str(repo_path.resolve())
+    if "IN_NIX_SHELL" in os.environ and "src" in os.environ:
+        pwd = os.environ['src']
+    else:
+        pwd = str(repo_path.resolve())
 
     container_list = base.get("types", {}).get("containers")
     if container_list:

From 1e9a43e6af566343e2bb3a8bd90d7d837bcbdfbd Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 14 Aug 2024 21:17:32 +0100
Subject: [PATCH 185/188] nix: add building oid to the flake

---
 .circleci/config.yml                       |  18 +-
 .github/workflows/ci.yml                   |  53 +++++
 .github/workflows/object-introspection.yml |  15 --
 .github/workflows/test-report.xml          |  20 ++
 CMakeLists.txt                             | 230 ++++++++++++---------
 README.md                                  |  58 ++++++
 cmake/Finduring.cmake                      |  27 +++
 flake.nix                                  |  96 ++++++++-
 oi/CMakeLists.txt                          |   3 -
 oi/type_graph/CMakeLists.txt               |   3 -
 tools/config_gen.py                        |   6 +-
 11 files changed, 402 insertions(+), 127 deletions(-)
 create mode 100644 .github/workflows/ci.yml
 delete mode 100644 .github/workflows/object-introspection.yml
 create mode 100644 .github/workflows/test-report.xml
 create mode 100644 cmake/Finduring.cmake

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 589245a..dba98d7 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -83,20 +83,26 @@ jobs:
               libgtest-dev \
               libjemalloc-dev \
               libmsgpack-dev \
+              libomp-12-dev \
+              liburing-dev \
               libzstd-dev \
               llvm-15-dev \
               ninja-build \
               pkg-config \
               python3-setuptools
             pip3 install toml
+
+            # Ubuntu 22.04 CMake is too old and we don't have a newer image yet
+            git clone --depth 1 --branch v3.30.2 https://github.com/Kitware/CMake.git /tmp/cmake
+            (cd /tmp/cmake && cmake -B build/ -G Ninja && cmake --build build/)
           environment:
             DEBIAN_FRONTEND: noninteractive
       - checkout
       - run:
           name: Build
           command: |
-            cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
-            cmake --build build/
+            /tmp/cmake/build/bin/cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
+            ninja -C build/
             # Testing rubbish:
             cp test/ci.oid.toml build/testing.oid.toml
       - persist_to_workspace:
@@ -131,10 +137,12 @@ jobs:
             sudo apt-get install -y \
               clang-15 \
               libboost-all-dev \
-              libgflags-dev \
-              llvm-15-dev \
               libfmt-dev \
-              libjemalloc-dev
+              libgflags-dev \
+              libgoogle-glog-dev \
+              libjemalloc-dev \
+              libomp-12-dev \
+              llvm-15-dev
           environment:
             DEBIAN_FRONTEND: noninteractive
       - run:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..ab8cef2
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,53 @@
+name: CI
+on:
+  pull_request:
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.7
+    - uses: cachix/install-nix-action@v27
+      with:
+        github_access_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: nix fmt
+      run: |-
+        nix --experimental-features 'nix-command flakes' fmt
+        git diff --exit-code
+
+  build-test:
+    runs-on: 16-core-ubuntu
+    strategy:
+      matrix:
+        llvm_version: [15]
+    steps:
+    - uses: actions/checkout@v4.1.7
+    - uses: cachix/install-nix-action@v27
+      with:
+        github_access_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: build (LLVM ${{ matrix.llvm_version }})
+      # Run the build manually in `nix develop` to keep non-outputs around
+      run: |
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ninja -C build
+    - name: test (LLVM ${{ matrix.llvm_version }})
+      env:
+        # disable drgn multithreading as tests are already run in parallel
+        OMP_NUM_THREADS: 1
+      run: |
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ./tools/config_gen.py -c clang++ build/testing.oid.toml
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ctest \
+          --test-dir build/test/ \
+          --test-action Test \
+          --parallel \
+          --no-compress-output \
+          --output-on-failure \
+          --schedule-random \
+          --timeout 60 \
+          --repeat until-pass:2 \
+          --output-junit results.xml
+    - name: upload results
+      uses: actions/upload-artifact@v4
+      if: success() || failure()
+      with:
+        name: test-results
+        path: build/test/results.xml
diff --git a/.github/workflows/object-introspection.yml b/.github/workflows/object-introspection.yml
deleted file mode 100644
index 70b41aa..0000000
--- a/.github/workflows/object-introspection.yml
+++ /dev/null
@@ -1,15 +0,0 @@
-name: facebookexperimental/object-introspection
-on:
-  pull_request:
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: cachix/install-nix-action@v25
-      with:
-        github_access_token: ${{ secrets.GITHUB_TOKEN }}
-    - name: nix fmt
-      run: |-
-        nix --experimental-features 'nix-command flakes' fmt
-        git diff --exit-code
diff --git a/.github/workflows/test-report.xml b/.github/workflows/test-report.xml
new file mode 100644
index 0000000..f8b62b9
--- /dev/null
+++ b/.github/workflows/test-report.xml
@@ -0,0 +1,20 @@
+name: 'Test Report'
+on:
+  workflow_run:
+    workflows: ['CI']
+    types:
+      - completed
+permissions:
+  contents: read
+  actions: read
+  checks: write
+jobs:
+  report:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: dorny/test-reporter@v1
+      with:
+        artifact: test-results
+        name: CTest Tests
+        path: 'build/test/*.xml'
+        reporter: jest-junit
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8674d46..426b68c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,5 +1,5 @@
 # object-introspection
-cmake_minimum_required(VERSION 3.20)
+cmake_minimum_required(VERSION 3.24)
 project(object-introspection)
 
 # Lets find_program() locate SETUID binaries
@@ -33,75 +33,108 @@ find_package(gflags REQUIRED)
 ### tomlplusplus (for configuration files)
 FetchContent_Declare(
   tomlplusplus
-  GIT_REPOSITORY https://github.com/marzer/tomlplusplus.git
-  GIT_TAG        4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
-  GIT_PROGRESS   TRUE
+  GIT_REPOSITORY    https://github.com/marzer/tomlplusplus.git
+  GIT_TAG           4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
 FetchContent_MakeAvailable(tomlplusplus)
 
 ### glog
-FetchContent_Declare(
-  glog
-  GIT_REPOSITORY https://github.com/google/glog.git
-  GIT_TAG        96a2f23dca4cc7180821ca5f32e526314395d26a
-  GIT_PROGRESS   TRUE
-)
-FetchContent_MakeAvailable(glog)
+find_package(glog)
+if (NOT glog_FOUND)
+  FetchContent_Declare(
+    glog
+    GIT_REPOSITORY    https://github.com/google/glog.git
+    GIT_TAG           96a2f23dca4cc7180821ca5f32e526314395d26a
+    GIT_PROGRESS      TRUE
+    FIND_PACKAGE_ARGS
+  )
+  FetchContent_MakeAvailable(glog)
 
-# These glog executable targets still generate warnings - disable warnings for
-# them explicitly
-target_compile_options(demangle_unittest PRIVATE "-w")
-target_compile_options(logging_unittest PRIVATE "-w")
-target_compile_options(stl_logging_unittest PRIVATE "-w")
-target_compile_options(symbolize_unittest PRIVATE "-w")
-target_compile_options(utilities_unittest PRIVATE "-w")
+  # These glog executable targets still generate warnings - disable warnings for
+  # them explicitly
+  target_compile_options(demangle_unittest PRIVATE "-w")
+  target_compile_options(logging_unittest PRIVATE "-w")
+  target_compile_options(stl_logging_unittest PRIVATE "-w")
+  target_compile_options(symbolize_unittest PRIVATE "-w")
+  target_compile_options(utilities_unittest PRIVATE "-w")
+endif()
 
-### googletest
+### GTest
 # Do this in the main file so it can be fetched before setting project warnings.
 # After this is fixed with FetchContent, move to test/CMakeLists.txt.
 FetchContent_Declare(
-  googletest
-  GIT_REPOSITORY https://github.com/google/googletest.git
-  GIT_TAG        1ed6a8c67a0bd675149ece27bbec0ef1759854cf
-  GIT_PROGRESS   TRUE
+  GTest
+  GIT_REPOSITORY    https://github.com/google/googletest.git
+  GIT_TAG           1ed6a8c67a0bd675149ece27bbec0ef1759854cf
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
-FetchContent_MakeAvailable(googletest)
+FetchContent_MakeAvailable(GTest)
+
+### liburing (for RocksDB)
+find_package(uring REQUIRED)
 
 ### rocksdb
-FetchContent_Declare(
-  rocksdb
-  GIT_REPOSITORY https://github.com/facebook/rocksdb.git
-  GIT_TAG        f32521662acf3352397d438b732144c7813bbbec # v8.5.3
-  GIT_PROGRESS   TRUE
-)
-FetchContent_Populate(rocksdb)
-add_custom_target(librocksdb ALL
-  WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
-  COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
-  COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
-  BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
-  COMMENT "Building RocksDB"
-  USES_TERMINAL
-)
-include_directories(SYSTEM "${rocksdb_SOURCE_DIR}/include")
+find_package(RocksDB 8.11 CONFIG)
+if (NOT RocksDB_FOUND)
+  FetchContent_Declare(
+    rocksdb
+    GIT_REPOSITORY https://github.com/facebook/rocksdb.git
+    GIT_TAG        f32521662acf3352397d438b732144c7813bbbec # v8.5.3
+    GIT_PROGRESS   TRUE
+  )
+  FetchContent_Populate(rocksdb)
+
+  add_custom_target(librocksdb_build ALL
+    WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
+    COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
+    COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
+    BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
+    COMMENT "Building RocksDB"
+    USES_TERMINAL
+  )
+
+  ### zstd (for rocksdb)
+  find_package(zstd REQUIRED)
+
+  add_library(librocksdb INTERFACE)
+  add_dependencies(librocksdb librocksdb_build)
+  target_include_directories(librocksdb INTERFACE SYSTEM "${rocksdb_SOURCE_DIR}/include")
+  target_link_libraries(librocksdb INTERFACE ${rocksdb_BINARY_DIR}/librocksdb.a zstd::zstd)
+
+  add_library(RocksDB::rocksdb ALIAS librocksdb)
+endif()
 
 ### folly
 ### use folly as a header only library. some features won't be supported.
-FetchContent_Declare(
-  folly
-  GIT_REPOSITORY https://github.com/facebook/folly.git
-  GIT_TAG        c5aa5c46291a27f69acc920894d43605ceb43eba
-  GIT_PROGRESS   TRUE
-  PATCH_COMMAND  ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
-)
-FetchContent_Populate(folly)
+find_package(folly CONFIG)
+if (NOT folly_FOUND)
+  FetchContent_Declare(
+    folly
+    GIT_REPOSITORY    https://github.com/facebook/folly.git
+    GIT_TAG           c5aa5c46291a27f69acc920894d43605ceb43eba
+    GIT_PROGRESS      TRUE
+    PATCH_COMMAND     ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
+  )
+  FetchContent_Populate(folly)
+
+  add_library(folly_headers INTERFACE)
+  target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
+  target_link_libraries(folly_headers INTERFACE Boost::headers)
+else()
+  add_library(folly_headers INTERFACE)
+  target_include_directories(folly_headers SYSTEM INTERFACE ${FOLLY_INCLUDE_DIR})
+endif()
 
 ### range-v3
 FetchContent_Declare(
   range-v3
-  GIT_REPOSITORY https://github.com/ericniebler/range-v3.git
-  GIT_TAG        a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
-  GIT_PROGRESS   TRUE
+  GIT_REPOSITORY    https://github.com/ericniebler/range-v3.git
+  GIT_TAG           a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
 FetchContent_MakeAvailable(range-v3)
 
@@ -124,6 +157,18 @@ endif()
 ### (Re)download submodules
 find_package(Git QUIET)
 
+if (DEFINED drgn_SOURCE_DIR)
+    # drgn's autotools build requires source modification. in case we have a
+    # readonly source (read: nix) we must copy this. do this always to avoid
+    # polluting the source.
+    file(COPY "${drgn_SOURCE_DIR}/" DESTINATION "${FETCHCONTENT_BASE_DIR}/drgn-src" NO_SOURCE_PERMISSIONS)
+    SET(drgn_SOURCE_DIR "${FETCHCONTENT_BASE_DIR}/drgn-src")
+    SET(drgn_BINARY_DIR "${CMAKE_BINARY_DIR}/_deps/drgn-build")
+else()
+    SET(drgn_SOURCE_DIR "${PROJECT_SOURCE_DIR}/extern/drgn")
+    SET(drgn_BINARY_DIR "${drgn_SOURCE_DIR}/build")
+endif()
+
 # TODO: No idea if this huge block is required, just picked from an example. There may be a short-hand.
 if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
 # Update submodules as needed
@@ -145,20 +190,17 @@ if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
         if(NOT GIT_SUBMOD_RESULT EQUAL "0")
             message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules")
         endif()
+
+        if(NOT EXISTS "${drgn_SOURCE_DIR}")
+            message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
+        endif()
     endif()
 endif()
 
-if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn")
-    message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
-endif()
 
 ### Select Python version
 find_program(PYTHON NAMES python3.9 python3)
 
-add_library(folly_headers INTERFACE)
-target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
-target_link_libraries(folly_headers INTERFACE Boost::headers)
-
 ### bison & flex (for oid_parser)
 find_package(BISON 3.5 REQUIRED)
 find_package(FLEX)
@@ -194,8 +236,11 @@ find_package(msgpack REQUIRED CONFIG)
 get_target_property(MSGPACK_INCLUDE_DIRS msgpackc INTERFACE_INCLUDE_DIRECTORIES)
 include_directories(SYSTEM ${MSGPACK_INCLUDE_DIRS})
 
-### zstd (for rocksdb)
-find_package(zstd REQUIRED)
+### drgn/elfutils dependencies
+find_package(BZip2 REQUIRED)
+find_package(OpenMP REQUIRED)
+find_package(LibLZMA REQUIRED)
+find_package(ZLIB REQUIRED)
 
 ### drgn
 # The setup.py script in drgn is really meant to build drgn (python
@@ -214,46 +259,38 @@ find_package(zstd REQUIRED)
 # make
 #
 # Since setup.py has a single cmd to do this, just use it for now.
-#
-# Another extemely annoying point. drgn pretty much has to be compiled with gcc only
-# clang-12 does NOT work. clang fails with the following error :-
-# configure: error: gcc with GNU99 support required
 
 set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no")
 if (ASAN)
   list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes")
 endif()
-add_custom_target(libdrgn ALL
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn
-  COMMAND unset BISON_PKGDATADIR && CC=gcc CFLAGS="${DRGN_CFLAGS}" CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp build
-  BYPRODUCTS ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/.libs/libdrgnimpl.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdw/libdw.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libelf/libelf.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdwelf/libdwelf.a
+
+add_custom_target(drgn_build ALL
+  WORKING_DIRECTORY ${drgn_SOURCE_DIR}
+  COMMAND unset BISON_PKGDATADIR && CFLAGS="${DRGN_CFLAGS}"
+  CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp "${drgn_BINARY_DIR}"
+  BYPRODUCTS ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
+    ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
+    ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
+    ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
   COMMENT "Building drgn"
   USES_TERMINAL
 )
-set(DRGN_PATH "${PROJECT_SOURCE_DIR}/extern/drgn/build")
 
-# Ideally drgn stuff should be together at the end. But looks like rpath needs
-# to be set before add_executable() unfortunately. Maybe split libdrgn stuff
-# into a separate file later.
-set(CMAKE_SKIP_BUILD_RPATH FALSE)
-set(CMAKE_INSTALL_RPATH
-  "${DRGN_PATH}/.libs"
-  "${DRGN_PATH}/velfutils/libdw"
-  "${DRGN_PATH}/velfutils/libelf"
-  "${DRGN_PATH}/velfutils/libdwelf"
-)
-set(CMAKE_BUILD_RPATH
-  "${DRGN_PATH}/.libs"
-  "${DRGN_PATH}/velfutils/libdw"
-  "${DRGN_PATH}/velfutils/libelf"
-  "${DRGN_PATH}/velfutils/libdwelf"
-)
-set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+add_library(drgn INTERFACE)
+add_dependencies(drgn drgn_build)
+target_link_libraries(drgn INTERFACE
+  ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
+  ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
+  ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
+  ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
 
-include_directories(SYSTEM "${DRGN_PATH}")
+  BZip2::BZip2
+  LibLZMA::LibLZMA
+  OpenMP::OpenMP_CXX
+  ZLIB::ZLIB
+)
+target_include_directories(drgn SYSTEM INTERFACE "${drgn_SOURCE_DIR}" "${drgn_BINARY_DIR}")
 
 if (STATIC_LINK)
   # glog links against the `gflags` target, which is an alias for `gflags_shared`
@@ -289,7 +326,6 @@ add_library(oicore
   oi/PaddingHunter.cpp
   oi/Serialize.cpp
 )
-add_dependencies(oicore libdrgn)
 target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -323,7 +359,6 @@ else()
 endif()
 
 target_link_libraries(oicore
-  "-L${DRGN_PATH}/.libs"
   drgn
   dw
   pthread
@@ -334,11 +369,9 @@ add_library(treebuilder
   oi/TreeBuilder.cpp
   oi/exporters/TypeCheckingWalker.cpp
 )
-add_dependencies(treebuilder librocksdb)
 target_link_libraries(treebuilder
-  ${rocksdb_BINARY_DIR}/librocksdb.a
+  RocksDB::rocksdb
   oicore # overkill but it does need a lot of stuff
-  zstd::zstd
 )
 
 
@@ -375,10 +408,8 @@ target_link_libraries(oip oicore)
 
 ### Object Introspection RocksDB Printer (OIRP)
 add_executable(oirp tools/OIRP.cpp)
-add_dependencies(oirp librocksdb)
 target_link_libraries(oirp
-  ${rocksdb_BINARY_DIR}/librocksdb.a
-  zstd::zstd
+  RocksDB::rocksdb
   msgpackc
 )
 
@@ -417,3 +448,6 @@ endif()
 if (DEFINED ENV{CMAKE_HOOK})
   include($ENV{CMAKE_HOOK})
 endif()
+
+install(TARGETS oid DESTINATION ${CMAKE_INSTALL_BINDIR})
+
diff --git a/README.md b/README.md
index 9f95bf8..abce6f8 100644
--- a/README.md
+++ b/README.md
@@ -14,3 +14,61 @@ See the [CONTRIBUTING](CONTRIBUTING.md) file for how to help out.
 
 ## License
 Object Introspection is licensed under the [Apache 2.0 License](LICENSE).
+
+## Getting started with Nix
+
+Nix is the easiest way to get started with `oid` as it is non-trivial to build otherwise. Explicit Nix support for Object Introspection as a Library will come down the line, but Nix can currently provide you a reproducible development environment in which to build it.
+
+These examples expect you to have `nix` installed and available with no other dependencies required. Find the installation guide at https://nixos.org/download.html.
+
+We also required flake support. To enable flakes globally run:
+
+    $ mkdir -p ~/.config/nix
+    $ echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf
+
+Or suffix every `nix` command with `nix --extra-experimental-features 'nix-command flakes'`.
+
+### Run upstream OID without modifying the source
+
+    $ nix run github:facebookexperimental/object-introspection -- --help
+
+This will download the latest source into your Nix store along with all of its dependencies, running help afterwards.
+
+### Build OID locally
+
+    $ git clone https://github.com/facebookexperimental/object-introspection
+    $ nix build
+    $ ./result/bin/oid --help
+
+This will build OID from your local sources. Please note that this will NOT pick up changes to `extern/drgn` or `extern/drgn/libdrgn/velfutils`.
+
+### Get a development environment
+
+    $ nix develop
+    $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+    $ ninja -C build
+    $ build/oid --help
+
+This command provides a development shell with all the required dependencies. This is the most flexible option and will pick up source changes as CMake normally would.
+
+Sometimes this developer environment can be polluted by things installed on your normal system. If this is an issue, use:
+
+    $ nix develop -i
+
+This removes the environment from your host system and makes the build pure.
+
+### Run the tests
+
+    $ nix develop
+    $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+    $ ninja -C build
+    $ ./tools/config_gen.py -c clang++ build/testing.oid.toml
+    $ ctest -j --test-dir build/test
+
+Running tests under `nix` is new to the project and may take some time to mature. The CI is the source of truth for now.
+
+### Format source
+
+    $ nix fmt
+
+This formats the Nix, C++, and Python code in the repository.
diff --git a/cmake/Finduring.cmake b/cmake/Finduring.cmake
new file mode 100644
index 0000000..3b007fe
--- /dev/null
+++ b/cmake/Finduring.cmake
@@ -0,0 +1,27 @@
+# - Find liburing
+#
+# uring_INCLUDE_DIR - Where to find liburing.h
+# uring_LIBRARIES - List of libraries when using uring.
+# uring_FOUND - True if uring found.
+
+find_path(uring_INCLUDE_DIR
+  NAMES liburing.h)
+find_library(uring_LIBRARIES
+  NAMES liburing.a liburing.so)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(uring
+  DEFAULT_MSG uring_LIBRARIES uring_INCLUDE_DIR)
+
+mark_as_advanced(
+  uring_INCLUDE_DIR
+  uring_LIBRARIES)
+
+if(uring_FOUND AND NOT TARGET uring::uring)
+  add_library(uring::uring UNKNOWN IMPORTED)
+  set_target_properties(uring::uring PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${uring_INCLUDE_DIR}"
+    IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+    IMPORTED_LOCATION "${uring_LIBRARIES}")
+endif()
+
diff --git a/flake.nix b/flake.nix
index 7a3382e..1dffbe0 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,5 +1,5 @@
 {
-  description = "A flake for building Object Introspection.";
+  description = "Object level memory profiler for C++";
 
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
@@ -18,7 +18,99 @@
       treefmt-nix,
       ...
     }@inputs:
-    flake-utils.lib.eachDefaultSystem (
+    flake-utils.lib.eachSystem [ flake-utils.lib.system.x86_64-linux ] (
+      system:
+      let
+        pkgs = import nixpkgs { inherit system; };
+
+        drgnSrc = pkgs.fetchFromGitHub {
+          owner = "JakeHillion";
+          repo = "drgn";
+          rev = "b1f8c3e8526611b6720800250ba858a713dd9e4f";
+          hash = "sha256-5WhMHgx/RKtqjxGx4AyiqVKMot5xulr+6c8i2E9IxiA=";
+          fetchSubmodules = true;
+        };
+
+        mkOidPackage =
+          llvmPackages:
+          with pkgs;
+          pkgs.llvmPackages_15.stdenv.mkDerivation rec {
+            name = "oid";
+
+            src = self;
+
+            nativeBuildInputs = [
+              autoconf
+              automake
+              bison
+              cmake
+              flex
+              gettext
+              git
+              hexdump
+              libtool
+              ninja
+              pkgconf
+              python312
+              python312Packages.setuptools
+              python312Packages.toml
+              glibcLocales
+            ];
+
+            buildInputs = [
+              llvmPackages.libclang
+              llvmPackages.llvm
+
+              boost
+              bzip2
+              curl
+              double-conversion
+              elfutils
+              flex
+              folly
+              folly.fmt
+              gflags
+              glog
+              gtest
+              icu
+              jemalloc
+              libarchive
+              libmicrohttpd
+              liburing
+              libxml2
+              lzma
+              msgpack
+              range-v3
+              rocksdb_8_11
+              sqlite
+              tomlplusplus
+              zstd
+
+              llvmPackages_15.openmp # should match the stdenv clang version, see: https://github.com/NixOS/nixpkgs/issues/79818
+            ];
+
+            cmakeFlags = [
+              "-Ddrgn_SOURCE_DIR=${drgnSrc}"
+              "-DFORCE_BOOST_STATIC=Off"
+            ];
+
+            outputs = [ "out" ];
+          };
+      in
+      {
+        packages = rec {
+          default = oid-llvm15;
+
+          oid-llvm15 = mkOidPackage pkgs.llvmPackages_15;
+        };
+
+        apps.default = {
+          type = "app";
+          program = "${self.packages.${system}.default}/bin/oid";
+        };
+      }
+    )
+    // flake-utils.lib.eachDefaultSystem (
       system:
       let
         pkgs = nixpkgs.legacyPackages.${system};
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index bb73871..7d805e5 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus)
 add_library(drgn_utils DrgnUtils.cpp)
 target_link_libraries(drgn_utils
   glog::glog
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
-add_dependencies(drgn_utils libdrgn)
 
 add_library(symbol_service
   Descs.cpp
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 6c4da13..c878e08 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -20,12 +20,9 @@ add_library(type_graph
   TypeIdentifier.cpp
   Types.cpp
 )
-add_dependencies(type_graph libdrgn)
 target_link_libraries(type_graph
   container_info
   symbol_service
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
 target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/tools/config_gen.py b/tools/config_gen.py
index c20ca40..5ac56ff 100755
--- a/tools/config_gen.py
+++ b/tools/config_gen.py
@@ -16,6 +16,7 @@
 
 import argparse
 import getpass
+import os
 import pathlib
 import subprocess
 import typing
@@ -170,7 +171,10 @@ def pull_base_toml() -> typing.Dict:
 
     # Now, we need to replace any placeholders that might be present in the base toml file with the real verisons.
     user = getpass.getuser()
-    pwd = str(repo_path.resolve())
+    if "IN_NIX_SHELL" in os.environ and "src" in os.environ:
+        pwd = os.environ['src']
+    else:
+        pwd = str(repo_path.resolve())
 
     container_list = base.get("types", {}).get("containers")
     if container_list:

From 1d01f8befdd72048d7c7719518ca078bdc758543 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 14 Aug 2024 21:19:14 +0100
Subject: [PATCH 186/188] nix: add building oid to the flake

---
 .circleci/config.yml                       |  18 +-
 .github/workflows/ci.yml                   |  54 +++++
 .github/workflows/object-introspection.yml |  15 --
 .github/workflows/test-report.xml          |  20 ++
 CMakeLists.txt                             | 230 ++++++++++++---------
 README.md                                  |  58 ++++++
 cmake/Finduring.cmake                      |  27 +++
 flake.nix                                  |  96 ++++++++-
 oi/CMakeLists.txt                          |   3 -
 oi/type_graph/CMakeLists.txt               |   3 -
 tools/config_gen.py                        |   6 +-
 11 files changed, 403 insertions(+), 127 deletions(-)
 create mode 100644 .github/workflows/ci.yml
 delete mode 100644 .github/workflows/object-introspection.yml
 create mode 100644 .github/workflows/test-report.xml
 create mode 100644 cmake/Finduring.cmake

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 589245a..dba98d7 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -83,20 +83,26 @@ jobs:
               libgtest-dev \
               libjemalloc-dev \
               libmsgpack-dev \
+              libomp-12-dev \
+              liburing-dev \
               libzstd-dev \
               llvm-15-dev \
               ninja-build \
               pkg-config \
               python3-setuptools
             pip3 install toml
+
+            # Ubuntu 22.04 CMake is too old and we don't have a newer image yet
+            git clone --depth 1 --branch v3.30.2 https://github.com/Kitware/CMake.git /tmp/cmake
+            (cd /tmp/cmake && cmake -B build/ -G Ninja && cmake --build build/)
           environment:
             DEBIAN_FRONTEND: noninteractive
       - checkout
       - run:
           name: Build
           command: |
-            cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
-            cmake --build build/
+            /tmp/cmake/build/bin/cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
+            ninja -C build/
             # Testing rubbish:
             cp test/ci.oid.toml build/testing.oid.toml
       - persist_to_workspace:
@@ -131,10 +137,12 @@ jobs:
             sudo apt-get install -y \
               clang-15 \
               libboost-all-dev \
-              libgflags-dev \
-              llvm-15-dev \
               libfmt-dev \
-              libjemalloc-dev
+              libgflags-dev \
+              libgoogle-glog-dev \
+              libjemalloc-dev \
+              libomp-12-dev \
+              llvm-15-dev
           environment:
             DEBIAN_FRONTEND: noninteractive
       - run:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..a8a0717
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,54 @@
+name: CI
+on:
+  pull_request:
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.7
+    - uses: cachix/install-nix-action@v27
+      with:
+        github_access_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: nix fmt
+      run: |-
+        nix --experimental-features 'nix-command flakes' fmt
+        git diff --exit-code
+
+  build-test:
+    runs-on: 16-core-ubuntu
+    strategy:
+      matrix:
+        llvm_version: [15]
+    steps:
+    - uses: actions/checkout@v4.1.7
+    - uses: cachix/install-nix-action@v27
+      with:
+        github_access_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: build (LLVM ${{ matrix.llvm_version }})
+      # Run the build manually in `nix develop` to keep non-outputs around
+      run: |
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ninja -C build
+    - name: test (LLVM ${{ matrix.llvm_version }})
+      env:
+        # disable drgn multithreading as tests are already run in parallel
+        OMP_NUM_THREADS: 1
+      run: |
+        echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ./tools/config_gen.py -c clang++ build/testing.oid.toml
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ctest \
+          --test-dir build/test/ \
+          --test-action Test \
+          --parallel \
+          --no-compress-output \
+          --output-on-failure \
+          --schedule-random \
+          --timeout 60 \
+          --repeat until-pass:2 \
+          --output-junit results.xml
+    - name: upload results
+      uses: actions/upload-artifact@v4
+      if: success() || failure()
+      with:
+        name: test-results
+        path: build/test/results.xml
diff --git a/.github/workflows/object-introspection.yml b/.github/workflows/object-introspection.yml
deleted file mode 100644
index 70b41aa..0000000
--- a/.github/workflows/object-introspection.yml
+++ /dev/null
@@ -1,15 +0,0 @@
-name: facebookexperimental/object-introspection
-on:
-  pull_request:
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: cachix/install-nix-action@v25
-      with:
-        github_access_token: ${{ secrets.GITHUB_TOKEN }}
-    - name: nix fmt
-      run: |-
-        nix --experimental-features 'nix-command flakes' fmt
-        git diff --exit-code
diff --git a/.github/workflows/test-report.xml b/.github/workflows/test-report.xml
new file mode 100644
index 0000000..f8b62b9
--- /dev/null
+++ b/.github/workflows/test-report.xml
@@ -0,0 +1,20 @@
+name: 'Test Report'
+on:
+  workflow_run:
+    workflows: ['CI']
+    types:
+      - completed
+permissions:
+  contents: read
+  actions: read
+  checks: write
+jobs:
+  report:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: dorny/test-reporter@v1
+      with:
+        artifact: test-results
+        name: CTest Tests
+        path: 'build/test/*.xml'
+        reporter: jest-junit
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8674d46..426b68c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,5 +1,5 @@
 # object-introspection
-cmake_minimum_required(VERSION 3.20)
+cmake_minimum_required(VERSION 3.24)
 project(object-introspection)
 
 # Lets find_program() locate SETUID binaries
@@ -33,75 +33,108 @@ find_package(gflags REQUIRED)
 ### tomlplusplus (for configuration files)
 FetchContent_Declare(
   tomlplusplus
-  GIT_REPOSITORY https://github.com/marzer/tomlplusplus.git
-  GIT_TAG        4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
-  GIT_PROGRESS   TRUE
+  GIT_REPOSITORY    https://github.com/marzer/tomlplusplus.git
+  GIT_TAG           4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
 FetchContent_MakeAvailable(tomlplusplus)
 
 ### glog
-FetchContent_Declare(
-  glog
-  GIT_REPOSITORY https://github.com/google/glog.git
-  GIT_TAG        96a2f23dca4cc7180821ca5f32e526314395d26a
-  GIT_PROGRESS   TRUE
-)
-FetchContent_MakeAvailable(glog)
+find_package(glog)
+if (NOT glog_FOUND)
+  FetchContent_Declare(
+    glog
+    GIT_REPOSITORY    https://github.com/google/glog.git
+    GIT_TAG           96a2f23dca4cc7180821ca5f32e526314395d26a
+    GIT_PROGRESS      TRUE
+    FIND_PACKAGE_ARGS
+  )
+  FetchContent_MakeAvailable(glog)
 
-# These glog executable targets still generate warnings - disable warnings for
-# them explicitly
-target_compile_options(demangle_unittest PRIVATE "-w")
-target_compile_options(logging_unittest PRIVATE "-w")
-target_compile_options(stl_logging_unittest PRIVATE "-w")
-target_compile_options(symbolize_unittest PRIVATE "-w")
-target_compile_options(utilities_unittest PRIVATE "-w")
+  # These glog executable targets still generate warnings - disable warnings for
+  # them explicitly
+  target_compile_options(demangle_unittest PRIVATE "-w")
+  target_compile_options(logging_unittest PRIVATE "-w")
+  target_compile_options(stl_logging_unittest PRIVATE "-w")
+  target_compile_options(symbolize_unittest PRIVATE "-w")
+  target_compile_options(utilities_unittest PRIVATE "-w")
+endif()
 
-### googletest
+### GTest
 # Do this in the main file so it can be fetched before setting project warnings.
 # After this is fixed with FetchContent, move to test/CMakeLists.txt.
 FetchContent_Declare(
-  googletest
-  GIT_REPOSITORY https://github.com/google/googletest.git
-  GIT_TAG        1ed6a8c67a0bd675149ece27bbec0ef1759854cf
-  GIT_PROGRESS   TRUE
+  GTest
+  GIT_REPOSITORY    https://github.com/google/googletest.git
+  GIT_TAG           1ed6a8c67a0bd675149ece27bbec0ef1759854cf
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
-FetchContent_MakeAvailable(googletest)
+FetchContent_MakeAvailable(GTest)
+
+### liburing (for RocksDB)
+find_package(uring REQUIRED)
 
 ### rocksdb
-FetchContent_Declare(
-  rocksdb
-  GIT_REPOSITORY https://github.com/facebook/rocksdb.git
-  GIT_TAG        f32521662acf3352397d438b732144c7813bbbec # v8.5.3
-  GIT_PROGRESS   TRUE
-)
-FetchContent_Populate(rocksdb)
-add_custom_target(librocksdb ALL
-  WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
-  COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
-  COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
-  BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
-  COMMENT "Building RocksDB"
-  USES_TERMINAL
-)
-include_directories(SYSTEM "${rocksdb_SOURCE_DIR}/include")
+find_package(RocksDB 8.11 CONFIG)
+if (NOT RocksDB_FOUND)
+  FetchContent_Declare(
+    rocksdb
+    GIT_REPOSITORY https://github.com/facebook/rocksdb.git
+    GIT_TAG        f32521662acf3352397d438b732144c7813bbbec # v8.5.3
+    GIT_PROGRESS   TRUE
+  )
+  FetchContent_Populate(rocksdb)
+
+  add_custom_target(librocksdb_build ALL
+    WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
+    COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
+    COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
+    BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
+    COMMENT "Building RocksDB"
+    USES_TERMINAL
+  )
+
+  ### zstd (for rocksdb)
+  find_package(zstd REQUIRED)
+
+  add_library(librocksdb INTERFACE)
+  add_dependencies(librocksdb librocksdb_build)
+  target_include_directories(librocksdb INTERFACE SYSTEM "${rocksdb_SOURCE_DIR}/include")
+  target_link_libraries(librocksdb INTERFACE ${rocksdb_BINARY_DIR}/librocksdb.a zstd::zstd)
+
+  add_library(RocksDB::rocksdb ALIAS librocksdb)
+endif()
 
 ### folly
 ### use folly as a header only library. some features won't be supported.
-FetchContent_Declare(
-  folly
-  GIT_REPOSITORY https://github.com/facebook/folly.git
-  GIT_TAG        c5aa5c46291a27f69acc920894d43605ceb43eba
-  GIT_PROGRESS   TRUE
-  PATCH_COMMAND  ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
-)
-FetchContent_Populate(folly)
+find_package(folly CONFIG)
+if (NOT folly_FOUND)
+  FetchContent_Declare(
+    folly
+    GIT_REPOSITORY    https://github.com/facebook/folly.git
+    GIT_TAG           c5aa5c46291a27f69acc920894d43605ceb43eba
+    GIT_PROGRESS      TRUE
+    PATCH_COMMAND     ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
+  )
+  FetchContent_Populate(folly)
+
+  add_library(folly_headers INTERFACE)
+  target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
+  target_link_libraries(folly_headers INTERFACE Boost::headers)
+else()
+  add_library(folly_headers INTERFACE)
+  target_include_directories(folly_headers SYSTEM INTERFACE ${FOLLY_INCLUDE_DIR})
+endif()
 
 ### range-v3
 FetchContent_Declare(
   range-v3
-  GIT_REPOSITORY https://github.com/ericniebler/range-v3.git
-  GIT_TAG        a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
-  GIT_PROGRESS   TRUE
+  GIT_REPOSITORY    https://github.com/ericniebler/range-v3.git
+  GIT_TAG           a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
 FetchContent_MakeAvailable(range-v3)
 
@@ -124,6 +157,18 @@ endif()
 ### (Re)download submodules
 find_package(Git QUIET)
 
+if (DEFINED drgn_SOURCE_DIR)
+    # drgn's autotools build requires source modification. in case we have a
+    # readonly source (read: nix) we must copy this. do this always to avoid
+    # polluting the source.
+    file(COPY "${drgn_SOURCE_DIR}/" DESTINATION "${FETCHCONTENT_BASE_DIR}/drgn-src" NO_SOURCE_PERMISSIONS)
+    SET(drgn_SOURCE_DIR "${FETCHCONTENT_BASE_DIR}/drgn-src")
+    SET(drgn_BINARY_DIR "${CMAKE_BINARY_DIR}/_deps/drgn-build")
+else()
+    SET(drgn_SOURCE_DIR "${PROJECT_SOURCE_DIR}/extern/drgn")
+    SET(drgn_BINARY_DIR "${drgn_SOURCE_DIR}/build")
+endif()
+
 # TODO: No idea if this huge block is required, just picked from an example. There may be a short-hand.
 if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
 # Update submodules as needed
@@ -145,20 +190,17 @@ if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
         if(NOT GIT_SUBMOD_RESULT EQUAL "0")
             message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules")
         endif()
+
+        if(NOT EXISTS "${drgn_SOURCE_DIR}")
+            message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
+        endif()
     endif()
 endif()
 
-if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn")
-    message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
-endif()
 
 ### Select Python version
 find_program(PYTHON NAMES python3.9 python3)
 
-add_library(folly_headers INTERFACE)
-target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
-target_link_libraries(folly_headers INTERFACE Boost::headers)
-
 ### bison & flex (for oid_parser)
 find_package(BISON 3.5 REQUIRED)
 find_package(FLEX)
@@ -194,8 +236,11 @@ find_package(msgpack REQUIRED CONFIG)
 get_target_property(MSGPACK_INCLUDE_DIRS msgpackc INTERFACE_INCLUDE_DIRECTORIES)
 include_directories(SYSTEM ${MSGPACK_INCLUDE_DIRS})
 
-### zstd (for rocksdb)
-find_package(zstd REQUIRED)
+### drgn/elfutils dependencies
+find_package(BZip2 REQUIRED)
+find_package(OpenMP REQUIRED)
+find_package(LibLZMA REQUIRED)
+find_package(ZLIB REQUIRED)
 
 ### drgn
 # The setup.py script in drgn is really meant to build drgn (python
@@ -214,46 +259,38 @@ find_package(zstd REQUIRED)
 # make
 #
 # Since setup.py has a single cmd to do this, just use it for now.
-#
-# Another extemely annoying point. drgn pretty much has to be compiled with gcc only
-# clang-12 does NOT work. clang fails with the following error :-
-# configure: error: gcc with GNU99 support required
 
 set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no")
 if (ASAN)
   list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes")
 endif()
-add_custom_target(libdrgn ALL
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn
-  COMMAND unset BISON_PKGDATADIR && CC=gcc CFLAGS="${DRGN_CFLAGS}" CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp build
-  BYPRODUCTS ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/.libs/libdrgnimpl.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdw/libdw.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libelf/libelf.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdwelf/libdwelf.a
+
+add_custom_target(drgn_build ALL
+  WORKING_DIRECTORY ${drgn_SOURCE_DIR}
+  COMMAND unset BISON_PKGDATADIR && CFLAGS="${DRGN_CFLAGS}"
+  CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp "${drgn_BINARY_DIR}"
+  BYPRODUCTS ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
+    ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
+    ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
+    ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
   COMMENT "Building drgn"
   USES_TERMINAL
 )
-set(DRGN_PATH "${PROJECT_SOURCE_DIR}/extern/drgn/build")
 
-# Ideally drgn stuff should be together at the end. But looks like rpath needs
-# to be set before add_executable() unfortunately. Maybe split libdrgn stuff
-# into a separate file later.
-set(CMAKE_SKIP_BUILD_RPATH FALSE)
-set(CMAKE_INSTALL_RPATH
-  "${DRGN_PATH}/.libs"
-  "${DRGN_PATH}/velfutils/libdw"
-  "${DRGN_PATH}/velfutils/libelf"
-  "${DRGN_PATH}/velfutils/libdwelf"
-)
-set(CMAKE_BUILD_RPATH
-  "${DRGN_PATH}/.libs"
-  "${DRGN_PATH}/velfutils/libdw"
-  "${DRGN_PATH}/velfutils/libelf"
-  "${DRGN_PATH}/velfutils/libdwelf"
-)
-set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+add_library(drgn INTERFACE)
+add_dependencies(drgn drgn_build)
+target_link_libraries(drgn INTERFACE
+  ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
+  ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
+  ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
+  ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
 
-include_directories(SYSTEM "${DRGN_PATH}")
+  BZip2::BZip2
+  LibLZMA::LibLZMA
+  OpenMP::OpenMP_CXX
+  ZLIB::ZLIB
+)
+target_include_directories(drgn SYSTEM INTERFACE "${drgn_SOURCE_DIR}" "${drgn_BINARY_DIR}")
 
 if (STATIC_LINK)
   # glog links against the `gflags` target, which is an alias for `gflags_shared`
@@ -289,7 +326,6 @@ add_library(oicore
   oi/PaddingHunter.cpp
   oi/Serialize.cpp
 )
-add_dependencies(oicore libdrgn)
 target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -323,7 +359,6 @@ else()
 endif()
 
 target_link_libraries(oicore
-  "-L${DRGN_PATH}/.libs"
   drgn
   dw
   pthread
@@ -334,11 +369,9 @@ add_library(treebuilder
   oi/TreeBuilder.cpp
   oi/exporters/TypeCheckingWalker.cpp
 )
-add_dependencies(treebuilder librocksdb)
 target_link_libraries(treebuilder
-  ${rocksdb_BINARY_DIR}/librocksdb.a
+  RocksDB::rocksdb
   oicore # overkill but it does need a lot of stuff
-  zstd::zstd
 )
 
 
@@ -375,10 +408,8 @@ target_link_libraries(oip oicore)
 
 ### Object Introspection RocksDB Printer (OIRP)
 add_executable(oirp tools/OIRP.cpp)
-add_dependencies(oirp librocksdb)
 target_link_libraries(oirp
-  ${rocksdb_BINARY_DIR}/librocksdb.a
-  zstd::zstd
+  RocksDB::rocksdb
   msgpackc
 )
 
@@ -417,3 +448,6 @@ endif()
 if (DEFINED ENV{CMAKE_HOOK})
   include($ENV{CMAKE_HOOK})
 endif()
+
+install(TARGETS oid DESTINATION ${CMAKE_INSTALL_BINDIR})
+
diff --git a/README.md b/README.md
index 9f95bf8..abce6f8 100644
--- a/README.md
+++ b/README.md
@@ -14,3 +14,61 @@ See the [CONTRIBUTING](CONTRIBUTING.md) file for how to help out.
 
 ## License
 Object Introspection is licensed under the [Apache 2.0 License](LICENSE).
+
+## Getting started with Nix
+
+Nix is the easiest way to get started with `oid` as it is non-trivial to build otherwise. Explicit Nix support for Object Introspection as a Library will come down the line, but Nix can currently provide you a reproducible development environment in which to build it.
+
+These examples expect you to have `nix` installed and available with no other dependencies required. Find the installation guide at https://nixos.org/download.html.
+
+We also required flake support. To enable flakes globally run:
+
+    $ mkdir -p ~/.config/nix
+    $ echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf
+
+Or suffix every `nix` command with `nix --extra-experimental-features 'nix-command flakes'`.
+
+### Run upstream OID without modifying the source
+
+    $ nix run github:facebookexperimental/object-introspection -- --help
+
+This will download the latest source into your Nix store along with all of its dependencies, running help afterwards.
+
+### Build OID locally
+
+    $ git clone https://github.com/facebookexperimental/object-introspection
+    $ nix build
+    $ ./result/bin/oid --help
+
+This will build OID from your local sources. Please note that this will NOT pick up changes to `extern/drgn` or `extern/drgn/libdrgn/velfutils`.
+
+### Get a development environment
+
+    $ nix develop
+    $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+    $ ninja -C build
+    $ build/oid --help
+
+This command provides a development shell with all the required dependencies. This is the most flexible option and will pick up source changes as CMake normally would.
+
+Sometimes this developer environment can be polluted by things installed on your normal system. If this is an issue, use:
+
+    $ nix develop -i
+
+This removes the environment from your host system and makes the build pure.
+
+### Run the tests
+
+    $ nix develop
+    $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+    $ ninja -C build
+    $ ./tools/config_gen.py -c clang++ build/testing.oid.toml
+    $ ctest -j --test-dir build/test
+
+Running tests under `nix` is new to the project and may take some time to mature. The CI is the source of truth for now.
+
+### Format source
+
+    $ nix fmt
+
+This formats the Nix, C++, and Python code in the repository.
diff --git a/cmake/Finduring.cmake b/cmake/Finduring.cmake
new file mode 100644
index 0000000..3b007fe
--- /dev/null
+++ b/cmake/Finduring.cmake
@@ -0,0 +1,27 @@
+# - Find liburing
+#
+# uring_INCLUDE_DIR - Where to find liburing.h
+# uring_LIBRARIES - List of libraries when using uring.
+# uring_FOUND - True if uring found.
+
+find_path(uring_INCLUDE_DIR
+  NAMES liburing.h)
+find_library(uring_LIBRARIES
+  NAMES liburing.a liburing.so)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(uring
+  DEFAULT_MSG uring_LIBRARIES uring_INCLUDE_DIR)
+
+mark_as_advanced(
+  uring_INCLUDE_DIR
+  uring_LIBRARIES)
+
+if(uring_FOUND AND NOT TARGET uring::uring)
+  add_library(uring::uring UNKNOWN IMPORTED)
+  set_target_properties(uring::uring PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${uring_INCLUDE_DIR}"
+    IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+    IMPORTED_LOCATION "${uring_LIBRARIES}")
+endif()
+
diff --git a/flake.nix b/flake.nix
index 7a3382e..1dffbe0 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,5 +1,5 @@
 {
-  description = "A flake for building Object Introspection.";
+  description = "Object level memory profiler for C++";
 
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
@@ -18,7 +18,99 @@
       treefmt-nix,
       ...
     }@inputs:
-    flake-utils.lib.eachDefaultSystem (
+    flake-utils.lib.eachSystem [ flake-utils.lib.system.x86_64-linux ] (
+      system:
+      let
+        pkgs = import nixpkgs { inherit system; };
+
+        drgnSrc = pkgs.fetchFromGitHub {
+          owner = "JakeHillion";
+          repo = "drgn";
+          rev = "b1f8c3e8526611b6720800250ba858a713dd9e4f";
+          hash = "sha256-5WhMHgx/RKtqjxGx4AyiqVKMot5xulr+6c8i2E9IxiA=";
+          fetchSubmodules = true;
+        };
+
+        mkOidPackage =
+          llvmPackages:
+          with pkgs;
+          pkgs.llvmPackages_15.stdenv.mkDerivation rec {
+            name = "oid";
+
+            src = self;
+
+            nativeBuildInputs = [
+              autoconf
+              automake
+              bison
+              cmake
+              flex
+              gettext
+              git
+              hexdump
+              libtool
+              ninja
+              pkgconf
+              python312
+              python312Packages.setuptools
+              python312Packages.toml
+              glibcLocales
+            ];
+
+            buildInputs = [
+              llvmPackages.libclang
+              llvmPackages.llvm
+
+              boost
+              bzip2
+              curl
+              double-conversion
+              elfutils
+              flex
+              folly
+              folly.fmt
+              gflags
+              glog
+              gtest
+              icu
+              jemalloc
+              libarchive
+              libmicrohttpd
+              liburing
+              libxml2
+              lzma
+              msgpack
+              range-v3
+              rocksdb_8_11
+              sqlite
+              tomlplusplus
+              zstd
+
+              llvmPackages_15.openmp # should match the stdenv clang version, see: https://github.com/NixOS/nixpkgs/issues/79818
+            ];
+
+            cmakeFlags = [
+              "-Ddrgn_SOURCE_DIR=${drgnSrc}"
+              "-DFORCE_BOOST_STATIC=Off"
+            ];
+
+            outputs = [ "out" ];
+          };
+      in
+      {
+        packages = rec {
+          default = oid-llvm15;
+
+          oid-llvm15 = mkOidPackage pkgs.llvmPackages_15;
+        };
+
+        apps.default = {
+          type = "app";
+          program = "${self.packages.${system}.default}/bin/oid";
+        };
+      }
+    )
+    // flake-utils.lib.eachDefaultSystem (
       system:
       let
         pkgs = nixpkgs.legacyPackages.${system};
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index bb73871..7d805e5 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus)
 add_library(drgn_utils DrgnUtils.cpp)
 target_link_libraries(drgn_utils
   glog::glog
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
-add_dependencies(drgn_utils libdrgn)
 
 add_library(symbol_service
   Descs.cpp
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 6c4da13..c878e08 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -20,12 +20,9 @@ add_library(type_graph
   TypeIdentifier.cpp
   Types.cpp
 )
-add_dependencies(type_graph libdrgn)
 target_link_libraries(type_graph
   container_info
   symbol_service
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
 target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/tools/config_gen.py b/tools/config_gen.py
index c20ca40..5ac56ff 100755
--- a/tools/config_gen.py
+++ b/tools/config_gen.py
@@ -16,6 +16,7 @@
 
 import argparse
 import getpass
+import os
 import pathlib
 import subprocess
 import typing
@@ -170,7 +171,10 @@ def pull_base_toml() -> typing.Dict:
 
     # Now, we need to replace any placeholders that might be present in the base toml file with the real verisons.
     user = getpass.getuser()
-    pwd = str(repo_path.resolve())
+    if "IN_NIX_SHELL" in os.environ and "src" in os.environ:
+        pwd = os.environ['src']
+    else:
+        pwd = str(repo_path.resolve())
 
     container_list = base.get("types", {}).get("containers")
     if container_list:

From c98017e55a6a903ca4c3327525bd0b00f1a20302 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 14 Aug 2024 21:19:14 +0100
Subject: [PATCH 187/188] nix: add building oid to the flake

---
 .circleci/config.yml                       |  18 +-
 .github/workflows/ci.yml                   |  53 +++++
 .github/workflows/object-introspection.yml |  15 --
 .github/workflows/test-report.yml          |  20 ++
 CMakeLists.txt                             | 230 ++++++++++++---------
 README.md                                  |  58 ++++++
 cmake/Finduring.cmake                      |  27 +++
 flake.nix                                  |  96 ++++++++-
 oi/CMakeLists.txt                          |   3 -
 oi/type_graph/CMakeLists.txt               |   3 -
 tools/config_gen.py                        |   6 +-
 11 files changed, 402 insertions(+), 127 deletions(-)
 create mode 100644 .github/workflows/ci.yml
 delete mode 100644 .github/workflows/object-introspection.yml
 create mode 100644 .github/workflows/test-report.yml
 create mode 100644 cmake/Finduring.cmake

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 589245a..dba98d7 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -83,20 +83,26 @@ jobs:
               libgtest-dev \
               libjemalloc-dev \
               libmsgpack-dev \
+              libomp-12-dev \
+              liburing-dev \
               libzstd-dev \
               llvm-15-dev \
               ninja-build \
               pkg-config \
               python3-setuptools
             pip3 install toml
+
+            # Ubuntu 22.04 CMake is too old and we don't have a newer image yet
+            git clone --depth 1 --branch v3.30.2 https://github.com/Kitware/CMake.git /tmp/cmake
+            (cd /tmp/cmake && cmake -B build/ -G Ninja && cmake --build build/)
           environment:
             DEBIAN_FRONTEND: noninteractive
       - checkout
       - run:
           name: Build
           command: |
-            cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
-            cmake --build build/
+            /tmp/cmake/build/bin/cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
+            ninja -C build/
             # Testing rubbish:
             cp test/ci.oid.toml build/testing.oid.toml
       - persist_to_workspace:
@@ -131,10 +137,12 @@ jobs:
             sudo apt-get install -y \
               clang-15 \
               libboost-all-dev \
-              libgflags-dev \
-              llvm-15-dev \
               libfmt-dev \
-              libjemalloc-dev
+              libgflags-dev \
+              libgoogle-glog-dev \
+              libjemalloc-dev \
+              libomp-12-dev \
+              llvm-15-dev
           environment:
             DEBIAN_FRONTEND: noninteractive
       - run:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..82f523a
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,53 @@
+name: CI
+on:
+  pull_request:
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.7
+    - uses: cachix/install-nix-action@v27
+      with:
+        github_access_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: nix fmt
+      run: |-
+        nix --experimental-features 'nix-command flakes' fmt
+        git diff --exit-code
+
+  build-test:
+    runs-on: 16-core-ubuntu
+    strategy:
+      matrix:
+        llvm_version: [15]
+    steps:
+    - uses: actions/checkout@v4.1.7
+    - uses: cachix/install-nix-action@v27
+      with:
+        github_access_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: build (LLVM ${{ matrix.llvm_version }})
+      # Run the build manually in `nix develop` to keep non-outputs around
+      run: |
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ninja -C build
+    - name: test (LLVM ${{ matrix.llvm_version }})
+      env:
+        # disable drgn multithreading as tests are already run in parallel
+        OMP_NUM_THREADS: 1
+      run: |
+        echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ./tools/config_gen.py -c clang++ build/testing.oid.toml
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ctest \
+          --test-dir build/test/ \
+          --test-action Test \
+          --parallel \
+          --no-compress-output \
+          --schedule-random \
+          --timeout 60 \
+          --repeat until-pass:2 \
+          --output-junit results.xml
+    - name: upload results
+      uses: actions/upload-artifact@v4
+      if: success() || failure()
+      with:
+        name: test-results
+        path: build/test/results.xml
diff --git a/.github/workflows/object-introspection.yml b/.github/workflows/object-introspection.yml
deleted file mode 100644
index 70b41aa..0000000
--- a/.github/workflows/object-introspection.yml
+++ /dev/null
@@ -1,15 +0,0 @@
-name: facebookexperimental/object-introspection
-on:
-  pull_request:
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: cachix/install-nix-action@v25
-      with:
-        github_access_token: ${{ secrets.GITHUB_TOKEN }}
-    - name: nix fmt
-      run: |-
-        nix --experimental-features 'nix-command flakes' fmt
-        git diff --exit-code
diff --git a/.github/workflows/test-report.yml b/.github/workflows/test-report.yml
new file mode 100644
index 0000000..f8b62b9
--- /dev/null
+++ b/.github/workflows/test-report.yml
@@ -0,0 +1,20 @@
+name: 'Test Report'
+on:
+  workflow_run:
+    workflows: ['CI']
+    types:
+      - completed
+permissions:
+  contents: read
+  actions: read
+  checks: write
+jobs:
+  report:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: dorny/test-reporter@v1
+      with:
+        artifact: test-results
+        name: CTest Tests
+        path: 'build/test/*.xml'
+        reporter: jest-junit
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8674d46..426b68c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,5 +1,5 @@
 # object-introspection
-cmake_minimum_required(VERSION 3.20)
+cmake_minimum_required(VERSION 3.24)
 project(object-introspection)
 
 # Lets find_program() locate SETUID binaries
@@ -33,75 +33,108 @@ find_package(gflags REQUIRED)
 ### tomlplusplus (for configuration files)
 FetchContent_Declare(
   tomlplusplus
-  GIT_REPOSITORY https://github.com/marzer/tomlplusplus.git
-  GIT_TAG        4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
-  GIT_PROGRESS   TRUE
+  GIT_REPOSITORY    https://github.com/marzer/tomlplusplus.git
+  GIT_TAG           4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
 FetchContent_MakeAvailable(tomlplusplus)
 
 ### glog
-FetchContent_Declare(
-  glog
-  GIT_REPOSITORY https://github.com/google/glog.git
-  GIT_TAG        96a2f23dca4cc7180821ca5f32e526314395d26a
-  GIT_PROGRESS   TRUE
-)
-FetchContent_MakeAvailable(glog)
+find_package(glog)
+if (NOT glog_FOUND)
+  FetchContent_Declare(
+    glog
+    GIT_REPOSITORY    https://github.com/google/glog.git
+    GIT_TAG           96a2f23dca4cc7180821ca5f32e526314395d26a
+    GIT_PROGRESS      TRUE
+    FIND_PACKAGE_ARGS
+  )
+  FetchContent_MakeAvailable(glog)
 
-# These glog executable targets still generate warnings - disable warnings for
-# them explicitly
-target_compile_options(demangle_unittest PRIVATE "-w")
-target_compile_options(logging_unittest PRIVATE "-w")
-target_compile_options(stl_logging_unittest PRIVATE "-w")
-target_compile_options(symbolize_unittest PRIVATE "-w")
-target_compile_options(utilities_unittest PRIVATE "-w")
+  # These glog executable targets still generate warnings - disable warnings for
+  # them explicitly
+  target_compile_options(demangle_unittest PRIVATE "-w")
+  target_compile_options(logging_unittest PRIVATE "-w")
+  target_compile_options(stl_logging_unittest PRIVATE "-w")
+  target_compile_options(symbolize_unittest PRIVATE "-w")
+  target_compile_options(utilities_unittest PRIVATE "-w")
+endif()
 
-### googletest
+### GTest
 # Do this in the main file so it can be fetched before setting project warnings.
 # After this is fixed with FetchContent, move to test/CMakeLists.txt.
 FetchContent_Declare(
-  googletest
-  GIT_REPOSITORY https://github.com/google/googletest.git
-  GIT_TAG        1ed6a8c67a0bd675149ece27bbec0ef1759854cf
-  GIT_PROGRESS   TRUE
+  GTest
+  GIT_REPOSITORY    https://github.com/google/googletest.git
+  GIT_TAG           1ed6a8c67a0bd675149ece27bbec0ef1759854cf
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
-FetchContent_MakeAvailable(googletest)
+FetchContent_MakeAvailable(GTest)
+
+### liburing (for RocksDB)
+find_package(uring REQUIRED)
 
 ### rocksdb
-FetchContent_Declare(
-  rocksdb
-  GIT_REPOSITORY https://github.com/facebook/rocksdb.git
-  GIT_TAG        f32521662acf3352397d438b732144c7813bbbec # v8.5.3
-  GIT_PROGRESS   TRUE
-)
-FetchContent_Populate(rocksdb)
-add_custom_target(librocksdb ALL
-  WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
-  COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
-  COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
-  BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
-  COMMENT "Building RocksDB"
-  USES_TERMINAL
-)
-include_directories(SYSTEM "${rocksdb_SOURCE_DIR}/include")
+find_package(RocksDB 8.11 CONFIG)
+if (NOT RocksDB_FOUND)
+  FetchContent_Declare(
+    rocksdb
+    GIT_REPOSITORY https://github.com/facebook/rocksdb.git
+    GIT_TAG        f32521662acf3352397d438b732144c7813bbbec # v8.5.3
+    GIT_PROGRESS   TRUE
+  )
+  FetchContent_Populate(rocksdb)
+
+  add_custom_target(librocksdb_build ALL
+    WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
+    COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
+    COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
+    BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
+    COMMENT "Building RocksDB"
+    USES_TERMINAL
+  )
+
+  ### zstd (for rocksdb)
+  find_package(zstd REQUIRED)
+
+  add_library(librocksdb INTERFACE)
+  add_dependencies(librocksdb librocksdb_build)
+  target_include_directories(librocksdb INTERFACE SYSTEM "${rocksdb_SOURCE_DIR}/include")
+  target_link_libraries(librocksdb INTERFACE ${rocksdb_BINARY_DIR}/librocksdb.a zstd::zstd)
+
+  add_library(RocksDB::rocksdb ALIAS librocksdb)
+endif()
 
 ### folly
 ### use folly as a header only library. some features won't be supported.
-FetchContent_Declare(
-  folly
-  GIT_REPOSITORY https://github.com/facebook/folly.git
-  GIT_TAG        c5aa5c46291a27f69acc920894d43605ceb43eba
-  GIT_PROGRESS   TRUE
-  PATCH_COMMAND  ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
-)
-FetchContent_Populate(folly)
+find_package(folly CONFIG)
+if (NOT folly_FOUND)
+  FetchContent_Declare(
+    folly
+    GIT_REPOSITORY    https://github.com/facebook/folly.git
+    GIT_TAG           c5aa5c46291a27f69acc920894d43605ceb43eba
+    GIT_PROGRESS      TRUE
+    PATCH_COMMAND     ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
+  )
+  FetchContent_Populate(folly)
+
+  add_library(folly_headers INTERFACE)
+  target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
+  target_link_libraries(folly_headers INTERFACE Boost::headers)
+else()
+  add_library(folly_headers INTERFACE)
+  target_include_directories(folly_headers SYSTEM INTERFACE ${FOLLY_INCLUDE_DIR})
+endif()
 
 ### range-v3
 FetchContent_Declare(
   range-v3
-  GIT_REPOSITORY https://github.com/ericniebler/range-v3.git
-  GIT_TAG        a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
-  GIT_PROGRESS   TRUE
+  GIT_REPOSITORY    https://github.com/ericniebler/range-v3.git
+  GIT_TAG           a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
 FetchContent_MakeAvailable(range-v3)
 
@@ -124,6 +157,18 @@ endif()
 ### (Re)download submodules
 find_package(Git QUIET)
 
+if (DEFINED drgn_SOURCE_DIR)
+    # drgn's autotools build requires source modification. in case we have a
+    # readonly source (read: nix) we must copy this. do this always to avoid
+    # polluting the source.
+    file(COPY "${drgn_SOURCE_DIR}/" DESTINATION "${FETCHCONTENT_BASE_DIR}/drgn-src" NO_SOURCE_PERMISSIONS)
+    SET(drgn_SOURCE_DIR "${FETCHCONTENT_BASE_DIR}/drgn-src")
+    SET(drgn_BINARY_DIR "${CMAKE_BINARY_DIR}/_deps/drgn-build")
+else()
+    SET(drgn_SOURCE_DIR "${PROJECT_SOURCE_DIR}/extern/drgn")
+    SET(drgn_BINARY_DIR "${drgn_SOURCE_DIR}/build")
+endif()
+
 # TODO: No idea if this huge block is required, just picked from an example. There may be a short-hand.
 if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
 # Update submodules as needed
@@ -145,20 +190,17 @@ if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
         if(NOT GIT_SUBMOD_RESULT EQUAL "0")
             message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules")
         endif()
+
+        if(NOT EXISTS "${drgn_SOURCE_DIR}")
+            message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
+        endif()
     endif()
 endif()
 
-if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn")
-    message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
-endif()
 
 ### Select Python version
 find_program(PYTHON NAMES python3.9 python3)
 
-add_library(folly_headers INTERFACE)
-target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
-target_link_libraries(folly_headers INTERFACE Boost::headers)
-
 ### bison & flex (for oid_parser)
 find_package(BISON 3.5 REQUIRED)
 find_package(FLEX)
@@ -194,8 +236,11 @@ find_package(msgpack REQUIRED CONFIG)
 get_target_property(MSGPACK_INCLUDE_DIRS msgpackc INTERFACE_INCLUDE_DIRECTORIES)
 include_directories(SYSTEM ${MSGPACK_INCLUDE_DIRS})
 
-### zstd (for rocksdb)
-find_package(zstd REQUIRED)
+### drgn/elfutils dependencies
+find_package(BZip2 REQUIRED)
+find_package(OpenMP REQUIRED)
+find_package(LibLZMA REQUIRED)
+find_package(ZLIB REQUIRED)
 
 ### drgn
 # The setup.py script in drgn is really meant to build drgn (python
@@ -214,46 +259,38 @@ find_package(zstd REQUIRED)
 # make
 #
 # Since setup.py has a single cmd to do this, just use it for now.
-#
-# Another extemely annoying point. drgn pretty much has to be compiled with gcc only
-# clang-12 does NOT work. clang fails with the following error :-
-# configure: error: gcc with GNU99 support required
 
 set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no")
 if (ASAN)
   list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes")
 endif()
-add_custom_target(libdrgn ALL
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn
-  COMMAND unset BISON_PKGDATADIR && CC=gcc CFLAGS="${DRGN_CFLAGS}" CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp build
-  BYPRODUCTS ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/.libs/libdrgnimpl.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdw/libdw.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libelf/libelf.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdwelf/libdwelf.a
+
+add_custom_target(drgn_build ALL
+  WORKING_DIRECTORY ${drgn_SOURCE_DIR}
+  COMMAND unset BISON_PKGDATADIR && CFLAGS="${DRGN_CFLAGS}"
+  CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp "${drgn_BINARY_DIR}"
+  BYPRODUCTS ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
+    ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
+    ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
+    ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
   COMMENT "Building drgn"
   USES_TERMINAL
 )
-set(DRGN_PATH "${PROJECT_SOURCE_DIR}/extern/drgn/build")
 
-# Ideally drgn stuff should be together at the end. But looks like rpath needs
-# to be set before add_executable() unfortunately. Maybe split libdrgn stuff
-# into a separate file later.
-set(CMAKE_SKIP_BUILD_RPATH FALSE)
-set(CMAKE_INSTALL_RPATH
-  "${DRGN_PATH}/.libs"
-  "${DRGN_PATH}/velfutils/libdw"
-  "${DRGN_PATH}/velfutils/libelf"
-  "${DRGN_PATH}/velfutils/libdwelf"
-)
-set(CMAKE_BUILD_RPATH
-  "${DRGN_PATH}/.libs"
-  "${DRGN_PATH}/velfutils/libdw"
-  "${DRGN_PATH}/velfutils/libelf"
-  "${DRGN_PATH}/velfutils/libdwelf"
-)
-set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+add_library(drgn INTERFACE)
+add_dependencies(drgn drgn_build)
+target_link_libraries(drgn INTERFACE
+  ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
+  ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
+  ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
+  ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
 
-include_directories(SYSTEM "${DRGN_PATH}")
+  BZip2::BZip2
+  LibLZMA::LibLZMA
+  OpenMP::OpenMP_CXX
+  ZLIB::ZLIB
+)
+target_include_directories(drgn SYSTEM INTERFACE "${drgn_SOURCE_DIR}" "${drgn_BINARY_DIR}")
 
 if (STATIC_LINK)
   # glog links against the `gflags` target, which is an alias for `gflags_shared`
@@ -289,7 +326,6 @@ add_library(oicore
   oi/PaddingHunter.cpp
   oi/Serialize.cpp
 )
-add_dependencies(oicore libdrgn)
 target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -323,7 +359,6 @@ else()
 endif()
 
 target_link_libraries(oicore
-  "-L${DRGN_PATH}/.libs"
   drgn
   dw
   pthread
@@ -334,11 +369,9 @@ add_library(treebuilder
   oi/TreeBuilder.cpp
   oi/exporters/TypeCheckingWalker.cpp
 )
-add_dependencies(treebuilder librocksdb)
 target_link_libraries(treebuilder
-  ${rocksdb_BINARY_DIR}/librocksdb.a
+  RocksDB::rocksdb
   oicore # overkill but it does need a lot of stuff
-  zstd::zstd
 )
 
 
@@ -375,10 +408,8 @@ target_link_libraries(oip oicore)
 
 ### Object Introspection RocksDB Printer (OIRP)
 add_executable(oirp tools/OIRP.cpp)
-add_dependencies(oirp librocksdb)
 target_link_libraries(oirp
-  ${rocksdb_BINARY_DIR}/librocksdb.a
-  zstd::zstd
+  RocksDB::rocksdb
   msgpackc
 )
 
@@ -417,3 +448,6 @@ endif()
 if (DEFINED ENV{CMAKE_HOOK})
   include($ENV{CMAKE_HOOK})
 endif()
+
+install(TARGETS oid DESTINATION ${CMAKE_INSTALL_BINDIR})
+
diff --git a/README.md b/README.md
index 9f95bf8..abce6f8 100644
--- a/README.md
+++ b/README.md
@@ -14,3 +14,61 @@ See the [CONTRIBUTING](CONTRIBUTING.md) file for how to help out.
 
 ## License
 Object Introspection is licensed under the [Apache 2.0 License](LICENSE).
+
+## Getting started with Nix
+
+Nix is the easiest way to get started with `oid` as it is non-trivial to build otherwise. Explicit Nix support for Object Introspection as a Library will come down the line, but Nix can currently provide you a reproducible development environment in which to build it.
+
+These examples expect you to have `nix` installed and available with no other dependencies required. Find the installation guide at https://nixos.org/download.html.
+
+We also required flake support. To enable flakes globally run:
+
+    $ mkdir -p ~/.config/nix
+    $ echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf
+
+Or suffix every `nix` command with `nix --extra-experimental-features 'nix-command flakes'`.
+
+### Run upstream OID without modifying the source
+
+    $ nix run github:facebookexperimental/object-introspection -- --help
+
+This will download the latest source into your Nix store along with all of its dependencies, running help afterwards.
+
+### Build OID locally
+
+    $ git clone https://github.com/facebookexperimental/object-introspection
+    $ nix build
+    $ ./result/bin/oid --help
+
+This will build OID from your local sources. Please note that this will NOT pick up changes to `extern/drgn` or `extern/drgn/libdrgn/velfutils`.
+
+### Get a development environment
+
+    $ nix develop
+    $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+    $ ninja -C build
+    $ build/oid --help
+
+This command provides a development shell with all the required dependencies. This is the most flexible option and will pick up source changes as CMake normally would.
+
+Sometimes this developer environment can be polluted by things installed on your normal system. If this is an issue, use:
+
+    $ nix develop -i
+
+This removes the environment from your host system and makes the build pure.
+
+### Run the tests
+
+    $ nix develop
+    $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+    $ ninja -C build
+    $ ./tools/config_gen.py -c clang++ build/testing.oid.toml
+    $ ctest -j --test-dir build/test
+
+Running tests under `nix` is new to the project and may take some time to mature. The CI is the source of truth for now.
+
+### Format source
+
+    $ nix fmt
+
+This formats the Nix, C++, and Python code in the repository.
diff --git a/cmake/Finduring.cmake b/cmake/Finduring.cmake
new file mode 100644
index 0000000..3b007fe
--- /dev/null
+++ b/cmake/Finduring.cmake
@@ -0,0 +1,27 @@
+# - Find liburing
+#
+# uring_INCLUDE_DIR - Where to find liburing.h
+# uring_LIBRARIES - List of libraries when using uring.
+# uring_FOUND - True if uring found.
+
+find_path(uring_INCLUDE_DIR
+  NAMES liburing.h)
+find_library(uring_LIBRARIES
+  NAMES liburing.a liburing.so)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(uring
+  DEFAULT_MSG uring_LIBRARIES uring_INCLUDE_DIR)
+
+mark_as_advanced(
+  uring_INCLUDE_DIR
+  uring_LIBRARIES)
+
+if(uring_FOUND AND NOT TARGET uring::uring)
+  add_library(uring::uring UNKNOWN IMPORTED)
+  set_target_properties(uring::uring PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${uring_INCLUDE_DIR}"
+    IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+    IMPORTED_LOCATION "${uring_LIBRARIES}")
+endif()
+
diff --git a/flake.nix b/flake.nix
index 7a3382e..74ed25a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,5 +1,5 @@
 {
-  description = "A flake for building Object Introspection.";
+  description = "Object level memory profiler for C++";
 
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
@@ -18,7 +18,99 @@
       treefmt-nix,
       ...
     }@inputs:
-    flake-utils.lib.eachDefaultSystem (
+    flake-utils.lib.eachSystem [ flake-utils.lib.system.x86_64-linux ] (
+      system:
+      let
+        pkgs = import nixpkgs { inherit system; };
+
+        drgnSrc = pkgs.fetchFromGitHub {
+          owner = "JakeHillion";
+          repo = "drgn";
+          rev = "b1f8c3e8526611b6720800250ba858a713dd9e4f";
+          hash = "sha256-5WhMHgx/RKtqjxGx4AyiqVKMot5xulr+6c8i2E9IxiA=";
+          fetchSubmodules = true;
+        };
+
+        mkOidPackage =
+          llvmPackages:
+          with pkgs;
+          pkgs.llvmPackages.stdenv.mkDerivation rec {
+            name = "oid";
+
+            src = self;
+
+            nativeBuildInputs = [
+              autoconf
+              automake
+              bison
+              cmake
+              flex
+              gettext
+              git
+              hexdump
+              libtool
+              ninja
+              pkgconf
+              python312
+              python312Packages.setuptools
+              python312Packages.toml
+              glibcLocales
+            ];
+
+            buildInputs = [
+              llvmPackages.libclang
+              llvmPackages.llvm
+
+              boost
+              bzip2
+              curl
+              double-conversion
+              elfutils
+              flex
+              folly
+              folly.fmt
+              gflags
+              glog
+              gtest
+              icu
+              jemalloc
+              libarchive
+              libmicrohttpd
+              liburing
+              libxml2
+              lzma
+              msgpack
+              range-v3
+              rocksdb_8_11
+              sqlite
+              tomlplusplus
+              zstd
+
+              llvmPackages_15.openmp # should match the stdenv clang version, see: https://github.com/NixOS/nixpkgs/issues/79818
+            ];
+
+            cmakeFlags = [
+              "-Ddrgn_SOURCE_DIR=${drgnSrc}"
+              "-DFORCE_BOOST_STATIC=Off"
+            ];
+
+            outputs = [ "out" ];
+          };
+      in
+      {
+        packages = rec {
+          default = oid-llvm15;
+
+          oid-llvm15 = mkOidPackage pkgs.llvmPackages_15;
+        };
+
+        apps.default = {
+          type = "app";
+          program = "${self.packages.${system}.default}/bin/oid";
+        };
+      }
+    )
+    // flake-utils.lib.eachDefaultSystem (
       system:
       let
         pkgs = nixpkgs.legacyPackages.${system};
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index bb73871..7d805e5 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus)
 add_library(drgn_utils DrgnUtils.cpp)
 target_link_libraries(drgn_utils
   glog::glog
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
-add_dependencies(drgn_utils libdrgn)
 
 add_library(symbol_service
   Descs.cpp
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 6c4da13..c878e08 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -20,12 +20,9 @@ add_library(type_graph
   TypeIdentifier.cpp
   Types.cpp
 )
-add_dependencies(type_graph libdrgn)
 target_link_libraries(type_graph
   container_info
   symbol_service
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
 target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/tools/config_gen.py b/tools/config_gen.py
index c20ca40..5ac56ff 100755
--- a/tools/config_gen.py
+++ b/tools/config_gen.py
@@ -16,6 +16,7 @@
 
 import argparse
 import getpass
+import os
 import pathlib
 import subprocess
 import typing
@@ -170,7 +171,10 @@ def pull_base_toml() -> typing.Dict:
 
     # Now, we need to replace any placeholders that might be present in the base toml file with the real verisons.
     user = getpass.getuser()
-    pwd = str(repo_path.resolve())
+    if "IN_NIX_SHELL" in os.environ and "src" in os.environ:
+        pwd = os.environ['src']
+    else:
+        pwd = str(repo_path.resolve())
 
     container_list = base.get("types", {}).get("containers")
     if container_list:

From f4ac2388112c7843b1e3193849a71810d2a75a13 Mon Sep 17 00:00:00 2001
From: Jake Hillion <jakehillion@meta.com>
Date: Wed, 14 Aug 2024 21:19:14 +0100
Subject: [PATCH 188/188] nix: add building oid to the flake

---
 .circleci/config.yml                          |  18 +-
 .github/workflows/ci.yml                      |  54 ++++
 .github/workflows/object-introspection.yml    |  15 --
 .github/workflows/test-report.yml             |  20 ++
 .github/workflows/tests_failing_under_nix.txt | 229 +++++++++++++++++
 CMakeLists.txt                                | 230 ++++++++++--------
 README.md                                     |  58 +++++
 cmake/Finduring.cmake                         |  27 ++
 flake.nix                                     |  96 +++++++-
 oi/CMakeLists.txt                             |   3 -
 oi/type_graph/CMakeLists.txt                  |   3 -
 tools/config_gen.py                           |   6 +-
 12 files changed, 632 insertions(+), 127 deletions(-)
 create mode 100644 .github/workflows/ci.yml
 delete mode 100644 .github/workflows/object-introspection.yml
 create mode 100644 .github/workflows/test-report.yml
 create mode 100644 .github/workflows/tests_failing_under_nix.txt
 create mode 100644 cmake/Finduring.cmake

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 589245a..dba98d7 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -83,20 +83,26 @@ jobs:
               libgtest-dev \
               libjemalloc-dev \
               libmsgpack-dev \
+              libomp-12-dev \
+              liburing-dev \
               libzstd-dev \
               llvm-15-dev \
               ninja-build \
               pkg-config \
               python3-setuptools
             pip3 install toml
+
+            # Ubuntu 22.04 CMake is too old and we don't have a newer image yet
+            git clone --depth 1 --branch v3.30.2 https://github.com/Kitware/CMake.git /tmp/cmake
+            (cd /tmp/cmake && cmake -B build/ -G Ninja && cmake --build build/)
           environment:
             DEBIAN_FRONTEND: noninteractive
       - checkout
       - run:
           name: Build
           command: |
-            cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
-            cmake --build build/
+            /tmp/cmake/build/bin/cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
+            ninja -C build/
             # Testing rubbish:
             cp test/ci.oid.toml build/testing.oid.toml
       - persist_to_workspace:
@@ -131,10 +137,12 @@ jobs:
             sudo apt-get install -y \
               clang-15 \
               libboost-all-dev \
-              libgflags-dev \
-              llvm-15-dev \
               libfmt-dev \
-              libjemalloc-dev
+              libgflags-dev \
+              libgoogle-glog-dev \
+              libjemalloc-dev \
+              libomp-12-dev \
+              llvm-15-dev
           environment:
             DEBIAN_FRONTEND: noninteractive
       - run:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..4c0a9eb
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,54 @@
+name: CI
+on:
+  pull_request:
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.7
+    - uses: cachix/install-nix-action@v27
+      with:
+        github_access_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: nix fmt
+      run: |-
+        nix --experimental-features 'nix-command flakes' fmt
+        git diff --exit-code
+
+  build-test:
+    runs-on: 16-core-ubuntu
+    strategy:
+      matrix:
+        llvm_version: [15]
+    steps:
+    - uses: actions/checkout@v4.1.7
+    - uses: cachix/install-nix-action@v27
+      with:
+        github_access_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: build (LLVM ${{ matrix.llvm_version }})
+      # Run the build manually in `nix develop` to keep non-outputs around
+      run: |
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ninja -C build
+    - name: test (LLVM ${{ matrix.llvm_version }})
+      env:
+        # disable drgn multithreading as tests are already run in parallel
+        OMP_NUM_THREADS: 1
+      run: |
+        echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ./tools/config_gen.py -c clang++ build/testing.oid.toml
+        nix develop .#oid-llvm${{ matrix.llvm_version }} --command ctest \
+          --test-dir build/test/ \
+          --test-action Test \
+          --parallel \
+          --no-compress-output \
+          --schedule-random \
+          --timeout 60 \
+          --repeat until-pass:2 \
+          --exclude-from-file .github/workflows/tests_failing_under_nix.txt \
+          --output-junit results.xml
+    - name: upload results
+      uses: actions/upload-artifact@v4
+      if: success() || failure()
+      with:
+        name: test-results
+        path: build/test/results.xml
diff --git a/.github/workflows/object-introspection.yml b/.github/workflows/object-introspection.yml
deleted file mode 100644
index 70b41aa..0000000
--- a/.github/workflows/object-introspection.yml
+++ /dev/null
@@ -1,15 +0,0 @@
-name: facebookexperimental/object-introspection
-on:
-  pull_request:
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: cachix/install-nix-action@v25
-      with:
-        github_access_token: ${{ secrets.GITHUB_TOKEN }}
-    - name: nix fmt
-      run: |-
-        nix --experimental-features 'nix-command flakes' fmt
-        git diff --exit-code
diff --git a/.github/workflows/test-report.yml b/.github/workflows/test-report.yml
new file mode 100644
index 0000000..7918963
--- /dev/null
+++ b/.github/workflows/test-report.yml
@@ -0,0 +1,20 @@
+name: 'Test Report'
+on:
+  workflow_run:
+    workflows: ['CI']
+    types:
+      - completed
+permissions:
+  contents: read
+  actions: read
+  checks: write
+jobs:
+  report:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: dorny/test-reporter@v1
+      with:
+        artifact: test-results
+        name: CTest Tests
+        path: results.xml
+        reporter: jest-junit
diff --git a/.github/workflows/tests_failing_under_nix.txt b/.github/workflows/tests_failing_under_nix.txt
new file mode 100644
index 0000000..adbc514
--- /dev/null
+++ b/.github/workflows/tests_failing_under_nix.txt
@@ -0,0 +1,229 @@
+ClangTypeParserTest.MemberAlignment
+ClangTypeParserTest.SimpleStruct
+OidIntegration.alignment_wrapper_member_alignment
+OidIntegration.alignment_wrapper_member_lower
+OidIntegration.alignment_wrapper_member_override
+OidIntegration.alignment_wrapper_struct
+OidIntegration.alignment_wrapper_two_members
+OidIntegration.alignment_wrapper_union_member
+OidIntegration.anonymous_anon_struct
+OidIntegration.anonymous_anon_typedef
+OidIntegration.anonymous_anon_union
+OidIntegration.anonymous_nested_anon_struct
+OidIntegration.anonymous_regular_struct
+OidIntegration.arrays_member_int0
+OidIntegration.arrays_member_int10
+OidIntegration.arrays_multidim
+OidIntegration.arrays_multidim_legacy
+OidIntegration.bitfields_enum
+OidIntegration.bitfields_mixed
+OidIntegration.bitfields_single
+OidIntegration.bitfields_straddle_bytes
+OidIntegration.bitfields_within_bytes
+OidIntegration.bitfields_zero_bits
+OidIntegration.cycles_raw_ptr
+OidIntegration.cycles_raw_ptr_wrapped
+OidIntegration.cycles_shared_ptr
+OidIntegration.cycles_unique_ptr
+OidIntegration.enums_params_scoped_enum_val
+OidIntegration.enums_params_scoped_enum_val_cast
+OidIntegration.enums_params_scoped_enum_val_gaps
+OidIntegration.enums_params_scoped_enum_val_negative
+OidIntegration.enums_params_unscoped_enum_val_cast
+OidIntegration.fbstring_empty
+OidIntegration.fbstring_heap_allocated
+OidIntegration.fbstring_inline
+OidIntegration.fbstring_string_pooled_unique
+OidIntegration.folly_f14_fast_map_a
+OidIntegration.folly_f14_fast_set_a
+OidIntegration.folly_f14_node_map_a
+OidIntegration.folly_f14_node_set_a
+OidIntegration.folly_f14_value_map_a
+OidIntegration.folly_f14_value_set_a
+OidIntegration.folly_f14_vector_map_a
+OidIntegration.folly_f14_vector_set_a
+OidIntegration.folly_small_vector_int_always_heap
+OidIntegration.folly_small_vector_int_default_empty
+OidIntegration.folly_small_vector_int_default_inlined
+OidIntegration.folly_small_vector_int_default_overflow
+OidIntegration.folly_small_vector_vector_3_empty
+OidIntegration.folly_small_vector_vector_3_inlined
+OidIntegration.folly_small_vector_vector_3_overflow
+OidIntegration.folly_sorted_vector_map_int_int_empty
+OidIntegration.folly_sorted_vector_map_int_int_reserve
+OidIntegration.folly_sorted_vector_map_int_int_some
+OidIntegration.ignored_member
+OidIntegration.ignored_roottype
+OidIntegration.ignored_subtype
+OidIntegration.inheritance_access_private
+OidIntegration.inheritance_access_protected
+OidIntegration.inheritance_access_public
+OidIntegration.inheritance_access_public_as_base
+OidIntegration.inheritance_multiple_a
+OidIntegration.inheritance_polymorphic_a_as_a
+OidIntegration.inheritance_polymorphic_b_as_a
+OidIntegration.inheritance_polymorphic_b_as_b
+OidIntegration.inheritance_polymorphic_c_as_a
+OidIntegration.inheritance_polymorphic_c_as_b
+OidIntegration.inheritance_polymorphic_c_as_c
+OidIntegration.inheritance_polymorphic_diamond_child_as_child
+OidIntegration.inheritance_polymorphic_diamond_child_as_middle1
+OidIntegration.inheritance_polymorphic_diamond_child_as_middle1_root
+OidIntegration.inheritance_polymorphic_diamond_child_as_middle2
+OidIntegration.inheritance_polymorphic_diamond_child_as_middle2_root
+OidIntegration.inheritance_polymorphic_diamond_middle1_as_middle1
+OidIntegration.inheritance_polymorphic_diamond_middle1_as_root
+OidIntegration.inheritance_polymorphic_diamond_middle2_as_middle2
+OidIntegration.inheritance_polymorphic_diamond_middle2_as_root
+OidIntegration.inheritance_polymorphic_diamond_root_as_root
+OidIntegration.inheritance_polymorphic_non_dynamic_base_a_as_a
+OidIntegration.inheritance_polymorphic_non_dynamic_base_a_no_polymorphic
+OidIntegration.inheritance_polymorphic_non_dynamic_base_b_as_a
+OidIntegration.inheritance_polymorphic_non_dynamic_base_b_as_b
+OidIntegration.inheritance_polymorphic_non_dynamic_base_b_no_polymorphic
+OidIntegration.inheritance_polymorphic_non_dynamic_base_c_as_a
+OidIntegration.inheritance_polymorphic_non_dynamic_base_c_as_b
+OidIntegration.inheritance_polymorphic_non_dynamic_base_c_as_c
+OidIntegration.inheritance_polymorphic_non_dynamic_base_c_no_polymorphic
+OidIntegration.multi_arg_tb_all_fail_crashes
+OidIntegration.multi_arg_tb_fail_first_arg
+OidIntegration.namespaces_queue
+OidIntegration.namespaces_stack
+OidIntegration.packed_a
+OidIntegration.padding_bool_padding
+OidIntegration.padding_nested_padding
+OidIntegration.pointers_feature_config
+OidIntegration.pointers_feature_flag_disabled
+OidIntegration.pointers_incomplete_containing_struct
+OidIntegration.pointers_incomplete_containing_struct_no_follow
+OidIntegration.pointers_incomplete_shared_ptr
+OidIntegration.pointers_incomplete_shared_ptr_null
+OidIntegration.pointers_incomplete_unique_ptr
+OidIntegration.pointers_incomplete_unique_ptr_null
+OidIntegration.pointers_struct_primitive_ptrs
+OidIntegration.pointers_struct_primitive_ptrs_no_follow
+OidIntegration.pointers_struct_primitive_ptrs_null
+OidIntegration.pointers_struct_vector_ptr
+OidIntegration.pointers_struct_vector_ptr_no_follow
+OidIntegration.pointers_struct_vector_ptr_null
+OidIntegration.pointers_vector_of_pointers
+OidIntegration.primitives_long_double
+OidIntegration.simple_class
+OidIntegration.simple_struct
+OidIntegration.simple_union
+OidIntegration.sorted_vector_set_no_ints
+OidIntegration.sorted_vector_set_some_ints
+OidIntegration.std_array_uint64_length_0
+OidIntegration.std_array_uint64_length_1
+OidIntegration.std_array_uint64_length_8
+OidIntegration.std_array_vector_length_1
+OidIntegration.std_array_vector_length_2
+OidIntegration.std_conditional_a
+OidIntegration.std_deque_del_allocator_a
+OidIntegration.std_deque_deque_int_empty
+OidIntegration.std_deque_deque_int_some
+OidIntegration.std_deque_int_empty
+OidIntegration.std_deque_int_some
+OidIntegration.std_list_del_allocator_a
+OidIntegration.std_list_int_empty
+OidIntegration.std_list_int_some
+OidIntegration.std_list_list_int_empty
+OidIntegration.std_list_list_int_some
+OidIntegration.std_list_struct_some
+OidIntegration.std_map_custom_comparator_a
+OidIntegration.std_multimap_custom_comparator_a
+OidIntegration.std_multiset_custom_comparator_a
+OidIntegration.std_optional_uint64_empty
+OidIntegration.std_optional_uint64_present
+OidIntegration.std_optional_vector_empty
+OidIntegration.std_optional_vector_present
+OidIntegration.std_pair_uint64_uint32
+OidIntegration.std_pair_uint64_uint64
+OidIntegration.std_pair_vector_vector
+OidIntegration.std_priority_queue_adapter_deque_empty
+OidIntegration.std_priority_queue_adapter_deque_some
+OidIntegration.std_priority_queue_int_empty
+OidIntegration.std_priority_queue_int_some
+OidIntegration.std_queue_adapter_vector_empty
+OidIntegration.std_queue_adapter_vector_some
+OidIntegration.std_queue_int_empty
+OidIntegration.std_queue_int_some
+OidIntegration.std_queue_queue_int_empty
+OidIntegration.std_queue_queue_int_some
+OidIntegration.std_reference_wrapper_int
+OidIntegration.std_reference_wrapper_vector
+OidIntegration.std_set_custom_comparator_a
+OidIntegration.std_smart_ptr_shared_ptr_const_uint64_empty
+OidIntegration.std_smart_ptr_shared_ptr_const_vector_empty
+OidIntegration.std_smart_ptr_shared_ptr_uint64_empty
+OidIntegration.std_smart_ptr_shared_ptr_uint64_present
+OidIntegration.std_smart_ptr_shared_ptr_vector_empty
+OidIntegration.std_smart_ptr_shared_ptr_vector_present
+OidIntegration.std_smart_ptr_shared_ptr_void_empty
+OidIntegration.std_smart_ptr_shared_ptr_void_present
+OidIntegration.std_smart_ptr_unique_ptr_const_uint64_empty
+OidIntegration.std_smart_ptr_unique_ptr_const_vector_empty
+OidIntegration.std_smart_ptr_unique_ptr_uint64_empty
+OidIntegration.std_smart_ptr_unique_ptr_uint64_present
+OidIntegration.std_smart_ptr_unique_ptr_vector_empty
+OidIntegration.std_smart_ptr_unique_ptr_vector_present
+OidIntegration.std_smart_ptr_unique_ptr_void_empty
+OidIntegration.std_smart_ptr_unique_ptr_void_present
+OidIntegration.std_smart_ptr_weak_ptr_int64_empty
+OidIntegration.std_smart_ptr_weak_ptr_int64_expired
+OidIntegration.std_smart_ptr_weak_ptr_int64_expired_chase
+OidIntegration.std_smart_ptr_weak_ptr_int64_present
+OidIntegration.std_smart_ptr_weak_ptr_int64_present_chase
+OidIntegration.std_smart_ptr_weak_ptr_int64_void_empty
+OidIntegration.std_stack_adapter_vector_empty
+OidIntegration.std_stack_adapter_vector_some
+OidIntegration.std_stack_int_empty
+OidIntegration.std_stack_int_some
+OidIntegration.std_stack_stack_int_empty
+OidIntegration.std_stack_stack_int_some
+OidIntegration.std_string_empty
+OidIntegration.std_string_heap_allocated
+OidIntegration.std_string_sso
+OidIntegration.std_tuple_uint64_uint64
+OidIntegration.std_unordered_map_custom_operator_a
+OidIntegration.std_unordered_multimap_custom_operator_a
+OidIntegration.std_unordered_multiset_custom_operator_a
+OidIntegration.std_unordered_set_custom_operator_a
+OidIntegration.std_variant_256_params_256
+OidIntegration.std_variant_256_params_empty
+OidIntegration.std_variant_char_int64_1
+OidIntegration.std_variant_char_int64_2
+OidIntegration.std_variant_empty
+OidIntegration.std_variant_optional
+OidIntegration.std_variant_vector_int_1
+OidIntegration.std_variant_vector_int_2
+OidIntegration.std_vector_del_allocator_a
+OidIntegration.std_vector_int_empty
+OidIntegration.std_vector_int_some
+OidIntegration.std_vector_reserve
+OidIntegration.std_vector_struct_some
+OidIntegration.std_vector_vector_int_empty
+OidIntegration.std_vector_vector_int_some
+OidIntegration.templates_int
+OidIntegration.templates_two
+OidIntegration.templates_value
+OidIntegration.templates_vector
+OidIntegration.typedefed_parent_multilevel_typedef_parent
+OidIntegration.typedefed_parent_simple_typedef_parent
+OidIntegration.typedefs_anonymous
+OidIntegration.typedefs_container
+OidIntegration.unions_alignment
+OidIntegration.unions_int
+OidIntegration.unions_tagged_int
+OidIntegration.unions_tagged_unordered_map
+OidIntegration.unions_tagged_vector
+OidIntegration.unions_unordered_map
+OidIntegration.unions_vector
+OilIntegration.folly_f14_fast_map_a
+OilIntegration.folly_f14_fast_set_a
+OilIntegration.folly_f14_node_map_a
+OilIntegration.folly_f14_node_set_a
+OilIntegration.folly_f14_value_map_a
+OilIntegration.folly_f14_value_set_a
+OilIntegration.folly_f14_vector_map_a
+OilIntegration.folly_f14_vector_set_a
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8674d46..426b68c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,5 +1,5 @@
 # object-introspection
-cmake_minimum_required(VERSION 3.20)
+cmake_minimum_required(VERSION 3.24)
 project(object-introspection)
 
 # Lets find_program() locate SETUID binaries
@@ -33,75 +33,108 @@ find_package(gflags REQUIRED)
 ### tomlplusplus (for configuration files)
 FetchContent_Declare(
   tomlplusplus
-  GIT_REPOSITORY https://github.com/marzer/tomlplusplus.git
-  GIT_TAG        4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
-  GIT_PROGRESS   TRUE
+  GIT_REPOSITORY    https://github.com/marzer/tomlplusplus.git
+  GIT_TAG           4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
 FetchContent_MakeAvailable(tomlplusplus)
 
 ### glog
-FetchContent_Declare(
-  glog
-  GIT_REPOSITORY https://github.com/google/glog.git
-  GIT_TAG        96a2f23dca4cc7180821ca5f32e526314395d26a
-  GIT_PROGRESS   TRUE
-)
-FetchContent_MakeAvailable(glog)
+find_package(glog)
+if (NOT glog_FOUND)
+  FetchContent_Declare(
+    glog
+    GIT_REPOSITORY    https://github.com/google/glog.git
+    GIT_TAG           96a2f23dca4cc7180821ca5f32e526314395d26a
+    GIT_PROGRESS      TRUE
+    FIND_PACKAGE_ARGS
+  )
+  FetchContent_MakeAvailable(glog)
 
-# These glog executable targets still generate warnings - disable warnings for
-# them explicitly
-target_compile_options(demangle_unittest PRIVATE "-w")
-target_compile_options(logging_unittest PRIVATE "-w")
-target_compile_options(stl_logging_unittest PRIVATE "-w")
-target_compile_options(symbolize_unittest PRIVATE "-w")
-target_compile_options(utilities_unittest PRIVATE "-w")
+  # These glog executable targets still generate warnings - disable warnings for
+  # them explicitly
+  target_compile_options(demangle_unittest PRIVATE "-w")
+  target_compile_options(logging_unittest PRIVATE "-w")
+  target_compile_options(stl_logging_unittest PRIVATE "-w")
+  target_compile_options(symbolize_unittest PRIVATE "-w")
+  target_compile_options(utilities_unittest PRIVATE "-w")
+endif()
 
-### googletest
+### GTest
 # Do this in the main file so it can be fetched before setting project warnings.
 # After this is fixed with FetchContent, move to test/CMakeLists.txt.
 FetchContent_Declare(
-  googletest
-  GIT_REPOSITORY https://github.com/google/googletest.git
-  GIT_TAG        1ed6a8c67a0bd675149ece27bbec0ef1759854cf
-  GIT_PROGRESS   TRUE
+  GTest
+  GIT_REPOSITORY    https://github.com/google/googletest.git
+  GIT_TAG           1ed6a8c67a0bd675149ece27bbec0ef1759854cf
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
-FetchContent_MakeAvailable(googletest)
+FetchContent_MakeAvailable(GTest)
+
+### liburing (for RocksDB)
+find_package(uring REQUIRED)
 
 ### rocksdb
-FetchContent_Declare(
-  rocksdb
-  GIT_REPOSITORY https://github.com/facebook/rocksdb.git
-  GIT_TAG        f32521662acf3352397d438b732144c7813bbbec # v8.5.3
-  GIT_PROGRESS   TRUE
-)
-FetchContent_Populate(rocksdb)
-add_custom_target(librocksdb ALL
-  WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
-  COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
-  COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
-  BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
-  COMMENT "Building RocksDB"
-  USES_TERMINAL
-)
-include_directories(SYSTEM "${rocksdb_SOURCE_DIR}/include")
+find_package(RocksDB 8.11 CONFIG)
+if (NOT RocksDB_FOUND)
+  FetchContent_Declare(
+    rocksdb
+    GIT_REPOSITORY https://github.com/facebook/rocksdb.git
+    GIT_TAG        f32521662acf3352397d438b732144c7813bbbec # v8.5.3
+    GIT_PROGRESS   TRUE
+  )
+  FetchContent_Populate(rocksdb)
+
+  add_custom_target(librocksdb_build ALL
+    WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
+    COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
+    COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
+    BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
+    COMMENT "Building RocksDB"
+    USES_TERMINAL
+  )
+
+  ### zstd (for rocksdb)
+  find_package(zstd REQUIRED)
+
+  add_library(librocksdb INTERFACE)
+  add_dependencies(librocksdb librocksdb_build)
+  target_include_directories(librocksdb INTERFACE SYSTEM "${rocksdb_SOURCE_DIR}/include")
+  target_link_libraries(librocksdb INTERFACE ${rocksdb_BINARY_DIR}/librocksdb.a zstd::zstd)
+
+  add_library(RocksDB::rocksdb ALIAS librocksdb)
+endif()
 
 ### folly
 ### use folly as a header only library. some features won't be supported.
-FetchContent_Declare(
-  folly
-  GIT_REPOSITORY https://github.com/facebook/folly.git
-  GIT_TAG        c5aa5c46291a27f69acc920894d43605ceb43eba
-  GIT_PROGRESS   TRUE
-  PATCH_COMMAND  ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
-)
-FetchContent_Populate(folly)
+find_package(folly CONFIG)
+if (NOT folly_FOUND)
+  FetchContent_Declare(
+    folly
+    GIT_REPOSITORY    https://github.com/facebook/folly.git
+    GIT_TAG           c5aa5c46291a27f69acc920894d43605ceb43eba
+    GIT_PROGRESS      TRUE
+    PATCH_COMMAND     ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
+  )
+  FetchContent_Populate(folly)
+
+  add_library(folly_headers INTERFACE)
+  target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
+  target_link_libraries(folly_headers INTERFACE Boost::headers)
+else()
+  add_library(folly_headers INTERFACE)
+  target_include_directories(folly_headers SYSTEM INTERFACE ${FOLLY_INCLUDE_DIR})
+endif()
 
 ### range-v3
 FetchContent_Declare(
   range-v3
-  GIT_REPOSITORY https://github.com/ericniebler/range-v3.git
-  GIT_TAG        a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
-  GIT_PROGRESS   TRUE
+  GIT_REPOSITORY    https://github.com/ericniebler/range-v3.git
+  GIT_TAG           a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
+  GIT_PROGRESS      TRUE
+  FIND_PACKAGE_ARGS
 )
 FetchContent_MakeAvailable(range-v3)
 
@@ -124,6 +157,18 @@ endif()
 ### (Re)download submodules
 find_package(Git QUIET)
 
+if (DEFINED drgn_SOURCE_DIR)
+    # drgn's autotools build requires source modification. in case we have a
+    # readonly source (read: nix) we must copy this. do this always to avoid
+    # polluting the source.
+    file(COPY "${drgn_SOURCE_DIR}/" DESTINATION "${FETCHCONTENT_BASE_DIR}/drgn-src" NO_SOURCE_PERMISSIONS)
+    SET(drgn_SOURCE_DIR "${FETCHCONTENT_BASE_DIR}/drgn-src")
+    SET(drgn_BINARY_DIR "${CMAKE_BINARY_DIR}/_deps/drgn-build")
+else()
+    SET(drgn_SOURCE_DIR "${PROJECT_SOURCE_DIR}/extern/drgn")
+    SET(drgn_BINARY_DIR "${drgn_SOURCE_DIR}/build")
+endif()
+
 # TODO: No idea if this huge block is required, just picked from an example. There may be a short-hand.
 if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
 # Update submodules as needed
@@ -145,20 +190,17 @@ if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
         if(NOT GIT_SUBMOD_RESULT EQUAL "0")
             message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules")
         endif()
+
+        if(NOT EXISTS "${drgn_SOURCE_DIR}")
+            message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
+        endif()
     endif()
 endif()
 
-if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn")
-    message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
-endif()
 
 ### Select Python version
 find_program(PYTHON NAMES python3.9 python3)
 
-add_library(folly_headers INTERFACE)
-target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
-target_link_libraries(folly_headers INTERFACE Boost::headers)
-
 ### bison & flex (for oid_parser)
 find_package(BISON 3.5 REQUIRED)
 find_package(FLEX)
@@ -194,8 +236,11 @@ find_package(msgpack REQUIRED CONFIG)
 get_target_property(MSGPACK_INCLUDE_DIRS msgpackc INTERFACE_INCLUDE_DIRECTORIES)
 include_directories(SYSTEM ${MSGPACK_INCLUDE_DIRS})
 
-### zstd (for rocksdb)
-find_package(zstd REQUIRED)
+### drgn/elfutils dependencies
+find_package(BZip2 REQUIRED)
+find_package(OpenMP REQUIRED)
+find_package(LibLZMA REQUIRED)
+find_package(ZLIB REQUIRED)
 
 ### drgn
 # The setup.py script in drgn is really meant to build drgn (python
@@ -214,46 +259,38 @@ find_package(zstd REQUIRED)
 # make
 #
 # Since setup.py has a single cmd to do this, just use it for now.
-#
-# Another extemely annoying point. drgn pretty much has to be compiled with gcc only
-# clang-12 does NOT work. clang fails with the following error :-
-# configure: error: gcc with GNU99 support required
 
 set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no")
 if (ASAN)
   list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes")
 endif()
-add_custom_target(libdrgn ALL
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn
-  COMMAND unset BISON_PKGDATADIR && CC=gcc CFLAGS="${DRGN_CFLAGS}" CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp build
-  BYPRODUCTS ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/.libs/libdrgnimpl.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdw/libdw.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libelf/libelf.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdwelf/libdwelf.a
+
+add_custom_target(drgn_build ALL
+  WORKING_DIRECTORY ${drgn_SOURCE_DIR}
+  COMMAND unset BISON_PKGDATADIR && CFLAGS="${DRGN_CFLAGS}"
+  CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp "${drgn_BINARY_DIR}"
+  BYPRODUCTS ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
+    ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
+    ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
+    ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
   COMMENT "Building drgn"
   USES_TERMINAL
 )
-set(DRGN_PATH "${PROJECT_SOURCE_DIR}/extern/drgn/build")
 
-# Ideally drgn stuff should be together at the end. But looks like rpath needs
-# to be set before add_executable() unfortunately. Maybe split libdrgn stuff
-# into a separate file later.
-set(CMAKE_SKIP_BUILD_RPATH FALSE)
-set(CMAKE_INSTALL_RPATH
-  "${DRGN_PATH}/.libs"
-  "${DRGN_PATH}/velfutils/libdw"
-  "${DRGN_PATH}/velfutils/libelf"
-  "${DRGN_PATH}/velfutils/libdwelf"
-)
-set(CMAKE_BUILD_RPATH
-  "${DRGN_PATH}/.libs"
-  "${DRGN_PATH}/velfutils/libdw"
-  "${DRGN_PATH}/velfutils/libelf"
-  "${DRGN_PATH}/velfutils/libdwelf"
-)
-set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+add_library(drgn INTERFACE)
+add_dependencies(drgn drgn_build)
+target_link_libraries(drgn INTERFACE
+  ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
+  ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
+  ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
+  ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
 
-include_directories(SYSTEM "${DRGN_PATH}")
+  BZip2::BZip2
+  LibLZMA::LibLZMA
+  OpenMP::OpenMP_CXX
+  ZLIB::ZLIB
+)
+target_include_directories(drgn SYSTEM INTERFACE "${drgn_SOURCE_DIR}" "${drgn_BINARY_DIR}")
 
 if (STATIC_LINK)
   # glog links against the `gflags` target, which is an alias for `gflags_shared`
@@ -289,7 +326,6 @@ add_library(oicore
   oi/PaddingHunter.cpp
   oi/Serialize.cpp
 )
-add_dependencies(oicore libdrgn)
 target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
 target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
 target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -323,7 +359,6 @@ else()
 endif()
 
 target_link_libraries(oicore
-  "-L${DRGN_PATH}/.libs"
   drgn
   dw
   pthread
@@ -334,11 +369,9 @@ add_library(treebuilder
   oi/TreeBuilder.cpp
   oi/exporters/TypeCheckingWalker.cpp
 )
-add_dependencies(treebuilder librocksdb)
 target_link_libraries(treebuilder
-  ${rocksdb_BINARY_DIR}/librocksdb.a
+  RocksDB::rocksdb
   oicore # overkill but it does need a lot of stuff
-  zstd::zstd
 )
 
 
@@ -375,10 +408,8 @@ target_link_libraries(oip oicore)
 
 ### Object Introspection RocksDB Printer (OIRP)
 add_executable(oirp tools/OIRP.cpp)
-add_dependencies(oirp librocksdb)
 target_link_libraries(oirp
-  ${rocksdb_BINARY_DIR}/librocksdb.a
-  zstd::zstd
+  RocksDB::rocksdb
   msgpackc
 )
 
@@ -417,3 +448,6 @@ endif()
 if (DEFINED ENV{CMAKE_HOOK})
   include($ENV{CMAKE_HOOK})
 endif()
+
+install(TARGETS oid DESTINATION ${CMAKE_INSTALL_BINDIR})
+
diff --git a/README.md b/README.md
index 9f95bf8..abce6f8 100644
--- a/README.md
+++ b/README.md
@@ -14,3 +14,61 @@ See the [CONTRIBUTING](CONTRIBUTING.md) file for how to help out.
 
 ## License
 Object Introspection is licensed under the [Apache 2.0 License](LICENSE).
+
+## Getting started with Nix
+
+Nix is the easiest way to get started with `oid` as it is non-trivial to build otherwise. Explicit Nix support for Object Introspection as a Library will come down the line, but Nix can currently provide you a reproducible development environment in which to build it.
+
+These examples expect you to have `nix` installed and available with no other dependencies required. Find the installation guide at https://nixos.org/download.html.
+
+We also required flake support. To enable flakes globally run:
+
+    $ mkdir -p ~/.config/nix
+    $ echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf
+
+Or suffix every `nix` command with `nix --extra-experimental-features 'nix-command flakes'`.
+
+### Run upstream OID without modifying the source
+
+    $ nix run github:facebookexperimental/object-introspection -- --help
+
+This will download the latest source into your Nix store along with all of its dependencies, running help afterwards.
+
+### Build OID locally
+
+    $ git clone https://github.com/facebookexperimental/object-introspection
+    $ nix build
+    $ ./result/bin/oid --help
+
+This will build OID from your local sources. Please note that this will NOT pick up changes to `extern/drgn` or `extern/drgn/libdrgn/velfutils`.
+
+### Get a development environment
+
+    $ nix develop
+    $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+    $ ninja -C build
+    $ build/oid --help
+
+This command provides a development shell with all the required dependencies. This is the most flexible option and will pick up source changes as CMake normally would.
+
+Sometimes this developer environment can be polluted by things installed on your normal system. If this is an issue, use:
+
+    $ nix develop -i
+
+This removes the environment from your host system and makes the build pure.
+
+### Run the tests
+
+    $ nix develop
+    $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
+    $ ninja -C build
+    $ ./tools/config_gen.py -c clang++ build/testing.oid.toml
+    $ ctest -j --test-dir build/test
+
+Running tests under `nix` is new to the project and may take some time to mature. The CI is the source of truth for now.
+
+### Format source
+
+    $ nix fmt
+
+This formats the Nix, C++, and Python code in the repository.
diff --git a/cmake/Finduring.cmake b/cmake/Finduring.cmake
new file mode 100644
index 0000000..3b007fe
--- /dev/null
+++ b/cmake/Finduring.cmake
@@ -0,0 +1,27 @@
+# - Find liburing
+#
+# uring_INCLUDE_DIR - Where to find liburing.h
+# uring_LIBRARIES - List of libraries when using uring.
+# uring_FOUND - True if uring found.
+
+find_path(uring_INCLUDE_DIR
+  NAMES liburing.h)
+find_library(uring_LIBRARIES
+  NAMES liburing.a liburing.so)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(uring
+  DEFAULT_MSG uring_LIBRARIES uring_INCLUDE_DIR)
+
+mark_as_advanced(
+  uring_INCLUDE_DIR
+  uring_LIBRARIES)
+
+if(uring_FOUND AND NOT TARGET uring::uring)
+  add_library(uring::uring UNKNOWN IMPORTED)
+  set_target_properties(uring::uring PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${uring_INCLUDE_DIR}"
+    IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+    IMPORTED_LOCATION "${uring_LIBRARIES}")
+endif()
+
diff --git a/flake.nix b/flake.nix
index 7a3382e..74ed25a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,5 +1,5 @@
 {
-  description = "A flake for building Object Introspection.";
+  description = "Object level memory profiler for C++";
 
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
@@ -18,7 +18,99 @@
       treefmt-nix,
       ...
     }@inputs:
-    flake-utils.lib.eachDefaultSystem (
+    flake-utils.lib.eachSystem [ flake-utils.lib.system.x86_64-linux ] (
+      system:
+      let
+        pkgs = import nixpkgs { inherit system; };
+
+        drgnSrc = pkgs.fetchFromGitHub {
+          owner = "JakeHillion";
+          repo = "drgn";
+          rev = "b1f8c3e8526611b6720800250ba858a713dd9e4f";
+          hash = "sha256-5WhMHgx/RKtqjxGx4AyiqVKMot5xulr+6c8i2E9IxiA=";
+          fetchSubmodules = true;
+        };
+
+        mkOidPackage =
+          llvmPackages:
+          with pkgs;
+          pkgs.llvmPackages.stdenv.mkDerivation rec {
+            name = "oid";
+
+            src = self;
+
+            nativeBuildInputs = [
+              autoconf
+              automake
+              bison
+              cmake
+              flex
+              gettext
+              git
+              hexdump
+              libtool
+              ninja
+              pkgconf
+              python312
+              python312Packages.setuptools
+              python312Packages.toml
+              glibcLocales
+            ];
+
+            buildInputs = [
+              llvmPackages.libclang
+              llvmPackages.llvm
+
+              boost
+              bzip2
+              curl
+              double-conversion
+              elfutils
+              flex
+              folly
+              folly.fmt
+              gflags
+              glog
+              gtest
+              icu
+              jemalloc
+              libarchive
+              libmicrohttpd
+              liburing
+              libxml2
+              lzma
+              msgpack
+              range-v3
+              rocksdb_8_11
+              sqlite
+              tomlplusplus
+              zstd
+
+              llvmPackages_15.openmp # should match the stdenv clang version, see: https://github.com/NixOS/nixpkgs/issues/79818
+            ];
+
+            cmakeFlags = [
+              "-Ddrgn_SOURCE_DIR=${drgnSrc}"
+              "-DFORCE_BOOST_STATIC=Off"
+            ];
+
+            outputs = [ "out" ];
+          };
+      in
+      {
+        packages = rec {
+          default = oid-llvm15;
+
+          oid-llvm15 = mkOidPackage pkgs.llvmPackages_15;
+        };
+
+        apps.default = {
+          type = "app";
+          program = "${self.packages.${system}.default}/bin/oid";
+        };
+      }
+    )
+    // flake-utils.lib.eachDefaultSystem (
       system:
       let
         pkgs = nixpkgs.legacyPackages.${system};
diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt
index bb73871..7d805e5 100644
--- a/oi/CMakeLists.txt
+++ b/oi/CMakeLists.txt
@@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus)
 add_library(drgn_utils DrgnUtils.cpp)
 target_link_libraries(drgn_utils
   glog::glog
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
-add_dependencies(drgn_utils libdrgn)
 
 add_library(symbol_service
   Descs.cpp
diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt
index 6c4da13..c878e08 100644
--- a/oi/type_graph/CMakeLists.txt
+++ b/oi/type_graph/CMakeLists.txt
@@ -20,12 +20,9 @@ add_library(type_graph
   TypeIdentifier.cpp
   Types.cpp
 )
-add_dependencies(type_graph libdrgn)
 target_link_libraries(type_graph
   container_info
   symbol_service
-
-  "-L${DRGN_PATH}/.libs"
   drgn
 )
 target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
diff --git a/tools/config_gen.py b/tools/config_gen.py
index c20ca40..5ac56ff 100755
--- a/tools/config_gen.py
+++ b/tools/config_gen.py
@@ -16,6 +16,7 @@
 
 import argparse
 import getpass
+import os
 import pathlib
 import subprocess
 import typing
@@ -170,7 +171,10 @@ def pull_base_toml() -> typing.Dict:
 
     # Now, we need to replace any placeholders that might be present in the base toml file with the real verisons.
     user = getpass.getuser()
-    pwd = str(repo_path.resolve())
+    if "IN_NIX_SHELL" in os.environ and "src" in os.environ:
+        pwd = os.environ['src']
+    else:
+        pwd = str(repo_path.resolve())
 
     container_list = base.get("types", {}).get("containers")
     if container_list: