collapse TreeBuilderV2 features

Summary:

Currently there are two features between CodeGen v2 (TypeGraph) and TreeBuilder
v2. These are TypedDataSegment and TreeBuilderTypeChecking. Each of these
features currently has a full set of tests run in the CI and each have specific
exclusions.

Collapse these features into TreeBuilder v2. This allows for significantly
simplified testing as any OIL tests run under TreeBuilder v2 and any OID tests
run under TreeBuilder v1.

The reasoning behind this is I no longer intend to partially roll out this
feature. Full TreeBuilder v2 applies different conditions to containers than
the intermediate states, and writing these only to have them never deployed is
a waste of time.

Test Plan:
- it builds
- CI
This commit is contained in:
Jake Hillion 2023-11-03 06:06:25 -07:00
parent 315a5cde6c
commit 6c88c79083
12 changed files with 20 additions and 181 deletions

View File

@ -21,20 +21,6 @@ workflows:
oid_test_args: "-ftype-graph" oid_test_args: "-ftype-graph"
tests_regex: "OidIntegration\\..*" tests_regex: "OidIntegration\\..*"
exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0" exclude_regex: ".*inheritance_polymorphic.*|.*arrays_member_int0"
- test:
name: test-typed-data-segment-gcc
requires:
- build-gcc
oid_test_args: "-ftyped-data-segment"
tests_regex: "OidIntegration\\..*"
exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
- test:
name: test-tree-builder-type-checking-gcc
requires:
- build-gcc
oid_test_args: "-ftree-builder-type-checking"
tests_regex: "OidIntegration\\..*"
exclude_regex: ".*inheritance_polymorphic.*|.*pointers.*|.*arrays_member_int0|.*cycles_.*"
- coverage: - coverage:
name: coverage name: coverage
requires: requires:

View File

@ -117,17 +117,14 @@ void addIncludes(const TypeGraph& typeGraph,
FeatureSet features, FeatureSet features,
std::string& code) { std::string& code) {
std::set<std::string_view> includes{"cstddef"}; std::set<std::string_view> includes{"cstddef"};
if (features[Feature::TypedDataSegment]) { if (features[Feature::TreeBuilderV2]) {
code += "#define DEFINE_DESCRIBE 1\n"; // added before all includes
includes.emplace("functional"); includes.emplace("functional");
includes.emplace("oi/exporters/inst.h");
includes.emplace("oi/types/dy.h");
includes.emplace("oi/types/st.h"); includes.emplace("oi/types/st.h");
} }
if (features[Feature::TreeBuilderTypeChecking]) {
includes.emplace("oi/types/dy.h");
code += "#define DEFINE_DESCRIBE 1\n"; // added before all includes
}
if (features[Feature::TreeBuilderV2])
includes.emplace("oi/exporters/inst.h");
if (features[Feature::Library]) { if (features[Feature::Library]) {
includes.emplace("vector"); includes.emplace("vector");
includes.emplace("oi/IntrospectionResult.h"); includes.emplace("oi/IntrospectionResult.h");
@ -830,8 +827,7 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) {
code += " using type = "; code += " using type = ";
genClassStaticType(c, code); genClassStaticType(c, code);
code += ";\n"; code += ";\n";
if (config_.features[Feature::TreeBuilderV2]) genClassTreeBuilderInstructions(c, code);
genClassTreeBuilderInstructions(c, code);
genClassTraversalFunction(c, code); genClassTraversalFunction(c, code);
code += "};\n"; code += "};\n";
} }
@ -846,19 +842,6 @@ void genContainerTypeHandler(FeatureSet features,
if (!used.insert(&c).second) if (!used.insert(&c).second)
return; return;
if (!features[Feature::TreeBuilderV2]) {
const auto& handler = c.codegen.handler;
if (handler.empty()) {
LOG(ERROR) << "`codegen.handler` must be specified for all containers "
"under \"-ftyped-data-segment\", not specified for \"" +
c.typeName + "\"";
throw std::runtime_error("missing `codegen.handler`");
}
auto fmt = boost::format(c.codegen.handler) % c.typeName;
code += fmt.str();
return;
}
code += c.codegen.extra; code += c.codegen.extra;
// TODO: Move this check into the ContainerInfo parsing once always enabled. // TODO: Move this check into the ContainerInfo parsing once always enabled.
@ -1216,14 +1199,14 @@ void CodeGen::generate(
if (!config_.features[Feature::Library]) { if (!config_.features[Feature::Library]) {
FuncGen::DeclareExterns(code); FuncGen::DeclareExterns(code);
} }
if (!config_.features[Feature::TypedDataSegment]) { if (!config_.features[Feature::TreeBuilderV2]) {
defineMacros(code); defineMacros(code);
} }
addIncludes(typeGraph, config_.features, code); addIncludes(typeGraph, config_.features, code);
defineInternalTypes(code); defineInternalTypes(code);
FuncGen::DefineJitLog(code, config_.features); FuncGen::DefineJitLog(code, config_.features);
if (config_.features[Feature::TypedDataSegment]) { if (config_.features[Feature::TreeBuilderV2]) {
if (config_.features[Feature::Library]) { if (config_.features[Feature::Library]) {
FuncGen::DefineBackInserterDataBuffer(code); FuncGen::DefineBackInserterDataBuffer(code);
} else { } else {
@ -1231,10 +1214,8 @@ void CodeGen::generate(
} }
code += "using namespace oi;\n"; code += "using namespace oi;\n";
code += "using namespace oi::detail;\n"; code += "using namespace oi::detail;\n";
if (config_.features[Feature::TreeBuilderV2]) { code += "using oi::exporters::ParsedData;\n";
code += "using oi::exporters::ParsedData;\n"; code += "using namespace oi::exporters;\n";
code += "using namespace oi::exporters;\n";
}
code += "namespace OIInternal {\nnamespace {\n"; code += "namespace OIInternal {\nnamespace {\n";
FuncGen::DefineBasicTypeHandlers(code, config_.features); FuncGen::DefineBasicTypeHandlers(code, config_.features);
code += "} // namespace\n} // namespace OIInternal\n"; code += "} // namespace\n} // namespace OIInternal\n";
@ -1254,7 +1235,7 @@ void CodeGen::generate(
* process faster. * process faster.
*/ */
code += "namespace OIInternal {\nnamespace {\n"; code += "namespace OIInternal {\nnamespace {\n";
if (!config_.features[Feature::TypedDataSegment]) { if (!config_.features[Feature::TreeBuilderV2]) {
FuncGen::DefineEncodeData(code); FuncGen::DefineEncodeData(code);
FuncGen::DefineEncodeDataSize(code); FuncGen::DefineEncodeDataSize(code);
FuncGen::DefineStoreData(code); FuncGen::DefineStoreData(code);
@ -1269,7 +1250,7 @@ void CodeGen::generate(
genExclusiveSizes(typeGraph, code); genExclusiveSizes(typeGraph, code);
} }
if (config_.features[Feature::TypedDataSegment]) { if (config_.features[Feature::TreeBuilderV2]) {
addStandardTypeHandlers(typeGraph, config_.features, code); addStandardTypeHandlers(typeGraph, config_.features, code);
addTypeHandlers(typeGraph, code); addTypeHandlers(typeGraph, code);
} else { } else {
@ -1286,10 +1267,8 @@ void CodeGen::generate(
code += "} // namespace\n} // namespace OIInternal\n"; code += "} // namespace\n} // namespace OIInternal\n";
const auto typeName = SymbolService::getTypeName(drgnType); const auto typeName = SymbolService::getTypeName(drgnType);
if (config_.features[Feature::Library]) { if (config_.features[Feature::TreeBuilderV2]) {
FuncGen::DefineTopLevelIntrospect(code, typeName); FuncGen::DefineTopLevelIntrospect(code, typeName);
} else if (config_.features[Feature::TypedDataSegment]) {
FuncGen::DefineTopLevelGetSizeRefTyped(code, typeName, config_.features);
} else { } else {
FuncGen::DefineTopLevelGetSizeRef(code, typeName, config_.features); FuncGen::DefineTopLevelGetSizeRef(code, typeName, config_.features);
} }
@ -1298,8 +1277,6 @@ void CodeGen::generate(
FuncGen::DefineTreeBuilderInstructions(code, typeName, FuncGen::DefineTreeBuilderInstructions(code, typeName,
calculateExclusiveSize(rootType), calculateExclusiveSize(rootType),
enumerateTypeNames(rootType)); enumerateTypeNames(rootType));
} else if (config_.features[Feature::TreeBuilderTypeChecking]) {
FuncGen::DefineOutputType(code, typeName);
} }
if (!linkageName_.empty()) if (!linkageName_.empty())

View File

@ -274,10 +274,6 @@ ContainerInfo::ContainerInfo(const fs::path& path) {
} else { } else {
throw ContainerInfoError(path, "`codegen.decl` is a required field"); throw ContainerInfoError(path, "`codegen.decl` is a required field");
} }
if (std::optional<std::string> str =
codegenToml["handler"].value<std::string>()) {
codegen.handler = std::move(*str);
}
if (std::optional<std::string> str = if (std::optional<std::string> str =
codegenToml["traversal_func"].value<std::string>()) { codegenToml["traversal_func"].value<std::string>()) {
codegen.traversalFunc = std::move(*str); codegen.traversalFunc = std::move(*str);
@ -324,5 +320,5 @@ ContainerInfo::ContainerInfo(std::string typeName_,
ctype(ctype_), ctype(ctype_),
header(std::move(header_)), header(std::move(header_)),
codegen(Codegen{"// DummyDecl %1%\n", "// DummyFunc %1%\n", codegen(Codegen{"// DummyDecl %1%\n", "// DummyFunc %1%\n",
"// DummyHandler %1%\n", "// DummyFunc\n"}) { "// DummyFunc\n"}) {
} }

View File

@ -36,7 +36,6 @@ struct ContainerInfo {
struct Codegen { struct Codegen {
std::string decl; std::string decl;
std::string func; std::string func;
std::string handler = "";
std::string traversalFunc = ""; std::string traversalFunc = "";
std::string extra = ""; std::string extra = "";
std::vector<Processor> processors{}; std::vector<Processor> processors{};

View File

@ -40,11 +40,6 @@ std::optional<std::string_view> featureHelp(Feature f) {
return "Use Type Graph for code generation (CodeGen v2)."; return "Use Type Graph for code generation (CodeGen v2).";
case Feature::PruneTypeGraph: case Feature::PruneTypeGraph:
return "Prune unreachable nodes from the type graph"; return "Prune unreachable nodes from the type graph";
case Feature::TypedDataSegment:
return "Use Typed Data Segment in generated code.";
case Feature::TreeBuilderTypeChecking:
return "Use Typed Data Segment to perform runtime Type Checking in "
"TreeBuilder.";
case Feature::Library: case Feature::Library:
return std::nullopt; // Hide in OID help return std::nullopt; // Hide in OID help
case Feature::TreeBuilderV2: case Feature::TreeBuilderV2:
@ -65,14 +60,8 @@ std::optional<std::string_view> featureHelp(Feature f) {
std::span<const Feature> requirements(Feature f) { std::span<const Feature> requirements(Feature f) {
switch (f) { switch (f) {
case Feature::TypedDataSegment:
static constexpr std::array tds = {Feature::TypeGraph};
return tds;
case Feature::TreeBuilderTypeChecking:
static constexpr std::array tc = {Feature::TypedDataSegment};
return tc;
case Feature::TreeBuilderV2: case Feature::TreeBuilderV2:
static constexpr std::array tb2 = {Feature::TreeBuilderTypeChecking}; static constexpr std::array tb2 = {Feature::TypeGraph};
return tb2; return tb2;
case Feature::Library: case Feature::Library:
static constexpr std::array lib = {Feature::TreeBuilderV2}; static constexpr std::array lib = {Feature::TreeBuilderV2};

View File

@ -29,8 +29,6 @@
X(CaptureThriftIsset, "capture-thrift-isset") \ X(CaptureThriftIsset, "capture-thrift-isset") \
X(TypeGraph, "type-graph") \ X(TypeGraph, "type-graph") \
X(PruneTypeGraph, "prune-type-graph") \ X(PruneTypeGraph, "prune-type-graph") \
X(TypedDataSegment, "typed-data-segment") \
X(TreeBuilderTypeChecking, "tree-builder-type-checking") \
X(Library, "library") \ X(Library, "library") \
X(TreeBuilderV2, "tree-builder-v2") \ X(TreeBuilderV2, "tree-builder-v2") \
X(GenJitDebug, "gen-jit-debug") \ X(GenJitDebug, "gen-jit-debug") \

View File

@ -352,83 +352,6 @@ void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode,
testCode.append(fmt.str()); testCode.append(fmt.str());
} }
/*
* DefineTopLevelGetSizeRefTyped
*
* Top level function to run OI on a type utilising static types and enabled
* with feature '-ftyped-data-segment'.
*/
void FuncGen::DefineTopLevelGetSizeRefTyped(std::string& testCode,
const std::string& rawType,
FeatureSet features) {
std::string func = R"(
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunknown-attributes"
/* RawType: %1% */
void __attribute__((used, retain)) getSize_%2$016x(const OIInternal::__ROOT_TYPE__& t)
#pragma GCC diagnostic pop
{
)";
if (features[Feature::JitTiming]) {
func += " const auto startTime = std::chrono::steady_clock::now();\n";
}
func += R"(
pointers.initialize();
pointers.add((uintptr_t)&t);
auto data = reinterpret_cast<uintptr_t*>(dataBase);
// TODO: Replace these with types::st::Uint64 once the VarInt decoding
// logic is moved out of OIDebugger and into new TreeBuilder.
size_t dataSegOffset = 0;
data[dataSegOffset++] = oidMagicId;
data[dataSegOffset++] = cookieValue;
uintptr_t& writtenSize = data[dataSegOffset++];
writtenSize = 0;
uintptr_t& timeTakenNs = data[dataSegOffset++];
dataSegOffset *= sizeof(uintptr_t);
JLOG("%1% @");
JLOGPTR(&t);
using ContentType = OIInternal::TypeHandler<DataBuffer::DataSegment, OIInternal::__ROOT_TYPE__>::type;
using SuffixType = types::st::Pair<
DataBuffer::DataSegment,
types::st::VarInt<DataBuffer::DataSegment>,
types::st::VarInt<DataBuffer::DataSegment>
>;
using DataBufferType = types::st::Pair<
DataBuffer::DataSegment,
ContentType,
SuffixType
>;
DataBufferType db = DataBuffer::DataSegment(dataSegOffset);
SuffixType suffix = db.delegate([&t](auto ret) {
return OIInternal::getSizeType<DataBuffer::DataSegment>(t, ret);
});
types::st::Unit<DataBuffer::DataSegment> end = suffix
.write(123456789)
.write(123456789);
dataSegOffset = end.offset();
writtenSize = dataSegOffset;
dataBase += dataSegOffset;
)";
if (features[Feature::JitTiming]) {
func += R"(
timeTakenNs = std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::steady_clock::now() - startTime).count();
)";
}
func += R"(
}
)";
boost::format fmt =
boost::format(func) % rawType % std::hash<std::string>{}(rawType);
testCode.append(fmt.str());
}
/* /*
* DefineOutputType * DefineOutputType
* *
@ -803,23 +726,6 @@ ContainerInfo FuncGen::GetOiArrayContainerInfo() {
ContainerInfo oiArray{"OIArray", UNKNOWN_TYPE, ContainerInfo oiArray{"OIArray", UNKNOWN_TYPE,
"cstdint"}; // TODO: remove the need for a dummy header "cstdint"}; // TODO: remove the need for a dummy header
oiArray.codegen.handler = R"(
template<typename DB, typename T0, long unsigned int N>
struct TypeHandler<DB, %1%<T0, N>> {
using type = types::st::List<DB, typename TypeHandler<DB, T0>::type>;
static types::st::Unit<DB> getSizeType(
const %1%<T0, N> &container,
typename TypeHandler<DB, %1%<T0,N>>::type returnArg) {
auto tail = returnArg.write(N);
for (size_t i=0; i<N; i++) {
tail = tail.delegate([&container, i](auto ret) {
return TypeHandler<DB, T0>::getSizeType(container.vals[i], ret);
});
}
return tail.finish();
}
};
)";
oiArray.codegen.traversalFunc = R"( oiArray.codegen.traversalFunc = R"(
auto tail = returnArg.write(N0); auto tail = returnArg.write(N0);
for (size_t i=0; i<N0; i++) { for (size_t i=0; i<N0; i++) {

View File

@ -61,9 +61,6 @@ class FuncGen {
static void DefineTopLevelGetSizeRef(std::string& testCode, static void DefineTopLevelGetSizeRef(std::string& testCode,
const std::string& rawType, const std::string& rawType,
FeatureSet features); FeatureSet features);
static void DefineTopLevelGetSizeRefTyped(std::string& testCode,
const std::string& rawType,
FeatureSet features);
static void DefineOutputType(std::string& testCode, static void DefineOutputType(std::string& testCode,
const std::string& rawType); const std::string& rawType);
static void DefineTreeBuilderInstructions( static void DefineTreeBuilderInstructions(

View File

@ -520,9 +520,8 @@ bool OICompiler::compile(const std::string& code,
static const auto syntheticHeaders = std::array< static const auto syntheticHeaders = std::array<
std::pair<Feature, std::pair<std::string_view, std::string>>, 7>{{ std::pair<Feature, std::pair<std::string_view, std::string>>, 7>{{
{Feature::TypedDataSegment, {headers::oi_types_st_h, "oi/types/st.h"}}, {Feature::TreeBuilderV2, {headers::oi_types_st_h, "oi/types/st.h"}},
{Feature::TreeBuilderTypeChecking, {Feature::TreeBuilderV2, {headers::oi_types_dy_h, "oi/types/dy.h"}},
{headers::oi_types_dy_h, "oi/types/dy.h"}},
{Feature::TreeBuilderV2, {Feature::TreeBuilderV2,
{headers::oi_exporters_inst_h, "oi/exporters/inst.h"}}, {headers::oi_exporters_inst_h, "oi/exporters/inst.h"}},
{Feature::TreeBuilderV2, {Feature::TreeBuilderV2,

View File

@ -181,8 +181,6 @@ int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
std::map<Feature, bool> featuresMap = { std::map<Feature, bool> featuresMap = {
{Feature::TypeGraph, true}, {Feature::TypeGraph, true},
{Feature::TypedDataSegment, true},
{Feature::TreeBuilderTypeChecking, true},
{Feature::TreeBuilderV2, true}, {Feature::TreeBuilderV2, true},
{Feature::Library, true}, {Feature::Library, true},
{Feature::PackStructs, true}, {Feature::PackStructs, true},

View File

@ -174,8 +174,6 @@ namespace {
std::map<Feature, bool> convertFeatures(std::unordered_set<oi::Feature> fs) { std::map<Feature, bool> convertFeatures(std::unordered_set<oi::Feature> fs) {
std::map<Feature, bool> out{ std::map<Feature, bool> out{
{Feature::TypeGraph, true}, {Feature::TypeGraph, true},
{Feature::TypedDataSegment, true},
{Feature::TreeBuilderTypeChecking, true},
{Feature::TreeBuilderV2, true}, {Feature::TreeBuilderV2, true},
{Feature::Library, true}, {Feature::Library, true},
{Feature::PackStructs, true}, {Feature::PackStructs, true},

View File

@ -62,14 +62,10 @@ This document describes the format of the container definition files contained i
will collide if they share the same name. will collide if they share the same name.
## Changes introduced with Typed Data Segment
- `decl` and `func` fields are ignored when using `-ftyped-data-segment` and the
`handler` field is used instead.
## Changes introduced with TreeBuilder V2 ## Changes introduced with TreeBuilder V2
- `decl`, `func`, and `handler` fields are ignored when using `-ftree-builder-v2`. - `decl` and `func` fields are ignored when using `-ftree-builder-v2`. The
The `TypeHandler` is instead constructed from `traversal_func` and `processor` `TypeHandler` is constructed from `traversal_func` field and `processor`
entries. entries.
## Changes introduced with TypeGraph ## Changes introduced with TypeGraph
- `typeName` and `matcher` fields have been merged into the single field `type_name`. - `typeName` and `matcher` fields have been merged into the single field `type_name`.