diff --git a/CMakeLists.txt b/CMakeLists.txt index 12dc26d..39156ba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -334,9 +334,18 @@ target_link_libraries(treebuilder ## OI Outputs ### Object Introspection as a Library (OIL) -add_library(oil oi/OILibrary.cpp oi/OILibraryImpl.cpp) +add_library(oil + oi/IntrospectionResult.cpp + oi/exporters/ParsedData.cpp +) +target_link_libraries(oil folly_headers) target_include_directories(oil PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) -target_link_libraries(oil oicore) + +add_library(oil_jit + oi/OILibrary.cpp + oi/OILibraryImpl.cpp +) +target_link_libraries(oil_jit oicore oil) ### Object Introspection as a Library Generator (OILGen) add_executable(oilgen diff --git a/include/oi/IntrospectionResult-inl.h b/include/oi/IntrospectionResult-inl.h new file mode 100644 index 0000000..56f32a8 --- /dev/null +++ b/include/oi/IntrospectionResult-inl.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#if defined(INCLUDED_OI_INTROSPECTIONRESULT_INL_H) || \ + !defined(INCLUDED_OI_INTROSPECTIONRESULT_H) +static_assert(false, + "IntrospectionResult-inl.h provides inline declarations for " + "IntrospectionResult.h and should only " + "be included by IntrospectionResult.h"); +#endif +#define INCLUDED_OI_INTROSPECTIONRESULT_INL_H 1 + +#include + +#include "IntrospectionResult.h" + +namespace oi { + +inline IntrospectionResult::IntrospectionResult(std::vector buf, + exporters::inst::Inst inst) + : buf_(std::move(buf)), inst_(inst) { +} + +inline IntrospectionResult::const_iterator::const_iterator( + std::vector::const_iterator data, exporters::inst::Inst type) + : data_(data), stack_({type}) { +} +inline IntrospectionResult::const_iterator::const_iterator( + std::vector::const_iterator data) + : data_(data) { +} +inline IntrospectionResult::const_iterator IntrospectionResult::cbegin() const { + return ++const_iterator{buf_.cbegin(), inst_}; +} +inline IntrospectionResult::const_iterator IntrospectionResult::cend() const { + return {buf_.cend()}; +} + +inline IntrospectionResult::const_iterator +IntrospectionResult::const_iterator::operator++(int) { + auto old = *this; + operator++(); + return old; +} +inline const result::Element& IntrospectionResult::const_iterator::operator*() + const { + assert(next_); + return *next_; +} +inline const result::Element* IntrospectionResult::const_iterator::operator->() + const { + return &*next_; +} + +inline bool IntrospectionResult::const_iterator::operator==( + const IntrospectionResult::const_iterator& that) const { + return this->data_ == that.data_ && !this->next_.has_value() && + !that.next_.has_value(); // TODO: is this sufficient? kind of hacky as + // this only works for comparing to .end() +} +inline bool IntrospectionResult::const_iterator::operator!=( + const IntrospectionResult::const_iterator& that) const { + return !(*this == that); +} + +} // namespace oi diff --git a/include/oi/IntrospectionResult.h b/include/oi/IntrospectionResult.h new file mode 100644 index 0000000..279a775 --- /dev/null +++ b/include/oi/IntrospectionResult.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INCLUDED_OI_INTROSPECTIONRESULT_H +#define INCLUDED_OI_INTROSPECTIONRESULT_H 1 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace oi { + +class IntrospectionResult { + public: + class const_iterator { + friend class IntrospectionResult; + + public: + bool operator==(const const_iterator& that) const; + bool operator!=(const const_iterator& that) const; + const result::Element& operator*() const; + const result::Element* operator->() const; + const_iterator& operator++(); + const_iterator operator++(int); + + private: + const_iterator(std::vector::const_iterator data, + exporters::inst::Inst type); + const_iterator(std::vector::const_iterator data); + + std::vector::const_iterator data_; + std::stack stack_; + std::optional next_; + + std::vector type_path_; + }; + + IntrospectionResult(std::vector buf, exporters::inst::Inst inst); + + const_iterator cbegin() const; + const_iterator cend() const; + + private: + std::vector buf_; + exporters::inst::Inst inst_; +}; + +} // namespace oi + +#include +#endif diff --git a/include/oi/exporters/Json.h b/include/oi/exporters/Json.h new file mode 100644 index 0000000..0015e28 --- /dev/null +++ b/include/oi/exporters/Json.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INCLUDED_OI_EXPORTERS_JSON_H +#define INCLUDED_OI_EXPORTERS_JSON_H 1 + +#include + +#include + +namespace oi::exporters { + +class Json { + public: + Json(std::ostream& out); + + void print(const IntrospectionResult&); + void print(IntrospectionResult::const_iterator& it, + IntrospectionResult::const_iterator end); + + void setPretty(bool pretty) { + pretty_ = pretty; + } + + private: + bool pretty_ = false; + std::ostream& out_; +}; + +} // namespace oi::exporters + +#endif diff --git a/include/oi/exporters/ParsedData.h b/include/oi/exporters/ParsedData.h new file mode 100644 index 0000000..05c2985 --- /dev/null +++ b/include/oi/exporters/ParsedData.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INCLUDED_OI_EXPORTERS_PARSED_DATA_H +#define INCLUDED_OI_EXPORTERS_PARSED_DATA_H 1 + +#include + +#include +#include +#include +#include + +namespace oi::exporters { + +struct ParsedData { + class Lazy { + public: + Lazy(std::vector::const_iterator& it, types::dy::Dynamic ty) + : it_(it), ty_(ty) { + } + + ParsedData operator()() { + return ParsedData::parse(it_, ty_); + } + + private: + std::vector::const_iterator& it_; + types::dy::Dynamic ty_; + }; + + struct Unit {}; + struct VarInt { + uint64_t value; + }; + struct Pair { + Lazy first; + Lazy second; + }; + struct List { + uint64_t length; + Lazy values; + }; + struct Sum { + uint64_t index; + Lazy value; + }; + + static ParsedData parse(std::vector::const_iterator& it, + types::dy::Dynamic ty); + + ParsedData(Unit&& val_) : val(val_) { + } + ParsedData(VarInt&& val_) : val(val_) { + } + ParsedData(Pair&& val_) : val(val_) { + } + ParsedData(List&& val_) : val(val_) { + } + ParsedData(Sum&& val_) : val(val_) { + } + + std::variant val; +}; + +} // namespace oi::exporters + +#endif diff --git a/include/oi/exporters/inst.h b/include/oi/exporters/inst.h new file mode 100644 index 0000000..3d9170e --- /dev/null +++ b/include/oi/exporters/inst.h @@ -0,0 +1,90 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef INCLUDED_OI_EXPORTERS_INST_H +#define INCLUDED_OI_EXPORTERS_INST_H 1 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace oi::exporters::inst { + +struct PopTypePath; +struct Field; + +using Inst = std::variant>; +using Processor = void (*)(result::Element&, std::stack&, ParsedData); +using ProcessorInst = std::pair; + +struct PopTypePath {}; + +struct Field { + template + constexpr Field(size_t static_size_, + size_t exclusive_size_, + std::string_view name_, + const std::array& type_names_, + const std::array& fields_, + const std::array& processors_); + template + constexpr Field(size_t static_size_, + std::string_view name_, + const std::array& type_names_, + const std::array& fields_, + const std::array& processors_) + : Field(static_size_, + static_size_, + name_, + type_names_, + fields_, + processors_) { + } + constexpr Field(const Field&) = default; // no idea why this is needed + + size_t static_size; + size_t exclusive_size; + std::string_view name; + std::span type_names; + std::span fields; + std::span processors; +}; + +template +constexpr Field::Field(size_t static_size_, + size_t exclusive_size_, + std::string_view name_, + const std::array& type_names_, + const std::array& fields_, + const std::array& processors_) + : static_size(static_size_), + exclusive_size(exclusive_size_), + name(name_), + type_names(type_names_), + fields(fields_), + processors(processors_) { +} + +} // namespace oi::exporters::inst + +#endif diff --git a/include/oi/oi-jit-inl.h b/include/oi/oi-jit-inl.h new file mode 100644 index 0000000..a08a24d --- /dev/null +++ b/include/oi/oi-jit-inl.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#if defined(INCLUDED_OI_OI_JIT_INL_H) || !defined(INCLUDED_OI_OI_JIT_H) +static_assert( + false, + "oi-jit-inl.h provides inline declarations for oi-jit.h and should only " + "be included by oi-jit.h"); +#endif +#define INCLUDED_OI_OI_JIT_INL_H 1 + +#include + +#include "oi-jit.h" + +namespace oi { + +template +inline std::optional setupAndIntrospect( + const T& objectAddr, const GeneratorOptions& opts) { + if (!CodegenHandler::init(opts)) + return std::nullopt; + + return CodegenHandler::introspect(objectAddr); +} + +template +inline std::atomic& CodegenHandler::getIsCritical() { + static std::atomic isCritical = false; + return isCritical; +} + +template +inline std::atomic&)>& +CodegenHandler::getIntrospectionFunc() { + static std::atomic&)> func = nullptr; + return func; +} + +template +inline std::atomic& +CodegenHandler::getTreeBuilderInstructions() { + static std::atomic ty = nullptr; + return ty; +} + +template +inline bool CodegenHandler::init(const GeneratorOptions& opts) { + if (getIntrospectionFunc().load() != nullptr && + getTreeBuilderInstructions().load() != nullptr) + return true; // already initialised + if (getIsCritical().exchange(true)) + return false; // other thread is initialising/has failed + + auto lib = + OILibrary(reinterpret_cast(&getIntrospectionFunc), {Fs...}, opts); + auto [vfp, ty] = lib.init(); + + getIntrospectionFunc().store(reinterpret_cast(vfp)); + getTreeBuilderInstructions().store(&ty); + return true; +} + +template +inline IntrospectionResult CodegenHandler::introspect( + const T& objectAddr) { + func_type func = getIntrospectionFunc().load(); + const exporters::inst::Inst* ty = getTreeBuilderInstructions().load(); + + if (func == nullptr || ty == nullptr) + throw std::logic_error("introspect(const T&) called when uninitialised"); + + std::vector buf; + static_assert(sizeof(std::vector) == 24); + func(objectAddr, buf); + return IntrospectionResult{std::move(buf), *ty}; +} + +} // namespace oi diff --git a/include/oi/oi-jit.h b/include/oi/oi-jit.h new file mode 100644 index 0000000..d356a4a --- /dev/null +++ b/include/oi/oi-jit.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INCLUDED_OI_OI_JIT_H +#define INCLUDED_OI_OI_JIT_H 1 + +#include +#include +#include +#include +#include +#include +#include + +#include "exporters/inst.h" +#include "oi.h" + +namespace oi::detail { +class OILibraryImpl; +} + +namespace oi { + +struct GeneratorOptions { + std::filesystem::path configFilePath; + std::filesystem::path sourceFileDumpPath; + int debugLevel = 0; +}; + +class OILibrary { + public: + OILibrary(void* atomicHome, + std::unordered_set, + GeneratorOptions opts); + ~OILibrary(); + std::pair init(); + + private: + std::unique_ptr pimpl_; +}; + +/* + * setupAndIntrospect + * + * Execute JIT compilation then introspect the given object. Throws on error. + * Returns std::nullopt rather than duplicating initialisation if its ongoing. + */ +template +std::optional setupAndIntrospect( + const T& objectAddr, const GeneratorOptions& opts); + +template +class CodegenHandler { + public: + static bool init(const GeneratorOptions& opts); + static IntrospectionResult introspect(const T& objectAddr); + + private: + using func_type = void (*)(const T&, std::vector&); + + static std::atomic& getIsCritical(); + static std::atomic& getIntrospectionFunc(); + static std::atomic& + getTreeBuilderInstructions(); +}; + +} // namespace oi + +#include "oi-jit-inl.h" +#endif diff --git a/include/oi/oi.h b/include/oi/oi.h new file mode 100644 index 0000000..d2fa851 --- /dev/null +++ b/include/oi/oi.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INCLUDED_OI_OI_H +#define INCLUDED_OI_OI_H 1 + +#include +#include + +#include +#include + +namespace oi { + +enum class Feature { + ChaseRawPointers, + CaptureThriftIsset, + GenJitDebug, +}; + +template +IntrospectionResult __attribute__((noinline)) introspect(const T& objectAddr); + +} // namespace oi + +#ifndef OIL_AOT_COMPILATION +#include "oi-jit.h" +#else + +template +IntrospectionResult __attribute__((noinline)) introspect(const T& objectAddr); +{ static_assert(false, "OIL v2 does not yet support AoT compilation."); } + +#endif +#endif diff --git a/include/oi/result/Element.h b/include/oi/result/Element.h new file mode 100644 index 0000000..07428e1 --- /dev/null +++ b/include/oi/result/Element.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INCLUDED_OI_RESULT_ELEMENT_H +#define INCLUDED_OI_RESULT_ELEMENT_H 1 + +#include +#include +#include +#include + +namespace oi::result { + +struct Element { + struct ContainerStats { + size_t capacity; + size_t length; + }; + struct IsSetStats { + bool is_set; + }; + + std::string_view name; + std::vector + type_path; // TODO: should be span + std::span type_names; + size_t static_size; + size_t exclusive_size; + + std::optional pointer; + std::optional container_stats; + std::optional is_set_stats; +}; + +} // namespace oi::result + +#endif diff --git a/include/oi/types/dy.h b/include/oi/types/dy.h index df088aa..0efcfab 100644 --- a/include/oi/types/dy.h +++ b/include/oi/types/dy.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef OI_TYPES_DY_H -#define OI_TYPES_DY_H 1 +#ifndef INCLUDED_OI_TYPES_DY_H +#define INCLUDED_OI_TYPES_DY_H 1 /* * Dynamic Types @@ -45,11 +45,11 @@ namespace oi::types::dy { -class Unit; -class VarInt; -class Pair; -class Sum; -class List; +struct Unit; +struct VarInt; +struct Pair; +struct Sum; +struct List; /* * Dynamic @@ -63,11 +63,10 @@ using Dynamic = std::variant, std::reference_wrapper, std::reference_wrapper >; -class Unit {}; -class VarInt {}; +struct Unit {}; +struct VarInt {}; -class Pair { - public: +struct Pair { constexpr Pair(Dynamic first_, Dynamic second_) : first(first_), second(second_) { } @@ -76,8 +75,7 @@ class Pair { Dynamic second; }; -class Sum { - public: +struct Sum { template constexpr Sum(const std::array& variants_) : variants(variants_) { } @@ -85,8 +83,7 @@ class Sum { std::span variants; }; -class List { - public: +struct List { constexpr List(Dynamic element_) : element(element_) { } diff --git a/include/oi/types/st.h b/include/oi/types/st.h index 721b1b6..5465aae 100644 --- a/include/oi/types/st.h +++ b/include/oi/types/st.h @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef OI_TYPES_ST_H -#define OI_TYPES_ST_H 1 +#ifndef INCLUDED_OI_TYPES_ST_H +#define INCLUDED_OI_TYPES_ST_H 1 /* * Static Types @@ -123,7 +123,7 @@ class VarInt { } Unit write(uint64_t val) { - while (val >= 128) { + while (val >= 0x80) { _buf.write_byte(0x80 | (val & 0x7f)); val >>= 7; } diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt index 3c924e1..72c89a8 100644 --- a/oi/CMakeLists.txt +++ b/oi/CMakeLists.txt @@ -52,4 +52,10 @@ target_link_libraries(codegen glog::glog ) +add_library(exporters_json + exporters/Json.cpp +) +target_include_directories(exporters_json PUBLIC ${CMAKE_SOURCE_DIR}/include) +target_link_libraries(exporters_json oil) + add_subdirectory(type_graph) diff --git a/oi/CodeGen.cpp b/oi/CodeGen.cpp index 84a0d79..cddc1e9 100644 --- a/oi/CodeGen.cpp +++ b/oi/CodeGen.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -54,9 +55,11 @@ using type_graph::Enum; using type_graph::Flattener; using type_graph::Member; using type_graph::NameGen; +using type_graph::Primitive; using type_graph::Prune; using type_graph::RemoveMembers; using type_graph::RemoveTopLevelPointer; +using type_graph::TemplateParam; using type_graph::TopoSorter; using type_graph::Type; using type_graph::Typedef; @@ -67,6 +70,18 @@ template using ref = std::reference_wrapper; namespace { + +std::vector enumerateTypeNames(Type& type) { + std::vector names; + Type* t = &type; + while (const Typedef* td = dynamic_cast(t)) { + names.emplace_back(t->inputName()); + t = &td->underlyingType(); + } + names.emplace_back(t->inputName()); + return names; +} + void defineMacros(std::string& code) { if (true /* TODO: config.useDataSegment*/) { code += R"( @@ -93,6 +108,23 @@ struct OIArray { void defineJitLog(FeatureSet features, std::string& code) { if (features[Feature::JitLogging]) { code += R"( +extern int logFile; + +void __jlogptr(uintptr_t ptr) { + static constexpr char hexdigits[] = "0123456789abcdef"; + static constexpr size_t ptrlen = 2 * sizeof(ptr); + + static char hexstr[ptrlen + 1] = {}; + + size_t i = ptrlen; + while (i--) { + hexstr[i] = hexdigits[ptr & 0xf]; + ptr = ptr >> 4; + } + hexstr[ptrlen] = '\n'; + write(logFile, hexstr, sizeof(hexstr)); +} + #define JLOG(str) \ do { \ if (__builtin_expect(logFile, 0)) { \ @@ -128,6 +160,10 @@ void addIncludes(const TypeGraph& typeGraph, code += "#define DEFINE_DESCRIBE 1\n"; // added before all includes } + if (features[Feature::TreeBuilderV2]) + includes.emplace("oi/exporters/inst.h"); + if (features[Feature::Library]) + includes.emplace("vector"); if (features[Feature::JitTiming]) { includes.emplace("chrono"); } @@ -569,38 +605,6 @@ void CodeGen::addGetSizeFuncDefs(const TypeGraph& typeGraph, namespace { -void addStandardTypeHandlers(std::string& code) { - // Provide a wrapper function, getSizeType, to infer T instead of having to - // explicitly specify it with TypeHandler::getSizeType every time. - code += R"( - template - types::st::Unit - getSizeType(const T &t, typename TypeHandler::type returnArg) { - JLOG("obj @"); - JLOGPTR(&t); - return TypeHandler::getSizeType(t, returnArg); - } -)"; - - code += R"( - template - struct TypeHandler> { - using type = types::st::List::type>; - static types::st::Unit getSizeType( - const OIArray &container, - typename TypeHandler>::type returnArg) { - auto tail = returnArg.write(N); - for (size_t i=0; i::getSizeType(container.vals[i], ret); - }); - } - return tail.finish(); - } - }; -)"; -} - // Find the last member that isn't padding's index. Return -1 if no such member. size_t getLastNonPaddingMemberIndex(const std::vector& members) { for (size_t i = members.size() - 1; i != (size_t)-1; --i) { @@ -724,6 +728,70 @@ void CodeGen::genClassStaticType(const Class& c, std::string& code) { } } +namespace { + +size_t calculateExclusiveSize(const Type& t) { + if (const auto* c = dynamic_cast(&t)) { + return std::accumulate(c->members.cbegin(), c->members.cend(), 0, + [](size_t a, const auto& m) { + if (m.name.starts_with(AddPadding::MemberPrefix)) + return a + m.type().size(); + return a; + }); + } + return t.size(); +} + +} // namespace + +void CodeGen::genClassTreeBuilderInstructions(const Class& c, + std::string& code) { + code += " private:\n"; + size_t index = 0; + for (const auto& m : c.members) { + ++index; + if (m.name.starts_with(AddPadding::MemberPrefix)) + continue; + + auto names = enumerateTypeNames(m.type()); + code += " static constexpr std::array member_" + std::to_string(index) + + "_type_names = {"; + for (const auto& name : names) { + code += "\""; + code += name; + code += "\","; + } + code += "};\n"; + } + + code += " public:\n"; + size_t numFields = + std::count_if(c.members.cbegin(), c.members.cend(), [](const auto& m) { + return !m.name.starts_with(AddPadding::MemberPrefix); + }); + code += " static constexpr std::array::fields, TypeHandler::processors},\n"; + } + code += " };\n"; + code += + "static constexpr std::array " + "processors{};\n"; +} + void CodeGen::genClassTypeHandler(const Class& c, std::string& code) { std::string helpers; @@ -756,30 +824,159 @@ void CodeGen::genClassTypeHandler(const Class& c, std::string& code) { code += " using type = "; genClassStaticType(c, code); code += ";\n"; + if (config_.features[Feature::TreeBuilderV2]) + genClassTreeBuilderInstructions(c, code); genClassTraversalFunction(c, code); code += "};\n"; } namespace { -void getContainerTypeHandler(std::unordered_set& used, - const Container& c, +void genContainerTypeHandler(FeatureSet features, + std::unordered_set& used, + const ContainerInfo& c, + std::span templateParams, std::string& code) { - if (!used.insert(&c.containerInfo_).second) { + if (!used.insert(&c).second) + return; + + if (!features[Feature::TreeBuilderV2]) { + const auto& handler = c.codegen.handler; + if (handler.empty()) { + LOG(ERROR) << "`codegen.handler` must be specified for all containers " + "under \"-ftyped-data-segment\", not specified for \"" + + c.typeName + "\""; + throw std::runtime_error("missing `codegen.handler`"); + } + auto fmt = boost::format(c.codegen.handler) % c.typeName; + code += fmt.str(); return; } - const auto& handler = c.containerInfo_.codegen.handler; // TODO: Move this check into the ContainerInfo parsing once always enabled. - if (handler.empty()) { - LOG(ERROR) << "`codegen.handler` must be specified for all containers " - "under \"-ftyped-data-segment\", not specified for \"" + - c.containerInfo_.typeName + "\""; - throw std::runtime_error("missing `codegen.handler`"); + const auto& func = c.codegen.traversalFunc; + const auto& processors = c.codegen.processors; + + if (func.empty()) { + LOG(ERROR) + << "`codegen.traversal_func` must be specified for all containers " + "under \"-ftree-builder-v2\", not specified for \"" + + c.typeName + "\""; + throw std::runtime_error("missing `codegen.traversal_func`"); } - auto fmt = boost::format(c.containerInfo_.codegen.handler) % - c.containerInfo_.typeName; - code += fmt.str(); + + std::string containerWithTypes = c.typeName; + if (!templateParams.empty()) + containerWithTypes += '<'; + size_t types = 0, values = 0; + for (const auto& p : templateParams) { + if (types > 0 || values > 0) + containerWithTypes += ", "; + if (p.value) { + containerWithTypes += "N" + std::to_string(values++); + } else { + containerWithTypes += "T" + std::to_string(types++); + } + } + if (!templateParams.empty()) + containerWithTypes += '>'; + + code += "template type; + if (it != processors.cend() - 1) + code += ", "; + } + code += std::string(processors.size() - 1, '>'); + } + code += ";\n"; + + code += " static types::st::Unit getSizeType(\n"; + code += " const "; + code += containerWithTypes; + code += "& container,\n"; + code += " typename TypeHandler& ins, ParsedData d) {\n"; + code += pr.func; // bad indentation + code += " }\n"; + } + + code += " public:\n"; + code += + " static constexpr std::array fields{};\n"; + code += " static constexpr std::array::getSizeType every time. + code += R"( + template + types::st::Unit + getSizeType(const T &t, typename TypeHandler::type returnArg) { + JLOG("obj @"); + JLOGPTR(&t); + return TypeHandler::getSizeType(t, returnArg); + } +)"; + + // TODO: bit of a hack - making ContainerInfo a node in the type graph and + // traversing for it would remove the need for this set altogether. + std::unordered_set used{}; + std::vector arrayParams{ + TemplateParam{typeGraph.makeType(Primitive::Kind::UInt64)}, + TemplateParam{typeGraph.makeType(Primitive::Kind::UInt64), + "0"}, + }; + genContainerTypeHandler(features, used, FuncGen::GetOiArrayContainerInfo(), + arrayParams, code); } } // namespace @@ -789,7 +986,8 @@ void CodeGen::addTypeHandlers(const TypeGraph& typeGraph, std::string& code) { if (const auto* c = dynamic_cast(&t)) { genClassTypeHandler(*c, code); } else if (const auto* con = dynamic_cast(&t)) { - getContainerTypeHandler(definedContainers_, *con, code); + genContainerTypeHandler(config_.features, definedContainers_, + con->containerInfo_, con->templateParams, code); } } } @@ -886,6 +1084,9 @@ void CodeGen::generate( struct drgn_type* drgnType /* TODO: this argument should not be required */ ) { code = headers::oi_OITraceCode_cpp; + if (!config_.features[Feature::Library]) { + FuncGen::DeclareExterns(code); + } if (!config_.features[Feature::TypedDataSegment]) { defineMacros(code); } @@ -894,12 +1095,19 @@ void CodeGen::generate( defineJitLog(config_.features, code); if (config_.features[Feature::TypedDataSegment]) { - FuncGen::DefineDataSegmentDataBuffer(code); + if (config_.features[Feature::Library]) { + FuncGen::DefineBackInserterDataBuffer(code); + } else { + FuncGen::DefineDataSegmentDataBuffer(code); + } code += "using namespace oi;\n"; code += "using namespace oi::detail;\n"; - + if (config_.features[Feature::TreeBuilderV2]) { + code += "using oi::exporters::ParsedData;\n"; + code += "using namespace oi::exporters;\n"; + } code += "namespace OIInternal {\nnamespace {\n"; - FuncGen::DefineBasicTypeHandlers(code); + FuncGen::DefineBasicTypeHandlers(code, config_.features); code += "} // namespace\n} // namespace OIInternal\n"; } @@ -930,7 +1138,7 @@ void CodeGen::generate( genStaticAsserts(typeGraph, code); if (config_.features[Feature::TypedDataSegment]) { - addStandardTypeHandlers(code); + addStandardTypeHandlers(typeGraph, config_.features, code); addTypeHandlers(typeGraph, code); } else { addStandardGetSizeFuncDecls(code); @@ -946,13 +1154,19 @@ void CodeGen::generate( code += "} // namespace\n} // namespace OIInternal\n"; const auto typeName = SymbolService::getTypeName(drgnType); - if (config_.features[Feature::TypedDataSegment]) { + if (config_.features[Feature::Library]) { + FuncGen::DefineTopLevelIntrospect(code, typeName); + } else if (config_.features[Feature::TypedDataSegment]) { FuncGen::DefineTopLevelGetSizeRefTyped(code, typeName, config_.features); } else { FuncGen::DefineTopLevelGetSizeRef(code, typeName, config_.features); } - if (config_.features[Feature::TreeBuilderTypeChecking]) { + if (config_.features[Feature::TreeBuilderV2]) { + FuncGen::DefineTreeBuilderInstructions(code, typeName, + calculateExclusiveSize(rootType), + enumerateTypeNames(rootType)); + } else if (config_.features[Feature::TreeBuilderTypeChecking]) { FuncGen::DefineOutputType(code, typeName); } diff --git a/oi/CodeGen.h b/oi/CodeGen.h index b85a413..bc88d69 100644 --- a/oi/CodeGen.h +++ b/oi/CodeGen.h @@ -80,6 +80,8 @@ class CodeGen { void genClassTypeHandler(const type_graph::Class& c, std::string& code); void genClassStaticType(const type_graph::Class& c, std::string& code); void genClassTraversalFunction(const type_graph::Class& c, std::string& code); + void genClassTreeBuilderInstructions(const type_graph::Class& c, + std::string& code); }; } // namespace oi::detail diff --git a/oi/ContainerInfo.cpp b/oi/ContainerInfo.cpp index d2fc34d..be6ee70 100644 --- a/oi/ContainerInfo.cpp +++ b/oi/ContainerInfo.cpp @@ -265,6 +265,38 @@ ContainerInfo::ContainerInfo(const fs::path& path) { codegenToml["handler"].value()) { codegen.handler = std::move(*str); } + if (std::optional str = + codegenToml["traversal_func"].value()) { + codegen.traversalFunc = std::move(*str); + } + + if (toml::array* arr = codegenToml["processor"].as_array()) { + codegen.processors.reserve(arr->size()); + arr->for_each([&](auto&& el) { + if (toml::table* proc = el.as_table()) { + std::string type, func; + if (std::optional str = + (*proc)["type"].value()) { + type = std::move(*str); + } else { + throw ContainerInfoError( + path, "codegen.processor.type is a required field"); + } + if (std::optional str = + (*proc)["func"].value()) { + func = std::move(*str); + } else { + throw ContainerInfoError( + path, "codegen.processor.func is a required field"); + } + codegen.processors.emplace_back( + Processor{std::move(type), std::move(func)}); + } else { + throw ContainerInfoError( + path, "codegen.processor should only contain tables"); + } + }); + } } ContainerInfo::ContainerInfo(std::string typeName_, @@ -275,5 +307,5 @@ ContainerInfo::ContainerInfo(std::string typeName_, ctype(ctype_), header(std::move(header_)), codegen(Codegen{"// DummyDecl %1%\n", "// DummyFunc %1%\n", - "// DummyHandler %1%\n"}) { + "// DummyHandler %1%\n", "// DummyFunc\n"}) { } diff --git a/oi/ContainerInfo.h b/oi/ContainerInfo.h index 1b759b6..c3e52c5 100644 --- a/oi/ContainerInfo.h +++ b/oi/ContainerInfo.h @@ -27,10 +27,17 @@ ContainerTypeEnum containerTypeEnumFromStr(std::string& str); const char* containerTypeEnumToStr(ContainerTypeEnum ty); struct ContainerInfo { + struct Processor { + std::string type; + std::string func; + }; + struct Codegen { std::string decl; std::string func; std::string handler = ""; + std::string traversalFunc = ""; + std::vector processors{}; }; explicit ContainerInfo(const std::filesystem::path& path); // Throws diff --git a/oi/Features.cpp b/oi/Features.cpp index a98fcd6..baf9344 100644 --- a/oi/Features.cpp +++ b/oi/Features.cpp @@ -45,6 +45,8 @@ std::optional featureHelp(Feature f) { case Feature::TreeBuilderTypeChecking: return "Use Typed Data Segment to perform runtime Type Checking in " "TreeBuilder."; + case Feature::Library: + return std::nullopt; // Hide in OID help case Feature::TreeBuilderV2: return "Use Tree Builder v2 for reading the data segment"; case Feature::GenJitDebug: @@ -72,6 +74,20 @@ std::span requirements(Feature f) { case Feature::TreeBuilderV2: static constexpr std::array tb2 = {Feature::TreeBuilderTypeChecking}; return tb2; + case Feature::Library: + static constexpr std::array lib = {Feature::TreeBuilderV2}; + return lib; + default: + return {}; + } +} + +std::span conflicts(Feature f) { + switch (f) { + case Feature::Library: + static constexpr std::array lib = {Feature::JitLogging, + Feature::JitTiming}; + return lib; default: return {}; } @@ -147,6 +163,19 @@ std::optional handleFeatureConflicts(FeatureSet enabled, } } + for (const auto f : allFeatures) { + if (!enabled[f]) + continue; + + for (const auto c : conflicts(f)) { + if (enabled[c]) { + LOG(ERROR) << featureToStr(f) << " feature conflicts with " + << featureToStr(c) << " but both are enabled!"; + return std::nullopt; + } + } + } + return enabled; } diff --git a/oi/Features.h b/oi/Features.h index 911f0a9..2e92ba0 100644 --- a/oi/Features.h +++ b/oi/Features.h @@ -31,6 +31,7 @@ X(PruneTypeGraph, "prune-type-graph") \ X(TypedDataSegment, "typed-data-segment") \ X(TreeBuilderTypeChecking, "tree-builder-type-checking") \ + X(Library, "library") \ X(TreeBuilderV2, "tree-builder-v2") \ X(GenJitDebug, "gen-jit-debug") \ X(JitLogging, "jit-logging") \ diff --git a/oi/FuncGen.cpp b/oi/FuncGen.cpp index 8aadc20..540aa14 100644 --- a/oi/FuncGen.cpp +++ b/oi/FuncGen.cpp @@ -147,6 +147,16 @@ void FuncGen::DeclareTopLevelGetSize(std::string& testCode, boost::format fmt = boost::format("void getSizeType(const %1% &t);\n") % type; testCode.append(fmt.str()); } + +void FuncGen::DeclareExterns(std::string& code) { + constexpr std::string_view vars = R"( +extern uint8_t* dataBase; +extern size_t dataSize; +extern uintptr_t cookieValue; + )"; + code.append(vars); +} + void FuncGen::DeclareStoreData(std::string& testCode) { testCode.append("void StoreData(uintptr_t data, size_t& dataSegOffset);\n"); } @@ -235,6 +245,35 @@ void FuncGen::DefineTopLevelGetObjectSize(std::string& testCode, testCode.append(fmt.str()); } +void FuncGen::DefineTopLevelIntrospect(std::string& code, + const std::string& type) { + std::string func = R"( +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunknown-attributes" +/* RawType: %1% */ +void __attribute__((used, retain)) introspect_%2$016x( + const OIInternal::__ROOT_TYPE__& t, + std::vector& v) +#pragma GCC diagnostic pop +{ + pointers.initialize(); + pointers.add((uintptr_t)&t); + + v.clear(); + v.reserve(4096); + + using DataBufferType = DataBuffer::BackInserter>; + using ContentType = OIInternal::TypeHandler::type; + + ContentType ret{DataBufferType{v}}; + OIInternal::getSizeType(t, ret); +} +)"; + + code.append( + (boost::format(func) % type % std::hash{}(type)).str()); +} + void FuncGen::DefineTopLevelGetSizeRef(std::string& testCode, const std::string& rawType, FeatureSet features) { @@ -373,15 +412,58 @@ void FuncGen::DefineOutputType(std::string& code, const std::string& rawType) { #pragma GCC diagnostic ignored "-Wunknown-attributes" /* RawType: %1% */ extern const types::dy::Dynamic __attribute__((used, retain)) outputType%2$016x = - OIInternal::TypeHandler::type::describe; + OIInternal::TypeHandler::type::describe; #pragma GCC diagnostic pop - )"; +)"; boost::format fmt = boost::format(func) % rawType % std::hash{}(rawType); code.append(fmt.str()); } +void FuncGen::DefineTreeBuilderInstructions( + std::string& code, + const std::string& rawType, + size_t exclusiveSize, + std::span typeNames) { + std::string typeHash = + (boost::format("%1$016x") % std::hash{}(rawType)).str(); + + code += R"( +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunknown-attributes" +namespace { +const std::array::fields, " + "OIInternal::TypeHandler::processors};\n"; + code += "} // namespace\n"; + code += + "extern const exporters::inst::Inst __attribute__((used, retain)) " + "treeBuilderInstructions"; + code += typeHash; + code += " = rootInstructions"; + code += typeHash; + code += ";\n"; + code += "#pragma GCC diagnostic pop\n"; +} + void FuncGen::DefineTopLevelGetSizeRefRet(std::string& testCode, const std::string& rawType) { std::string func = R"( @@ -564,6 +646,34 @@ void FuncGen::DefineDataSegmentDataBuffer(std::string& testCode) { testCode.append(func); } +/* + * DefineBackInserterDataBuffer + * + * Provides a DataBuffer implementation that takes anything convertible with + * std::back_inserter. + */ +void FuncGen::DefineBackInserterDataBuffer(std::string& code) { + constexpr std::string_view buf = R"( +namespace oi::detail::DataBuffer { + +template +class BackInserter { + public: + BackInserter(Container& v) : buf(v) {} + + void write_byte(uint8_t byte) { + *buf = byte; + } + private: + std::back_insert_iterator buf; +}; + +} // namespace oi::detail::DataBuffer + )"; + + code.append(buf); +} + /* * DefineBasicTypeHandlers * @@ -573,8 +683,8 @@ void FuncGen::DefineDataSegmentDataBuffer(std::string& testCode) { * pointer's value always, then the value of the pointer if it is unique. void * is of type Unit and always stores nothing. */ -void FuncGen::DefineBasicTypeHandlers(std::string& testCode) { - constexpr std::string_view tHandler = R"( +void FuncGen::DefineBasicTypeHandlers(std::string& code, FeatureSet features) { + code += R"( template struct TypeHandler { private: @@ -582,10 +692,8 @@ void FuncGen::DefineBasicTypeHandlers(std::string& testCode) { if constexpr(std::is_pointer_v) { return std::type_identity, - types::st::Sum, - typename TypeHandler>::type - >>>(); + types::st::Sum, typename TypeHandler>::type> + >>(); } else { return std::type_identity>(); } @@ -593,7 +701,56 @@ void FuncGen::DefineBasicTypeHandlers(std::string& testCode) { public: using type = typename decltype(choose_type())::type; +)"; + if (features[Feature::TreeBuilderV2]) { + code += R"(private: + static void process_pointer(result::Element& el, std::stack& ins, ParsedData d) { + el.pointer = std::get(d.val).value; + } + static void process_pointer_content(result::Element& el, std::stack& ins, ParsedData d) { + static constexpr std::array names{"TODO"}; + static constexpr auto childField = inst::Field{ + sizeof(T), + "*", + names, + TypeHandler::fields, + TypeHandler::processors, + }; + const ParsedData::Sum& sum = std::get(d.val); + + el.container_stats.emplace(result::Element::ContainerStats{ .capacity = 1 }); + + if (sum.index == 0) + return; + + el.container_stats->length = 1; + ins.emplace(childField); + } + + static constexpr auto choose_fields() { + if constexpr(std::is_pointer_v) { + return std::array{}; + } else { + return std::array{}; + } + } + static constexpr auto choose_processors() { + if constexpr(std::is_pointer_v) { + return std::array{ + {types::st::VarInt::describe, &process_pointer}, + {types::st::Sum, typename TypeHandler>::type>::describe, &process_pointer_content}, + }; + } else { + return std::array{}; + } + } + public: + static constexpr auto fields = choose_fields(); + static constexpr auto processors = choose_processors(); +)"; + } + code += R"( static types::st::Unit getSizeType( const T& t, typename TypeHandler::type returnArg) { @@ -619,16 +776,75 @@ void FuncGen::DefineBasicTypeHandlers(std::string& testCode) { }; )"; - constexpr std::string_view voidHandler = R"( + code += R"( template class TypeHandler { public: using type = types::st::Unit; - }; - )"; +)"; + if (features[Feature::TreeBuilderV2]) { + code += + "static constexpr std::array fields{};\n"; + code += + "static constexpr std::array " + "processors{};\n"; + } + code += "};\n"; +} - testCode.append(tHandler); - testCode.append(voidHandler); +ContainerInfo FuncGen::GetOiArrayContainerInfo() { + ContainerInfo oiArray{"OIArray", UNKNOWN_TYPE, + "cstdint"}; // TODO: remove the need for a dummy header + + oiArray.codegen.handler = R"( +template +struct TypeHandler> { + using type = types::st::List::type>; + static types::st::Unit getSizeType( + const %1% &container, + typename TypeHandler>::type returnArg) { + auto tail = returnArg.write(N); + for (size_t i=0; i::getSizeType(container.vals[i], ret); + }); + } + return tail.finish(); + } +}; +)"; + oiArray.codegen.traversalFunc = R"( +auto tail = returnArg.write(N0); +for (size_t i=0; i::getSizeType(container.vals[i], ret); + }); +} +return tail.finish(); +)"; + oiArray.codegen.processors.emplace_back(ContainerInfo::Processor{ + .type = "types::st::List::type>", + .func = R"( +static constexpr std::array names{"TODO"}; +static constexpr auto childField = inst::Field{ + sizeof(T0), + "[]", + names, + TypeHandler::fields, + TypeHandler::processors, +}; + +el.exclusive_size = 0; +el.container_stats.emplace(result::Element::ContainerStats{ .capacity = N0, .length = N0 }); + +auto list = std::get(d.val); +// assert(list.length == N0); +for (size_t i = 0; i < N0; i++) + ins.emplace(childField); +)", + }); + + return oiArray; } } // namespace oi::detail diff --git a/oi/FuncGen.h b/oi/FuncGen.h index 65fbe71..16eb2af 100644 --- a/oi/FuncGen.h +++ b/oi/FuncGen.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include "oi/ContainerInfo.h" @@ -28,6 +29,8 @@ namespace oi::detail { class FuncGen { public: + static void DeclareExterns(std::string& code); + static void DeclareStoreData(std::string& testCode); static void DefineStoreData(std::string& testCode); @@ -56,6 +59,8 @@ class FuncGen { static void DefineTopLevelGetObjectSize(std::string& testCode, const std::string& type, const std::string& linkageName); + static void DefineTopLevelIntrospect(std::string& code, + const std::string& type); static void DefineTopLevelGetSizeRef(std::string& testCode, const std::string& rawType, @@ -65,6 +70,11 @@ class FuncGen { FeatureSet features); static void DefineOutputType(std::string& testCode, const std::string& rawType); + static void DefineTreeBuilderInstructions( + std::string& testCode, + const std::string& rawType, + size_t exclusiveSize, + std::span typeNames); static void DefineTopLevelGetSizeRefRet(std::string& testCode, const std::string& type); @@ -77,7 +87,10 @@ class FuncGen { const std::string& ctype); static void DefineDataSegmentDataBuffer(std::string& testCode); - static void DefineBasicTypeHandlers(std::string& testCode); + static void DefineBackInserterDataBuffer(std::string& code); + static void DefineBasicTypeHandlers(std::string& code, FeatureSet features); + + static ContainerInfo GetOiArrayContainerInfo(); }; } // namespace oi::detail diff --git a/oi/Headers.h b/oi/Headers.h index 191d54a..f2949f7 100644 --- a/oi/Headers.h +++ b/oi/Headers.h @@ -19,7 +19,10 @@ namespace oi::detail::headers { // These externs are provided by our build system. See resources/CMakeLists.txt extern const std::string_view oi_OITraceCode_cpp; -extern const std::string_view oi_types_st_h; +extern const std::string_view oi_exporters_ParsedData_h; +extern const std::string_view oi_exporters_inst_h; +extern const std::string_view oi_result_Element_h; extern const std::string_view oi_types_dy_h; +extern const std::string_view oi_types_st_h; } // namespace oi::detail::headers diff --git a/oi/IntrospectionResult.cpp b/oi/IntrospectionResult.cpp new file mode 100644 index 0000000..cd09806 --- /dev/null +++ b/oi/IntrospectionResult.cpp @@ -0,0 +1,80 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +#include +#include +#include + +template +inline constexpr bool always_false_v = false; + +namespace oi { + +IntrospectionResult::const_iterator& +IntrospectionResult::const_iterator::operator++() { + if (stack_.empty()) { + next_ = std::nullopt; + return *this; + } + + auto el = stack_.top(); + stack_.pop(); + + return std::visit( + [this](auto&& r) -> IntrospectionResult::const_iterator& { + using U = std::decay_t; + if constexpr (std::is_same_v) { + type_path_.pop_back(); + return operator++(); + } else { + // reference wrapper + auto ty = r.get(); + using T = std::decay_t; + + if constexpr (std::is_same_v) { + type_path_.emplace_back(ty.name); + stack_.emplace(exporters::inst::PopTypePath{}); + next_ = result::Element{ + .name = ty.name, + .type_path = type_path_, + .type_names = ty.type_names, + .static_size = ty.static_size, + .exclusive_size = ty.exclusive_size, + .container_stats = std::nullopt, + .is_set_stats = std::nullopt, + }; + + for (const auto& [dy, handler] : ty.processors) { + auto parsed = exporters::ParsedData::parse(data_, dy); + handler(*next_, stack_, parsed); + } + for (auto it = ty.fields.rbegin(); it != ty.fields.rend(); ++it) { + stack_.emplace(*it); + } + + return *this; + } else { + static_assert(always_false_v, "non-exhaustive visitor!"); + } + } + }, + el); +} + +} // namespace oi diff --git a/oi/OICodeGen.cpp b/oi/OICodeGen.cpp index e19eaaa..cfed5ae 100644 --- a/oi/OICodeGen.cpp +++ b/oi/OICodeGen.cpp @@ -2983,6 +2983,7 @@ void OICodeGen::declareThriftStruct(std::string& code, std::string_view name) { } bool OICodeGen::generateJitCode(std::string& code) { + FuncGen::DeclareExterns(code); // Include relevant headers code.append("// relevant header includes -----\n"); diff --git a/oi/OICompiler.cpp b/oi/OICompiler.cpp index d866b3a..ab9e918 100644 --- a/oi/OICompiler.cpp +++ b/oi/OICompiler.cpp @@ -37,6 +37,7 @@ #include #include +#include #include #include @@ -516,21 +517,32 @@ bool OICompiler::compile(const std::string& code, path.c_str(), clang::frontend::IncludeDirGroup::System, false, false); } - if (config.features[Feature::TypedDataSegment]) { + static const auto syntheticHeaders = std::array< + std::pair>, 5>{{ + {Feature::TypedDataSegment, {headers::oi_types_st_h, "oi/types/st.h"}}, + {Feature::TreeBuilderTypeChecking, + {headers::oi_types_dy_h, "oi/types/dy.h"}}, + {Feature::TreeBuilderV2, + {headers::oi_exporters_inst_h, "oi/exporters/inst.h"}}, + {Feature::TreeBuilderV2, + {headers::oi_exporters_ParsedData_h, "oi/exporters/ParsedData.h"}}, + {Feature::TreeBuilderV2, + {headers::oi_result_Element_h, "oi/result/Element.h"}}, + }}; + for (const auto& [k, v] : syntheticHeaders) { + if (!config.features[k]) + continue; compInv->getPreprocessorOpts().addRemappedFile( - "/synthetic/headers/oi/types/st.h", - MemoryBuffer::getMemBuffer(headers::oi_types_st_h).release()); - headerSearchOptions.AddPath( - "/synthetic/headers", clang::frontend::IncludeDirGroup::IndexHeaderMap, - false, false); + std::string{"/synthetic/headers/"} + v.second, + MemoryBuffer::getMemBuffer(v.first).release()); } - if (config.features[Feature::TreeBuilderTypeChecking]) { - compInv->getPreprocessorOpts().addRemappedFile( - "/synthetic/headers/oi/types/dy.h", - MemoryBuffer::getMemBuffer(headers::oi_types_dy_h).release()); - headerSearchOptions.AddPath( - "/synthetic/headers", clang::frontend::IncludeDirGroup::IndexHeaderMap, - false, false); + for (const auto& [k, _] : syntheticHeaders) { + if (config.features[k]) { + headerSearchOptions.AddPath( + "/synthetic/headers", + clang::frontend::IncludeDirGroup::IndexHeaderMap, false, false); + break; + } } compInv->getFrontendOpts().OutputFile = objectPath; diff --git a/oi/OILibrary.cpp b/oi/OILibrary.cpp index 9b0f4c1..80eaba9 100644 --- a/oi/OILibrary.cpp +++ b/oi/OILibrary.cpp @@ -14,49 +14,21 @@ * limitations under the License. */ #include "oi/OILibraryImpl.h" +#include "oi/oi-jit.h" -bool debug = false; +namespace oi { -namespace ObjectIntrospection { - -bool operator==(const options& lhs, const options& rhs) { - return lhs.configFilePath == rhs.configFilePath && - lhs.debugFilePath == rhs.debugFilePath && - lhs.debugLevel == rhs.debugLevel && - lhs.chaseRawPointers == rhs.chaseRawPointers; +OILibrary::OILibrary(void* atomicHole, + std::unordered_set fs, + GeneratorOptions opts) + : pimpl_{std::make_unique( + atomicHole, std::move(fs), std::move(opts))} { } - -bool operator!=(const options& lhs, const options& rhs) { - return !(lhs == rhs); -} - -OILibrary::OILibrary(void* TemplateFunc, options opt) : opts(opt) { - this->pimpl_ = new OILibraryImpl(this, TemplateFunc); -} - OILibrary::~OILibrary() { - delete pimpl_; } -int OILibrary::init() { - if (!pimpl_->processConfigFile()) { - return Response::OIL_BAD_CONFIG_FILE; - } - - if (!pimpl_->mapSegment()) { - return Response::OIL_SEGMENT_INIT_FAIL; - } - - pimpl_->initCompiler(); - return pimpl_->compileCode(); +std::pair OILibrary::init() { + return pimpl_->init(); } -int OILibrary::getObjectSize(void* ObjectAddr, size_t& size) { - if (fp == nullptr) { - return Response::OIL_UNINITIALISED; - } - - size = (*fp)(ObjectAddr); - return Response::OIL_SUCCESS; -} -} // namespace ObjectIntrospection +} // namespace oi diff --git a/oi/OILibraryImpl.cpp b/oi/OILibraryImpl.cpp index 7370ba1..47f5a9c 100644 --- a/oi/OILibraryImpl.cpp +++ b/oi/OILibraryImpl.cpp @@ -13,208 +13,241 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "oi/OILibraryImpl.h" +#include "OILibraryImpl.h" -#include #include -#include -#include #include -#include +#include #include +#include +#include #include +#include +#include "oi/DrgnUtils.h" #include "oi/Headers.h" -#include "oi/OIParser.h" #include "oi/OIUtils.h" -extern "C" { -#include +namespace oi::detail { +namespace { +// Map between the high level feature requests in the OIL API and the underlying +// codegen features. +std::map convertFeatures(std::unordered_set fs); + +// Extract the root type from an atomic function pointer +drgn_qualified_type getTypeFromAtomicHole(drgn_program* prog, void* hole); +} // namespace + +OILibraryImpl::LocalTextSegment::LocalTextSegment(size_t size) { + void* base = mmap(NULL, size, PROT_EXEC | PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (base == MAP_FAILED) + throw std::runtime_error(std::string("segment map failed: ") + + std::strerror(errno)); + + data_ = {static_cast(base), size}; } -namespace ObjectIntrospection { +OILibraryImpl::LocalTextSegment::~LocalTextSegment() { + if (data_.empty()) + return; -using namespace oi::detail; - -OILibraryImpl::OILibraryImpl(OILibrary* self, void* TemplateFunc) - : _self(self), _TemplateFunc(TemplateFunc) { - if (_self->opts.debugLevel != 0) { - google::LogToStderr(); - google::SetStderrLogging(0); - google::SetVLOGLevel("*", _self->opts.debugLevel); - // Upstream glog defines `GLOG_INFO` as 0 https://fburl.com/ydjajhz0, - // but internally it's defined as 1 https://fburl.com/code/9fwams75 - // - // We don't want to link gflags in OIL, so setting it via the flags rather - // than with gflags::SetCommandLineOption - FLAGS_minloglevel = 0; - } + PLOG_IF(ERROR, munmap(data_.data(), data_.size()) != 0) + << "segment unmap failed"; } -OILibraryImpl::~OILibraryImpl() { - unmapSegment(); +OILibraryImpl::MemoryFile::MemoryFile(const char* name) { + fd_ = memfd_create(name, 0); + if (fd_ == -1) + throw std::runtime_error(std::string("memfd creation failed: ") + + std::strerror(errno)); } -bool OILibraryImpl::mapSegment() { - void* textSeg = - mmap(NULL, segConfig.textSegSize, PROT_EXEC | PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (textSeg == MAP_FAILED) { - PLOG(ERROR) << "error mapping text segment"; - return false; - } - segConfig.textSegBase = textSeg; +OILibraryImpl::MemoryFile::~MemoryFile() { + if (fd_ == -1) + return; - return true; + PLOG_IF(ERROR, close(fd_) == -1) << "memfd close failed"; } -bool OILibraryImpl::unmapSegment() { - if (segConfig.textSegBase != nullptr && - munmap(segConfig.textSegBase, segConfig.textSegSize) != 0) { - PLOG(ERROR) << "error unmapping text segment"; - return false; - } - - return true; +std::filesystem::path OILibraryImpl::MemoryFile::path() { + return {(boost::format("/dev/fd/%1%") % fd_).str()}; } -void OILibraryImpl::initCompiler() { - symbols = std::make_shared(getpid()); - - generatorConfig.useDataSegment = false; +OILibraryImpl::OILibraryImpl(void* atomicHole, + std::unordered_set fs, + GeneratorOptions opts) + : atomicHole_(atomicHole), + requestedFeatures_(convertFeatures(std::move(fs))), + opts_(std::move(opts)) { } -bool OILibraryImpl::processConfigFile() { - auto features = utils::processConfigFile( - _self->opts.configFilePath, - { - {Feature::ChaseRawPointers, _self->opts.chaseRawPointers}, - {Feature::PackStructs, true}, - {Feature::PruneTypeGraph, true}, - {Feature::GenJitDebug, _self->opts.generateJitDebugInfo}, - }, - compilerConfig, generatorConfig); - if (!features) { - return false; - } - generatorConfig.features = *features; - compilerConfig.features = *features; - return true; +std::pair OILibraryImpl::init() { + processConfigFile(); + + constexpr size_t TextSegSize = 1u << 22; + textSeg = {TextSegSize}; + + return compileCode(); } -template -class Cleanup { - T resource; - F cleanupFunc; +void OILibraryImpl::processConfigFile() { + auto features = + utils::processConfigFile(opts_.configFilePath, requestedFeatures_, + compilerConfig_, generatorConfig_); + if (!features) + throw std::runtime_error("failed to process configuration"); - public: - Cleanup(T _resource, F _cleanupFunc) - : resource{_resource}, cleanupFunc{_cleanupFunc} {}; - ~Cleanup() { - cleanupFunc(resource); - } -}; - -void close_file(std::FILE* fp) { - std::fclose(fp); + generatorConfig_.features = *features; + compilerConfig_.features = *features; } -int OILibraryImpl::compileCode() { - OICompiler compiler{symbols, compilerConfig}; +std::pair OILibraryImpl::compileCode() { + auto symbols = std::make_shared(getpid()); - int objectMemfd = memfd_create("oil_object_code", 0); - if (!objectMemfd) { - PLOG(ERROR) << "failed to create memfd for object code"; - return Response::OIL_COMPILATION_FAILURE; - } + auto* prog = symbols->getDrgnProgram(); + CHECK(prog != nullptr) << "does this check need to exist?"; - using unique_file_t = std::unique_ptr; - unique_file_t objectStream(fdopen(objectMemfd, "w+"), &close_file); - if (!objectStream) { - PLOG(ERROR) << "failed to convert memfd to stream"; - // This only needs to be cleaned up in the error case, as the fclose - // on the unique_file_t will clean up the underlying fd if it was - // created successfully. - close(objectMemfd); - return Response::OIL_COMPILATION_FAILURE; - } - auto objectPath = - fs::path((boost::format("/dev/fd/%1%") % objectMemfd).str()); + auto rootType = getTypeFromAtomicHole(symbols->getDrgnProgram(), atomicHole_); - struct drgn_program* prog = symbols->getDrgnProgram(); - if (!prog) { - return Response::OIL_COMPILATION_FAILURE; - } - struct drgn_symbol* sym; - if (auto err = drgn_program_find_symbol_by_address( - prog, (uintptr_t)_TemplateFunc, &sym)) { - LOG(ERROR) << "Error when finding symbol by address " << err->code << " " - << err->message; - drgn_error_destroy(err); - return Response::OIL_COMPILATION_FAILURE; - } - const char* name = drgn_symbol_name(sym); - drgn_symbol_destroy(sym); + CodeGen codegen{generatorConfig_, *symbols}; - // TODO: change this to the new drgn interface from symbol -> type - auto rootType = symbols->getRootType(irequest{"entry", name, "arg0"}); - if (!rootType.has_value()) { - LOG(ERROR) << "Failed to get type of probe argument"; - return Response::OIL_COMPILATION_FAILURE; - } + std::string code; + if (!codegen.codegenFromDrgn(rootType.type, code)) + throw std::runtime_error("oil jit codegen failed!"); - std::string code(headers::oi_OITraceCode_cpp); - - auto codegen = OICodeGen::buildFromConfig(generatorConfig, *symbols); - if (!codegen) { - return OIL_COMPILATION_FAILURE; - } - - codegen->setRootType(rootType->type); - if (!codegen->generate(code)) { - return Response::OIL_COMPILATION_FAILURE; - } - - std::string sourcePath = _self->opts.sourceFileDumpPath; - if (_self->opts.sourceFileDumpPath.empty()) { - // This is the path Clang acts as if it has compiled from e.g. for debug - // information. It does not need to exist. - sourcePath = "oil_jit.cpp"; + std::string sourcePath = opts_.sourceFileDumpPath; + if (sourcePath.empty()) { + sourcePath = "oil_jit.cpp"; // fake path for JIT debug info } else { std::ofstream outputFile(sourcePath); outputFile << code; } - if (!compiler.compile(code, sourcePath, objectPath)) { - return Response::OIL_COMPILATION_FAILURE; - } + auto object = MemoryFile("oil_object_code"); + OICompiler compiler{symbols, compilerConfig_}; + if (!compiler.compile(code, sourcePath, object.path())) + throw std::runtime_error("oil jit compilation failed!"); auto relocRes = compiler.applyRelocs( - reinterpret_cast(segConfig.textSegBase), {objectPath}, {}); - if (!relocRes.has_value()) { - return Response::OIL_RELOCATION_FAILURE; - } + reinterpret_cast(textSeg.data().data()), {object.path()}, {}); + if (!relocRes) + throw std::runtime_error("oil jit relocation failed!"); - const auto& [_, segments, jitSymbols] = relocRes.value(); + const auto& [_, segments, jitSymbols] = *relocRes; - // Locate the probe's entry point - _self->fp = nullptr; + std::string nameHash = + (boost::format("%1$016x") % + std::hash{}(SymbolService::getTypeName(rootType.type))) + .str(); + std::string functionSymbolPrefix = "_Z27introspect_" + nameHash; + std::string typeSymbolName = "treeBuilderInstructions" + nameHash; + void* fp = nullptr; + const exporters::inst::Inst* ty = nullptr; for (const auto& [symName, symAddr] : jitSymbols) { - if (symName.starts_with("_Z7getSize")) { - _self->fp = (size_t(*)(const void*))symAddr; - break; + if (fp == nullptr && symName.starts_with(functionSymbolPrefix)) { + fp = reinterpret_cast(symAddr); + if (ty != nullptr) + break; + } else if (ty == nullptr && symName == typeSymbolName) { + ty = reinterpret_cast(symAddr); + if (fp != nullptr) + break; } } - if (!_self->fp) { - return Response::OIL_RELOCATION_FAILURE; - } - // Copy relocated segments in their final destination - for (const auto& [BaseAddr, RelocAddr, Size] : segments) - memcpy((void*)RelocAddr, (void*)BaseAddr, Size); + CHECK(fp != nullptr && ty != nullptr) + << "failed to find always present symbols!"; - return Response::OIL_SUCCESS; + for (const auto& [baseAddr, relocAddr, size] : segments) + std::memcpy(reinterpret_cast(relocAddr), + reinterpret_cast(baseAddr), size); + + textSeg.release(); // don't munmap() the region containing the code + return {fp, *ty}; } -} // namespace ObjectIntrospection +namespace { +std::map convertFeatures(std::unordered_set fs) { + std::map out{ + {Feature::TypeGraph, true}, + {Feature::TypedDataSegment, true}, + {Feature::TreeBuilderTypeChecking, true}, + {Feature::TreeBuilderV2, true}, + {Feature::Library, true}, + {Feature::PackStructs, true}, + {Feature::PruneTypeGraph, true}, + }; + + for (const auto f : fs) { + switch (f) { + case oi::Feature::ChaseRawPointers: + out[Feature::ChaseRawPointers] = true; + break; + case oi::Feature::CaptureThriftIsset: + out[Feature::CaptureThriftIsset] = true; + break; + case oi::Feature::GenJitDebug: + out[Feature::GenJitDebug] = true; + break; + } + } + + return out; +} + +drgn_qualified_type getTypeFromAtomicHole(drgn_program* prog, void* hole) { + // get the getter type: + // std::atomic (*)(const T&)>& getIntrospectionFunc(); + auto atomicGetterType = + SymbolService::findTypeOfAddr(prog, reinterpret_cast(hole)); + if (!atomicGetterType) + throw std::runtime_error("failed to lookup function"); + + // get the return type: + // std::atomic (*)(const T&)>& + CHECK(drgn_type_has_type(atomicGetterType->type)) + << "functions have a return type"; + auto retType = drgn_type_type(atomicGetterType->type); + + // get the atomic type: + // std::atomic (*)(const T&)> + CHECK(drgn_type_has_type(retType.type)) << "pointers have a value type"; + auto atomicType = drgn_type_type(retType.type); + + // get the function pointer type: + // std::vector (*)(const T&) + CHECK(drgn_type_has_template_parameters(atomicType.type)) + << "atomic should have template parameters"; + CHECK(drgn_type_num_template_parameters(atomicType.type) == 1) + << "atomic should have 1 template parameter"; + auto* templateParam = drgn_type_template_parameters(atomicType.type); + struct drgn_qualified_type funcPointerType; + if (auto err = drgn_template_parameter_type(templateParam, &funcPointerType)) + throw drgnplusplus::error(err); + + // get the function type: + // std::vector(const T&) + CHECK(drgn_type_has_type(funcPointerType.type)) + << "function pointers have a value type"; + auto funcType = drgn_type_type(funcPointerType.type); + + // get the argument type: + // const T& + CHECK(drgn_type_has_parameters(funcType.type)) << "functions have parameters"; + CHECK(drgn_type_num_parameters(funcType.type) == 2) + << "function should have 2 parameters"; + drgn_qualified_type argType; + if (auto err = + drgn_parameter_type(drgn_type_parameters(funcType.type), &argType)) + throw drgnplusplus::error(err); + + // get the type + CHECK(drgn_type_has_type(argType.type)) + << "reference types have a value type"; + return drgn_type_type(argType.type); +} + +} // namespace +} // namespace oi::detail diff --git a/oi/OILibraryImpl.h b/oi/OILibraryImpl.h index 46af148..89bc083 100644 --- a/oi/OILibraryImpl.h +++ b/oi/OILibraryImpl.h @@ -14,38 +14,76 @@ * limitations under the License. */ #pragma once +#include -#include "ObjectIntrospection.h" -#include "oi/OICodeGen.h" +#include +#include +#include +#include +#include + +#include "oi/CodeGen.h" +#include "oi/Features.h" #include "oi/OICompiler.h" -#include "oi/SymbolService.h" -namespace ObjectIntrospection { +namespace oi::detail { class OILibraryImpl { - public: - OILibraryImpl(OILibrary*, void*); - ~OILibraryImpl(); + private: + class LocalTextSegment { + public: + LocalTextSegment() = default; + LocalTextSegment(size_t size); + ~LocalTextSegment(); + LocalTextSegment(const LocalTextSegment&) = delete; + LocalTextSegment& operator=(const LocalTextSegment&) = delete; + LocalTextSegment(LocalTextSegment&& that) { + std::swap(this->data_, that.data_); + } + LocalTextSegment& operator=(LocalTextSegment&& that) { + std::swap(this->data_, that.data_); + return *this; + } - bool mapSegment(); - bool unmapSegment(); - void initCompiler(); - int compileCode(); - bool processConfigFile(); - void enableLayoutAnalysis(); + std::span data() { + return data_; + } + void release() { + data_ = {}; + } + + private: + std::span data_; + }; + class MemoryFile { + public: + MemoryFile(const char* name); + ~MemoryFile(); + + std::filesystem::path path(); + + private: + int fd_ = -1; + }; + + public: + OILibraryImpl(void* atomicHole, + std::unordered_set fs, + GeneratorOptions opts); + std::pair init(); private: - class OILibrary* _self; + void* atomicHole_; + std::map requestedFeatures_; + GeneratorOptions opts_; - void* _TemplateFunc; + oi::detail::OICompiler::Config compilerConfig_{}; + oi::detail::OICodeGen::Config generatorConfig_{}; - oi::detail::OICompiler::Config compilerConfig{}; - oi::detail::OICodeGen::Config generatorConfig{}; - std::shared_ptr symbols{}; + LocalTextSegment textSeg; - struct c { - void* textSegBase = nullptr; - size_t textSegSize = 1u << 22; - } segConfig; + void processConfigFile(); + std::pair compileCode(); }; -} // namespace ObjectIntrospection + +} // namespace oi::detail diff --git a/oi/OITraceCode.cpp b/oi/OITraceCode.cpp index 59c99c7..0660fbe 100644 --- a/oi/OITraceCode.cpp +++ b/oi/OITraceCode.cpp @@ -34,12 +34,6 @@ #define C10_USING_CUSTOM_GENERATED_MACROS -// These globals are set by oid, see end of OIDebugger::compileCode() -extern uint8_t* dataBase; -extern size_t dataSize; -extern uintptr_t cookieValue; -extern int logFile; - constexpr int oidMagicId = 0x01DE8; #include @@ -89,21 +83,6 @@ class { } } static pointers; -void __jlogptr(uintptr_t ptr) { - static constexpr char hexdigits[] = "0123456789abcdef"; - static constexpr size_t ptrlen = 2 * sizeof(ptr); - - static char hexstr[ptrlen + 1] = {}; - - size_t i = ptrlen; - while (i--) { - hexstr[i] = hexdigits[ptr & 0xf]; - ptr = ptr >> 4; - } - hexstr[ptrlen] = '\n'; - write(logFile, hexstr, sizeof(hexstr)); -} - } // namespace // alignas(0) is ignored according to docs so can be default diff --git a/oi/SymbolService.cpp b/oi/SymbolService.cpp index 514d4ee..d34ce79 100644 --- a/oi/SymbolService.cpp +++ b/oi/SymbolService.cpp @@ -298,7 +298,7 @@ bool SymbolService::loadModules() { return true; } -static std::optional findTypeOfSymbol( +std::optional SymbolService::findTypeOfSymbol( drgn_program* prog, const std::string& symbolName) { drgn_symbol* sym; if (auto* err = @@ -313,6 +313,16 @@ static std::optional findTypeOfSymbol( uint64_t addr = drgn_symbol_address(sym); drgn_symbol_destroy(sym); + if (auto t = findTypeOfAddr(prog, addr)) { + return t; + } else { + LOG(ERROR) << "Failed to lookup symbol '" << symbolName; + return std::nullopt; + } +} + +std::optional SymbolService::findTypeOfAddr( + drgn_program* prog, uintptr_t addr) { drgn_object obj; drgn_object_init(&obj, prog); @@ -320,7 +330,7 @@ static std::optional findTypeOfSymbol( if (auto* err = drgn_program_find_function_by_address(prog, addr, &name, &obj); err != nullptr) { - LOG(ERROR) << "Failed to lookup function '" << symbolName + LOG(ERROR) << "Failed to lookup function '" << reinterpret_cast(addr) << "': " << err->code << " " << err->message; drgn_error_destroy(err); return std::nullopt; @@ -598,7 +608,7 @@ static std::optional> createFuncDesc( struct drgn_program* prog, const irequest& request) { VLOG(1) << "Creating function description for: " << request.func; - auto ft = findTypeOfSymbol(prog, request.func); + auto ft = SymbolService::findTypeOfSymbol(prog, request.func); if (!ft) { return std::nullopt; } diff --git a/oi/SymbolService.h b/oi/SymbolService.h index b741098..c95f125 100644 --- a/oi/SymbolService.h +++ b/oi/SymbolService.h @@ -56,6 +56,11 @@ class SymbolService { static std::string getTypeName(struct drgn_type*); std::optional getRootType(const irequest&); + static std::optional findTypeOfSymbol( + drgn_program*, const std::string& symbolName); + static std::optional findTypeOfAddr(drgn_program*, + uintptr_t addr); + std::unordered_map> funcDescs; std::unordered_map> globalDescs; diff --git a/oi/exporters/Json.cpp b/oi/exporters/Json.cpp new file mode 100644 index 0000000..c3bdc5b --- /dev/null +++ b/oi/exporters/Json.cpp @@ -0,0 +1,125 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +#include + +template +inline constexpr bool always_false_v = false; + +namespace oi::exporters { +namespace { + +template +void printStringList(std::ostream& out, It it, It end, bool pretty) { + out << '['; + for (; it != end; ++it) { + out << '"' << (*it) << '"'; + if (it != end - 1) + out << (pretty ? ", " : ","); + } + out << ']'; +} + +std::string makeIndent(std::vector& tp) { + return std::string((tp.size() - 1) * 4, ' '); +} + +} // namespace + +Json::Json(std::ostream& out) : out_(out) { +} + +void Json::print(const IntrospectionResult& r) { + auto begin = r.cbegin(); + return print(begin, r.cend()); +} + +void Json::print(IntrospectionResult::const_iterator& it, + IntrospectionResult::const_iterator end) { + std::vector firstTypePath = it->type_path; + + const auto indent = pretty_ ? makeIndent(firstTypePath) : ""; + const auto* tab = pretty_ ? " " : ""; + const auto* space = pretty_ ? " " : ""; + const auto* endl = pretty_ ? "\n" : ""; + + out_ << '['; + out_ << endl << indent; + + bool first = true; + while (it != end) { + if (it->type_path.size() < firstTypePath.size()) { + // no longer a sibling, must be a sibling of the type we're printing + break; + } + + if (!first) + out_ << ',' << endl << indent; + first = false; + + out_ << '{' << endl << indent; + + out_ << tab << "\"name\"" << space << ':' << space << "\"" << it->name + << "\"," << endl + << indent; + + out_ << tab << "\"typePath\"" << space << ':' << space << ""; + printStringList(out_, it->type_path.begin(), it->type_path.end(), pretty_); + out_ << (pretty_ ? ",\n" : ",") << indent; + + out_ << tab << "\"typeNames\"" << space << ':' << space; + printStringList(out_, it->type_names.begin(), it->type_names.end(), + pretty_); + out_ << ',' << endl << indent; + + out_ << tab << "\"staticSize\":" << space << it->static_size << ',' << endl + << indent; + out_ << tab << "\"exclusiveSize\":" << space << it->exclusive_size << ',' + << endl + << indent; + + if (it->pointer.has_value()) { + out_ << tab << "\"pointer\":" << space << *(it->pointer) << ',' << endl + << indent; + } + if (it->container_stats.has_value()) { + out_ << tab << "\"length\":" << space << it->container_stats->length + << ',' << endl + << indent; + out_ << tab << "\"capacity\":" << space << it->container_stats->capacity + << ',' << endl + << indent; + } + if (it->is_set_stats.has_value()) { + out_ << tab << "\"is_set\":" << space << it->is_set_stats->is_set << ',' + << endl + << indent; + } + + out_ << tab << "\"members\":" << space; + if ((++it)->type_path.size() > firstTypePath.size()) { + print(it, end); + } else { + out_ << "[]" << endl << indent; + } + + out_ << "}"; + } + out_ << endl << indent << ']' << endl; +} + +} // namespace oi::exporters diff --git a/oi/exporters/ParsedData.cpp b/oi/exporters/ParsedData.cpp new file mode 100644 index 0000000..b8e779b --- /dev/null +++ b/oi/exporters/ParsedData.cpp @@ -0,0 +1,76 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +#include +#include +#include + +template +constexpr bool always_false_v = false; + +namespace oi::exporters { +namespace { +uint64_t parseVarint(std::vector::const_iterator& it); +} + +ParsedData ParsedData::parse(std::vector::const_iterator& it, + types::dy::Dynamic dy) { + return std::visit( + [&it](const auto el) -> ParsedData { + auto ty = el.get(); + using T = std::decay_t; + if constexpr (std::is_same_v) { + return ParsedData::Unit{}; + } else if constexpr (std::is_same_v) { + return ParsedData::VarInt{.value = parseVarint(it)}; + } else if constexpr (std::is_same_v) { + return ParsedData::Pair{ + .first = Lazy{it, ty.first}, + .second = Lazy{it, ty.second}, + }; + } else if constexpr (std::is_same_v) { + return ParsedData::List{ + .length = parseVarint(it), + .values = {it, ty.element}, + }; + } else if constexpr (std::is_same_v) { + auto index = parseVarint(it); + assert(index < ty.variants.size()); + return ParsedData::Sum{ + .index = parseVarint(it), + .value = {it, ty.variants[index]}, + }; + } else { + static_assert(always_false_v, "non-exhaustive visitor!"); + } + }, + dy); +} + +namespace { +uint64_t parseVarint(std::vector::const_iterator& it) { + uint64_t v = 0; + int shift = 0; + while (*it >= 0x80) { + v |= static_cast(*it++ & 0x7f) << shift; + shift += 7; + } + v |= static_cast(*it++ & 0x7f) << shift; + return v; +} +} // namespace +} // namespace oi::exporters diff --git a/oi/type_graph/NameGen.cpp b/oi/type_graph/NameGen.cpp index 4bd48cd..7b9ea67 100644 --- a/oi/type_graph/NameGen.cpp +++ b/oi/type_graph/NameGen.cpp @@ -71,6 +71,8 @@ void NameGen::visit(Class& c) { std::string name = c.name(); removeTemplateParams(name); deduplicate(name); + if (c.name().empty()) + c.setInputName(name); c.setName(name); // Deduplicate member names. Duplicates may be present after flattening. @@ -151,6 +153,8 @@ void NameGen::visit(Container& c) { void NameGen::visit(Enum& e) { std::string name = e.name(); deduplicate(name); + if (e.name().empty()) + e.setInputName(name); e.setName(name); } diff --git a/oi/type_graph/PassManager.cpp b/oi/type_graph/PassManager.cpp index 2b41589..dff4a28 100644 --- a/oi/type_graph/PassManager.cpp +++ b/oi/type_graph/PassManager.cpp @@ -40,7 +40,6 @@ namespace { void print(const TypeGraph& typeGraph) { if (!VLOG_IS_ON(1)) return; - std::stringstream out; Printer printer{out, typeGraph.resetTracker(), typeGraph.size()}; for (const auto& type : typeGraph.rootTypes()) { diff --git a/oi/type_graph/Types.h b/oi/type_graph/Types.h index 6f2ee4a..201aab5 100644 --- a/oi/type_graph/Types.h +++ b/oi/type_graph/Types.h @@ -231,6 +231,10 @@ class Class : public Type { name_ = std::move(name); } + void setInputName(std::string name) { + inputName_ = std::move(name); + } + virtual size_t size() const override { return size_; } @@ -376,6 +380,10 @@ class Enum : public Type { return inputName_; } + void setInputName(std::string name) { + inputName_ = std::move(name); + } + void setName(std::string name) { name_ = std::move(name); } diff --git a/resources/CMakeLists.txt b/resources/CMakeLists.txt index 2fc0df8..fe16d69 100644 --- a/resources/CMakeLists.txt +++ b/resources/CMakeLists.txt @@ -7,6 +7,9 @@ function(embed_headers output) file(APPEND ${output} "namespace oi::detail::headers {\n") set(HEADERS + ../include/oi/exporters/ParsedData.h + ../include/oi/exporters/inst.h + ../include/oi/result/Element.h ../include/oi/types/dy.h ../include/oi/types/st.h ../oi/OITraceCode.cpp diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index db48992..12cb239 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -50,7 +50,7 @@ add_executable(integration_test_target ${INTEGRATION_TEST_TARGET_SRC} folly_shims.cpp) target_compile_options(integration_test_target PRIVATE -O1) -target_link_libraries(integration_test_target PRIVATE oil Boost::headers ${Boost_LIBRARIES}) +target_link_libraries(integration_test_target PRIVATE oil_jit exporters_json Boost::headers ${Boost_LIBRARIES}) add_executable(integration_test_runner ${INTEGRATION_TEST_RUNNER_SRC} runner_common.cpp) target_include_directories(integration_test_runner PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/test/integration/alignment.toml b/test/integration/alignment.toml index ca3c70f..f118548 100644 --- a/test/integration/alignment.toml +++ b/test/integration/alignment.toml @@ -56,6 +56,12 @@ definitions = ''' {"typeName": "Align16", "staticSize": 16, "exclusiveSize": 15, "members": [ {"typeName": "char", "staticSize": 1, "exclusiveSize": 1} ]}]}]''' + expect_json_v2 = '''[ + {"staticSize": 32, "exclusiveSize": 15, "members": [ + {"typeNames": ["int8_t", "__int8_t", "int8_t"], "staticSize": 1, "exclusiveSize": 1}, + {"typeNames": ["Align16"], "staticSize": 16, "exclusiveSize": 15, "members": [ + {"typeNames": ["int8_t"], "staticSize": 1, "exclusiveSize": 1} + ]}]}]''' [cases.container_struct] skip = "container alignment is broken (#143)" param_types = ["const std::optional&"] @@ -78,6 +84,16 @@ definitions = ''' {"typeName": "Align16", "staticSize": 16, "exclusiveSize": 15, "members": [ {"typeName": "char", "staticSize": 1, "exclusiveSize": 1}]} ]}]}]''' + expect_json_v2 = '''[ + {"staticSize": 64, "exclusiveSize": 15, "members": [ + {"typeNames": ["int8_t", "__int8_t", "int8_t"], "staticSize": 1, "exclusiveSize": 1}, + {"typeNames": ["TwoStruct"], "staticSize": 48, "exclusiveSize": 15, "members": [ + {"typeNames": ["Align16"], "staticSize": 16, "exclusiveSize": 15, "members": [ + {"typeNames": ["int8_t"], "staticSize": 1, "exclusiveSize": 1}]}, + {"typeNames": ["int8_t"], "staticSize": 1, "exclusiveSize": 1}, + {"typeNames": ["Align16"], "staticSize": 16, "exclusiveSize": 15, "members": [ + {"typeNames": ["int8_t"], "staticSize": 1, "exclusiveSize": 1}]} + ]}]}]''' [cases.container_two_members] skip = "container alignment is broken (#143)" param_types = ["const std::optional&"] @@ -101,6 +117,13 @@ definitions = ''' {"typeName": "char", "staticSize": 1, "exclusiveSize": 1}, {"typeName": "char", "staticSize": 1, "exclusiveSize": 1} ]}]}]''' + expect_json_v2 = '''[ + {"staticSize": 96, "exclusiveSize": 31, "members": [ + {"typeNames": ["int8_t", "__int8_t", "int8_t"], "staticSize": 1, "exclusiveSize": 1}, + {"typeNames": ["MemberAlignment"], "staticSize": 64, "exclusiveSize": 62, "members": [ + {"typeNames": ["int8_t"], "staticSize": 1, "exclusiveSize": 1}, + {"typeNames": ["int8_t"], "staticSize": 1, "exclusiveSize": 1} + ]}]}]''' [cases.container_member_alignment] skip = "container alignment is broken (#143)" param_types = ["const std::optional&"] @@ -119,6 +142,11 @@ definitions = ''' {"typeName": "int8_t", "staticSize": 1, "exclusiveSize": 1}, {"typeName": "UnionMember", "staticSize": 32, "exclusiveSize": 32, "NOT":"members"} ]}]''' + expect_json_v2 = '''[ + {"staticSize": 64, "exclusiveSize": 31, "members": [ + {"typeNames": ["int8_t", "__int8_t", "int8_t"], "staticSize": 1, "exclusiveSize": 1}, + {"typeNames": ["UnionMember"], "staticSize": 32, "exclusiveSize": 32, "members":[]} + ]}]''' [cases.container_union_member] skip = "container alignment is broken (#143)" param_types = ["const std::optional&"] @@ -138,6 +166,14 @@ definitions = ''' {"typeName": "Align16", "staticSize": 16, "exclusiveSize": 15, "members": [ {"typeName": "char", "staticSize": 1, "exclusiveSize": 1} ]}]}]}]''' + expect_json_v2 = '''[ + {"staticSize": 96, "exclusiveSize": 31, "members": [ + {"typeNames": ["int8_t", "__int8_t", "int8_t"], "staticSize": 1, "exclusiveSize": 1}, + {"typeNames": ["MemberAlignmentOverriden"], "staticSize": 64, "exclusiveSize": 47, "members": [ + {"typeNames": ["int8_t"], "staticSize": 1, "exclusiveSize": 1}, + {"typeNames": ["Align16"], "staticSize": 16, "exclusiveSize": 15, "members": [ + {"typeNames": ["int8_t"], "staticSize": 1, "exclusiveSize": 1} + ]}]}]}]''' [cases.container_member_override] skip = "container alignment is broken (#143)" param_types = ["const std::optional&"] @@ -159,6 +195,13 @@ definitions = ''' {"typeName": "char", "staticSize": 1, "exclusiveSize": 1}, {"typeName": "char", "staticSize": 1, "exclusiveSize": 1} ]}]}]''' + expect_json_v2 = '''[ + {"staticSize": 256, "exclusiveSize": 127, "members": [ + {"typeNames": ["int8_t", "__int8_t", "int8_t"], "staticSize": 1, "exclusiveSize": 1}, + {"typeNames": ["AlignedStructMemberAlignLower"], "staticSize": 128, "exclusiveSize": 126, "members": [ + {"typeNames": ["int8_t"], "staticSize": 1, "exclusiveSize": 1}, + {"typeNames": ["int8_t"], "staticSize": 1, "exclusiveSize": 1} + ]}]}]''' [cases.container_member_lower] skip = "container alignment is broken (#143)" param_types = ["const std::optional&"] diff --git a/test/integration/anonymous.toml b/test/integration/anonymous.toml index c72ee91..929b6ce 100644 --- a/test/integration/anonymous.toml +++ b/test/integration/anonymous.toml @@ -192,6 +192,7 @@ definitions = ''' }]''' [cases.anon_union] + oil_skip = "anonymous unions are fully stubbed in the generated code" # https://github.com/facebookexperimental/object-introspection/issues/292 param_types = ["const AnonUnionContainer&"] setup = 'return AnonUnionContainer{ .a = 3 };' cli_options = ["-fchase-raw-pointers"] @@ -204,6 +205,15 @@ definitions = ''' {"name":"e", "staticSize":4, "dynamicSize":0, "typeName":"int"} ] }]''' + expect_json_v2 = '''[{ + "staticSize": 24, + "exclusiveSize": 10, + "members": [ + {"name":"__anon_member_0", "staticSize":2, "exclusiveSize":2}, + {"name":"__anon_member_1", "staticSize":8, "exclusiveSize":8}, + {"name":"e", "staticSize":4, "exclusiveSize":4, "typeNames":["int32_t"]} + ] + }]''' [cases.nested_anon_struct] oil_disable = "oil can't chase raw pointers safely" diff --git a/test/integration/arrays.toml b/test/integration/arrays.toml index a7b5478..3efb2d2 100644 --- a/test/integration/arrays.toml +++ b/test/integration/arrays.toml @@ -30,7 +30,17 @@ definitions = ''' "capacity":10, "elementStaticSize":4 }]}]''' + expect_json_v2 = '''[{ + "staticSize":40, + "exclusiveSize":0, + "members":[{ + "staticSize":40, + "exclusiveSize":0, + "length":10, + "capacity":10 + }]}]''' [cases.member_int0] + oil_skip = 'zero length arrays fail codegen v2' # https://github.com/facebookexperimental/object-introspection/issues/295 # WARNING: zero-length arrays are handled differently to non-empty arrays. # They end up not being treated as containers. This should probably change # in the future. @@ -43,8 +53,16 @@ definitions = ''' "staticSize":0, "dynamicSize":0 }]}]''' + expect_json_v2 = '''[{ + "staticSize":1, + "exclusiveSize":1, + "members":[{ + "staticSize":0, + "exclusiveSize":0 + }]}]''' [cases.multidim_legacy] # Test for legacy behaviour. Remove with OICodeGen - cli_options = ["-Ftype-graph", "-Ftyped-data-segment", "-Ftree-builder-type-checking"] + oil_disable = 'oil only runs on codegen v2' + cli_options = ["-Ftype-graph", "-Ftyped-data-segment", "-Ftree-builder-type-checking", "-Ftree-builder-v2"] param_types = ["const MultiDim&"] setup = "return {};" expect_json = '''[{ @@ -67,6 +85,12 @@ definitions = ''' {"staticSize":12, "dynamicSize":0, "exclusiveSize":12, "length":3, "capacity":3, "elementStaticSize":4}, {"staticSize":12, "dynamicSize":0, "exclusiveSize":12, "length":3, "capacity":3, "elementStaticSize":4}] }]}]''' + expect_json_v2 = '''[ + {"staticSize":24, "exclusiveSize":0, "members":[ + {"staticSize":24, "exclusiveSize":0, "length":2, "capacity":2, "members":[ + {"staticSize":12, "exclusiveSize":0, "length":3, "capacity":3}, + {"staticSize":12, "exclusiveSize":0, "length":3, "capacity":3}] + }]}]''' [cases.direct_int10] skip = "Direct array arguments don't work" param_types = ["int[10]"] diff --git a/test/integration/cycles.toml b/test/integration/cycles.toml index d98d9eb..44ad7f1 100644 --- a/test/integration/cycles.toml +++ b/test/integration/cycles.toml @@ -154,6 +154,7 @@ definitions = ''' ''' [cases.unique_ptr] + oil_skip = "cycles are broken" # https://github.com/facebookexperimental/object-introspection/issues/293 param_types = ["std::reference_wrapper&"] setup = ''' auto first = std::make_unique(); @@ -231,6 +232,7 @@ definitions = ''' ''' [cases.shared_ptr] + oil_skip = "cycles are broken" # https://github.com/facebookexperimental/object-introspection/issues/293 param_types = ["std::reference_wrapper&"] setup = ''' auto first = std::make_shared(); diff --git a/test/integration/enums_params.toml b/test/integration/enums_params.toml index 1426a3e..cf61d6f 100644 --- a/test/integration/enums_params.toml +++ b/test/integration/enums_params.toml @@ -38,7 +38,8 @@ definitions = ''' [cases.scoped_enum_val_cast] param_types = ["const std::array(MyNS::ScopedEnum::Two)>&"] setup = "return {};" - expect_json = '[{"staticSize":8, "dynamicSize":0, "length":2, "capacity":2, "elementStaticSize":4}]' + expect_json = '[{"staticSize":8, "length":2, "capacity":2, "elementStaticSize":4}]' + expect_json_v2 = '[{"staticSize":8, "dynamicSize":0, "length":2, "capacity":2}]' [cases.scoped_enum_val] param_types = ["const MyClass&"] @@ -59,4 +60,5 @@ definitions = ''' [cases.unscoped_enum_val_cast] param_types = ["const std::array&"] setup = "return {};" - expect_json = '[{"staticSize":4, "dynamicSize":0, "length":1, "capacity":1, "elementStaticSize":4}]' + expect_json = '[{"staticSize":4, "length":1, "capacity":1, "elementStaticSize":4}]' + expect_json_v2 = '[{"staticSize":4, "length":1, "capacity":1}]' diff --git a/test/integration/fbstring.toml b/test/integration/fbstring.toml index a115555..9a3a255 100644 --- a/test/integration/fbstring.toml +++ b/test/integration/fbstring.toml @@ -1,6 +1,7 @@ includes = ["folly/FBString.h"] [cases] [cases.empty] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/322 param_types = ["folly::fbstring&"] setup = "return {};" expect_json = ''' @@ -26,6 +27,7 @@ includes = ["folly/FBString.h"] ''' [cases.inline] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/322 param_types = ["folly::fbstring&"] setup = 'return {"012345"};' expect_json = ''' @@ -51,6 +53,7 @@ includes = ["folly/FBString.h"] ''' [cases.heap_allocated] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/322 param_types = ["folly::fbstring&"] setup = 'return {"abcdefghijklmnopqrstuvwxzy"};' expect_json = ''' @@ -76,6 +79,7 @@ includes = ["folly/FBString.h"] ''' [cases.string_pooled] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/322 param_types = ["folly::fbstring&"] setup = "return folly::fbstring(1024, 'c');" expect_json = ''' diff --git a/test/integration/folly_small_vector.toml b/test/integration/folly_small_vector.toml index 89e3ae3..360c323 100644 --- a/test/integration/folly_small_vector.toml +++ b/test/integration/folly_small_vector.toml @@ -1,23 +1,28 @@ includes = ["folly/small_vector.h", "vector"] [cases] [cases.int_default_empty] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/321 param_types = ["const folly::small_vector&"] setup = "return {};" expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "length":0, "capacity":2, "elementStaticSize":4}]' [cases.int_default_inlined] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/321 param_types = ["const folly::small_vector&"] setup = "return {{1,2}};" expect_json = '[{"staticSize":16, "dynamicSize":0, "exclusiveSize":16, "length":2, "capacity":2, "elementStaticSize":4}]' [cases.int_default_overflow] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/321 param_types = ["const folly::small_vector&"] setup = "return {{1,2,3,4}};" expect_json = '[{"staticSize":16, "dynamicSize":24, "exclusiveSize":40, "length":4, "capacity":6, "elementStaticSize":4}]' [cases.vector_3_empty] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/321 param_types = ["const folly::small_vector, 3>&"] setup = "return {};" expect_json = '[{"staticSize":80, "dynamicSize":0, "exclusiveSize":80, "length":0, "capacity":3, "elementStaticSize":24}]' [cases.vector_3_inlined] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/321 param_types = ["const folly::small_vector, 3>&"] setup = "return {{ {1,2,3}, {4}, {5,6} }};" expect_json = '''[ @@ -27,6 +32,7 @@ includes = ["folly/small_vector.h", "vector"] {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4} ]}]''' [cases.vector_3_overflow] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/321 param_types = ["const folly::small_vector, 3>&"] setup = "return {{ {1,2,3}, {4}, {5,6}, {7} }};" expect_json = '''[ @@ -38,6 +44,7 @@ includes = ["folly/small_vector.h", "vector"] ]}]''' [cases.int_always_heap] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/321 param_types = ["const folly::small_vector&"] setup = "return {{1}};" expect_json = '[{"staticSize":16, "dynamicSize":4, "exclusiveSize":20, "length":1, "capacity":1, "elementStaticSize":4}]' diff --git a/test/integration/folly_sorted_vector_map.toml b/test/integration/folly_sorted_vector_map.toml index 2db67ab..6d3a2a5 100644 --- a/test/integration/folly_sorted_vector_map.toml +++ b/test/integration/folly_sorted_vector_map.toml @@ -4,10 +4,39 @@ includes = ["folly/sorted_vector_types.h", "vector"] param_types = ["const folly::sorted_vector_map&"] setup = "return {};" expect_json = '[{"staticSize":24, "dynamicSize":0, "exclusiveSize":24, "length":0, "capacity":0, "elementStaticSize":8}]' + expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]' [cases.int_int_some] param_types = ["const folly::sorted_vector_map&"] setup = "return {{ {1,2}, {3,4} }};" expect_json = '[{"staticSize":24, "dynamicSize":16, "exclusiveSize":40, "length":2, "capacity":2, "elementStaticSize":8}]' + expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":2, "capacity":2, "members":[ + {"staticSize":8, "exclusiveSize":0, "members": [ + {"name":"key", "staticSize":4, "exclusiveSize":4}, + {"name":"value", "staticSize":4, "exclusiveSize":4} + ]}, + {"staticSize":8, "exclusiveSize":0, "members": [ + {"name":"key", "staticSize":4, "exclusiveSize":4}, + {"name":"value", "staticSize":4, "exclusiveSize":4} + ]} + ]}]''' + [cases.int_int_reserve] + param_types = ["const folly::sorted_vector_map&"] + setup = ''' + folly::sorted_vector_map m{{ {1,2}, {3,4} }}; + m.reserve(10); + return m; + ''' + expect_json = '[{"staticSize":24, "dynamicSize":80, "exclusiveSize":104, "length":2, "capacity":10, "elementStaticSize":8}]' + expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":88, "length":2, "capacity":10, "members":[ + {"staticSize":8, "exclusiveSize":0, "members": [ + {"name":"key", "staticSize":4, "exclusiveSize":4}, + {"name":"value", "staticSize":4, "exclusiveSize":4} + ]}, + {"staticSize":8, "exclusiveSize":0, "members": [ + {"name":"key", "staticSize":4, "exclusiveSize":4}, + {"name":"value", "staticSize":4, "exclusiveSize":4} + ]} + ]}]''' [cases.int_vector_empty] skip = "Wrong results" # https://github.com/facebookexperimental/object-introspection/issues/258 diff --git a/test/integration/gen_tests.py b/test/integration/gen_tests.py index f3db8fa..71c8504 100644 --- a/test/integration/gen_tests.py +++ b/test/integration/gen_tests.py @@ -38,7 +38,8 @@ def add_headers(f, custom_headers, thrift_headers): #include #include -#include +#include +#include """ ) @@ -83,11 +84,11 @@ def add_test_setup(f, config): def define_traceable_func(name, params, body): return ( f"\n" - f' extern "C" {{\n' - f" void __attribute__((noinline)) {name}({params}) {{\n" + # f' extern "C" {{\n' + f' extern "C" void __attribute__((noinline)) {name}({params}) {{\n' f"{body}" f" }}\n" - f" }}\n" + # f" }}\n" ) cases = config["cases"] @@ -134,22 +135,18 @@ def add_test_setup(f, config): ) oil_func_body = ( - f"\n" - f"ObjectIntrospection::options opts{{\n" - f" .configFilePath = configFile,\n" - f" .debugLevel = 3,\n" - f' .sourceFileDumpPath = "oil_jit_code.cpp",\n' - f"}};" + f" oi::GeneratorOptions opts{{\n" + f" .configFilePath = configFile,\n" + f' .sourceFileDumpPath = "oil_jit_code.cpp",\n' + f" .debugLevel = 3,\n" + f" }};\n\n" ) - oil_func_body += ' std::cout << "{\\"results\\": [" << std::endl;\n' - oil_func_body += ' std::cout << "," << std::endl;\n'.join( - f" size_t size{i} = 0;\n" - f" auto ret{i} = ObjectIntrospection::getObjectSize(a{i}, size{i}, opts);\n" - f' std::cout << "{{\\"ret\\": " << ret{i} << ", \\"size\\": " << size{i} << "}}" << std::endl;\n' - for i in range(len(case["param_types"])) - ) - oil_func_body += ' std::cout << "]}" << std::endl;\n' + oil_func_body += " auto pr = oi::exporters::Json(std::cout);\n" + oil_func_body += " pr.setPretty(true);\n" + for i in range(len(case["param_types"])): + oil_func_body += f" auto ret{i} = oi::setupAndIntrospect(a{i}, opts);\n" + oil_func_body += f" pr.print(*ret{i});\n" f.write( define_traceable_func( @@ -367,23 +364,14 @@ def add_oil_integration_test(f, config, case_name, case): f' .targetArgs = "oil {case_str}",\n' f" }}, std::move(configPrefix), std::move(configSuffix));\n\n" f" ASSERT_EQ(exit_code(target), {exit_code});\n" - f"\n" - f" bpt::ptree result_json;\n" - f" auto json_ss = std::stringstream(stdout_);\n" - f" bpt::read_json(json_ss, result_json);\n" - f" std::vector sizes;\n" - f' for (const auto& each : result_json.get_child("results")) {{\n' - f" const auto& result = each.second;\n" - f' int oilResult = result.get("ret");\n' - f' size_t oilSize = result.get("size");\n' - f" ASSERT_EQ(oilResult, 0);\n" - f" sizes.push_back(oilSize);\n" - f" }}" ) - if "expect_json" in case: + key = "expect_json" + if "expect_json_v2" in case: + key = "expect_json_v2" + if key in case: try: - json.loads(case["expect_json"]) + json.loads(case[key]) except json.decoder.JSONDecodeError as error: print( f"\x1b[31m`expect_json` value for test case {config['suite']}.{case_name} was invalid JSON: {error}\x1b[0m", @@ -394,17 +382,12 @@ def add_oil_integration_test(f, config, case_name, case): f.write( f"\n" f" std::stringstream expected_json_ss;\n" - f' expected_json_ss << R"--({case["expect_json"]})--";\n' - f" bpt::ptree expected_json;\n" + f' expected_json_ss << R"--({case[key]})--";\n' + f" auto result_json_ss = std::stringstream(stdout_);\n" + f" bpt::ptree expected_json, actual_json;\n" f" bpt::read_json(expected_json_ss, expected_json);\n" - f" auto sizes_it = sizes.begin();\n" - f" for (auto it = expected_json.begin(); it != expected_json.end(); ++it, ++sizes_it) {{\n" - f" auto node = it->second;\n" - f' size_t expected_size = node.get("staticSize");\n' - f' if (node.find("dynamicSize") != node.not_found())\n' # Assume dynamicSize is 0 if not set - f' expected_size += node.get("dynamicSize");\n' - f" EXPECT_EQ(*sizes_it, expected_size);\n" - f" }}\n" + f" bpt::read_json(result_json_ss, actual_json);\n" + f" compare_json(expected_json, actual_json);\n" ) f.write(f"}}\n") diff --git a/test/integration/ignored.toml b/test/integration/ignored.toml index 7276374..1ce3abb 100644 --- a/test/integration/ignored.toml +++ b/test/integration/ignored.toml @@ -10,6 +10,7 @@ definitions = ''' [cases] [cases.a] + oil_skip = 'v2 hides the member entirely when it should show it with static size' # todo: github issue param_types = ["const Bar&"] setup = """ return Bar{ diff --git a/test/integration/inheritance_access.toml b/test/integration/inheritance_access.toml index f19b1a2..faeb35c 100644 --- a/test/integration/inheritance_access.toml +++ b/test/integration/inheritance_access.toml @@ -1,18 +1,18 @@ definitions = ''' class Base { - int base_int; + int32_t base_int; }; class Public : public Base { - int public_int; + int32_t public_int; }; class Protected : protected Base { - int protected_int; + int32_t protected_int; }; class Private : private Base { - int private_int; + int32_t private_int; }; ''' [cases] @@ -21,30 +21,27 @@ definitions = ''' setup = "return {};" expect_json = '''[{ "staticSize":8, - "dynamicSize":0, "members":[ - {"name":"base_int", "staticSize":4, "dynamicSize":0, "typeName": "int"}, - {"name":"public_int", "staticSize":4, "dynamicSize":0, "typeName": "int"} + {"name":"base_int", "staticSize":4, "typeName": "int32_t"}, + {"name":"public_int", "staticSize":4, "typeName": "int32_t"} ]}]''' [cases.protected] param_types = ["const Protected&"] setup = "return {};" expect_json = '''[{ "staticSize":8, - "dynamicSize":0, "members":[ - {"name":"base_int", "staticSize":4, "dynamicSize":0, "typeName": "int"}, - {"name":"protected_int", "staticSize":4, "dynamicSize":0, "typeName": "int"} + {"name":"base_int", "staticSize":4, "typeName": "int32_t"}, + {"name":"protected_int", "staticSize":4, "typeName": "int32_t"} ]}]''' [cases.private] param_types = ["const Private&"] setup = "return {};" expect_json = '''[{ "staticSize":8, - "dynamicSize":0, "members":[ - {"name":"base_int", "staticSize":4, "dynamicSize":0, "typeName": "int"}, - {"name":"private_int", "staticSize":4, "dynamicSize":0, "typeName": "int"} + {"name":"base_int", "staticSize":4, "typeName": "int32_t"}, + {"name":"private_int", "staticSize":4, "typeName": "int32_t"} ]}]''' [cases.public_as_base] param_types = ["const Base&"] @@ -52,7 +49,6 @@ definitions = ''' setup = "return {};" expect_json = '''[{ "staticSize":4, - "dynamicSize":0, "members":[ - {"name":"base_int", "staticSize":4, "dynamicSize":0, "typeName": "int"} + {"name":"base_int", "staticSize":4, "typeName": "int32_t"} ]}]''' diff --git a/test/integration/inheritance_multiple.toml b/test/integration/inheritance_multiple.toml index cf297d0..82da4b3 100644 --- a/test/integration/inheritance_multiple.toml +++ b/test/integration/inheritance_multiple.toml @@ -34,3 +34,14 @@ definitions = ''' {"name":"e", "staticSize":4, "dynamicSize":0, "typeName": "int"}, {"name":"f", "staticSize":4, "dynamicSize":0, "typeName": "int"} ]}]''' + expect_json_v2 = '''[{ + "staticSize":24, + "exclusiveSize":0, + "members":[ + {"name":"a", "staticSize":4, "exclusiveSize":4, "typeNames": ["int32_t"]}, + {"name":"b", "staticSize":4, "exclusiveSize":4, "typeNames": ["int32_t"]}, + {"name":"c", "staticSize":4, "exclusiveSize":4, "typeNames": ["int32_t"]}, + {"name":"d", "staticSize":4, "exclusiveSize":4, "typeNames": ["int32_t"]}, + {"name":"e", "staticSize":4, "exclusiveSize":4, "typeNames": ["int32_t"]}, + {"name":"f", "staticSize":4, "exclusiveSize":4, "typeNames": ["int32_t"]} + ]}]''' diff --git a/test/integration/inheritance_polymorphic_non_dynamic_base.toml b/test/integration/inheritance_polymorphic_non_dynamic_base.toml index 33b8427..3fdf9ee 100644 --- a/test/integration/inheritance_polymorphic_non_dynamic_base.toml +++ b/test/integration/inheritance_polymorphic_non_dynamic_base.toml @@ -82,6 +82,7 @@ definitions = ''' {"name":"vec_b", "staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4} ]}]''' [cases.b_no_polymorphic] + oil_skip = "vptr included in output when it should probably be hidden" # https://github.com/facebookexperimental/object-introspection/issues/291 param_types = ["const B&"] arg_types = ["B"] setup = ''' @@ -157,6 +158,7 @@ definitions = ''' {"name":"int_c", "staticSize":4, "dynamicSize":0} ]}]''' [cases.c_no_polymorphic] + oil_skip = 'vptrs are included int he json output' # https://github.com/facebookexperimental/object-introspection/issues/291 param_types = ["const C&"] arg_types = ["C"] setup = ''' @@ -174,3 +176,14 @@ definitions = ''' {"name":"vec_b", "staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}, {"name":"int_c", "staticSize":4, "dynamicSize":0} ]}]''' + expect_json_v2 = '''[{ + "typeNames": ["C"], + "staticSize": 48, + "exclusiveSize": 8, + "members": [ + {"name":"int_a", "staticSize":4, "exclusiveSize":4}, + {"staticSize":8, "exclusiveSize":8}, + {"name":"vec_b", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3}, + {"name":"int_c", "staticSize":4, "exclusiveSize":4} + ] + }]''' diff --git a/test/integration/multi_arg.toml b/test/integration/multi_arg.toml index 37e5786..571c30c 100644 --- a/test/integration/multi_arg.toml +++ b/test/integration/multi_arg.toml @@ -12,6 +12,7 @@ definitions = ''' [cases] [cases.a] + oil_disable = 'multi-argument probing has no meaning for oil' param_types = ["int", "double"] args = "arg0,arg1" setup = "return {1,2.0};" diff --git a/test/integration/namespaces.toml b/test/integration/namespaces.toml index 6100457..c38cd6b 100644 --- a/test/integration/namespaces.toml +++ b/test/integration/namespaces.toml @@ -16,6 +16,7 @@ definitions = ''' ''' [cases] [cases.queue] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/308 param_types = ["const std::queue>&"] setup = "return std::queue>({{ns_namespaces::nsA::Foo(), ns_namespaces::nsB::Foo()}});" expect_json = '''[{ @@ -23,6 +24,7 @@ definitions = ''' "staticSize": 80, "dynamicSize": 12, "length": 1, "capacity": 1, "elementStaticSize": 12 }]''' [cases.stack] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/305 param_types = ["const std::stack>&"] setup = "return std::stack>({{ns_namespaces::nsA::Foo(), ns_namespaces::nsB::Foo()}});" expect_json = '''[{ diff --git a/test/integration/padding.toml b/test/integration/padding.toml index 13000fe..6cc17d1 100644 --- a/test/integration/padding.toml +++ b/test/integration/padding.toml @@ -93,6 +93,7 @@ definitions = ''' ]}]''' [cases.parent_padding] + oid_skip = 'calculating incorrectly' param_types = ["const PaddedChild&"] setup = "return PaddedChild{};" expect_json = '''[{ @@ -100,3 +101,7 @@ definitions = ''' "dynamicSize": 0, "paddingSavingsSize": 19 }]''' + expect_json_v2 = '''[{ + "staticSize": 104, + "exclusiveSize": 32 + }]''' diff --git a/test/integration/pointers.toml b/test/integration/pointers.toml index 9f3d844..389ba29 100644 --- a/test/integration/pointers.toml +++ b/test/integration/pointers.toml @@ -14,8 +14,7 @@ definitions = ''' [cases] [cases.int] - oid_skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19 - oil_disable = "oil can't chase raw pointers safely" + skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19 param_types = ["int*"] setup = "return new int(1);" cli_options = ["-fchase-raw-pointers"] @@ -33,7 +32,7 @@ definitions = ''' ] }]''' [cases.int_no_follow] - oid_skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19 + skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19 param_types = ["int*"] setup = "return new int(1);" expect_json = '''[{ @@ -44,7 +43,7 @@ definitions = ''' "NOT": "members" }]''' [cases.int_null] - oid_skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19 + skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19 param_types = ["int*"] setup = "return nullptr;" expect_json = '''[{ @@ -57,8 +56,7 @@ definitions = ''' [cases.void] - oid_skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19 - oil_disable = "oil can't chase raw pointers safely" + skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19 param_types = ["void*"] setup = "return new int(1);" cli_options = ["-fchase-raw-pointers"] @@ -70,7 +68,7 @@ definitions = ''' "NOT": "members" }]''' [cases.void_no_follow] - oid_skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19 + skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19 param_types = ["void*"] setup = "return new int(1);" expect_json = '''[{ @@ -81,7 +79,7 @@ definitions = ''' "NOT": "members" }]''' [cases.void_null] - oid_skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19 + skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19 param_types = ["void*"] setup = "return nullptr;" expect_json = '''[{ @@ -94,8 +92,7 @@ definitions = ''' [cases.vector] - oid_skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19 - oil_disable = "oil can't chase raw pointers safely" + skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19 param_types = ["std::vector*"] setup = "return new std::vector{1,2,3};" cli_options = ["-fchase-raw-pointers"] @@ -113,7 +110,7 @@ definitions = ''' ] }]''' [cases.vector_no_follow] - oid_skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19 + skip = "top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19 param_types = ["std::vector*"] setup = "return new std::vector{1,2,3};" expect_json = '''[{ @@ -124,7 +121,7 @@ definitions = ''' "NOT": "members" }]''' [cases.vector_null] - oid_skip = "BAD DATA SEGMENT!!! top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19 + skip = "BAD DATA SEGMENT!!! top-level pointers are skipped over" # https://github.com/facebookexperimental/object-introspection/issues/19 param_types = ["std::vector*"] setup = "return nullptr;" expect_json = '''[{ @@ -224,7 +221,7 @@ definitions = ''' {"staticSize":8, "dynamicSize":4, "NOT": {"pointer":0}} ]}]''' [cases.vector_of_pointers_no_follow] - oid_skip = "pointer field is missing from results" # https://github.com/facebookexperimental/object-introspection/issues/21 + skip = "pointer field is missing from results" # https://github.com/facebookexperimental/object-introspection/issues/21 param_types = ["const std::vector&"] setup = "return {{new int(1), nullptr, new int(3)}};" expect_json = '''[{ diff --git a/test/integration/pointers_function.toml b/test/integration/pointers_function.toml index fd5a9e4..a62ef9a 100644 --- a/test/integration/pointers_function.toml +++ b/test/integration/pointers_function.toml @@ -14,7 +14,7 @@ definitions = ''' [cases] [cases.raw] - oid_skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22 + skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22 param_types = ["const FuncPtrStruct&"] setup = "return {{myFunction}};" expect_json = '''[{ @@ -46,7 +46,7 @@ definitions = ''' }] }]''' [cases.raw_null] - oid_skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22 + skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22 param_types = ["const FuncPtrStruct&"] setup = "return {{nullptr}};" expect_json = '''[{ @@ -62,7 +62,7 @@ definitions = ''' }]''' [cases.std_function] - oid_skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22 + skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22 param_types = ["std::function &"] setup = "return myFunction;" expect_json = '''[{ @@ -86,7 +86,7 @@ definitions = ''' "NOT": "members" }]''' [cases.std_function_null] - oid_skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22 + skip = "function pointers are not handled correctly" # https://github.com/facebookexperimental/object-introspection/issues/22 param_types = ["std::function &"] setup = "return nullptr;" expect_json = '''[{ diff --git a/test/integration/pointers_incomplete.toml b/test/integration/pointers_incomplete.toml index 8bc6b07..437041c 100644 --- a/test/integration/pointers_incomplete.toml +++ b/test/integration/pointers_incomplete.toml @@ -32,7 +32,7 @@ definitions = ''' "NOT": "members" }]''' [cases.raw_no_follow] - oid_skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17 + skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17 param_types = ["IncompleteType*"] setup = "return static_cast(::operator new(5));" expect_json = '''[{ @@ -43,7 +43,7 @@ definitions = ''' "NOT": "members" }]''' [cases.raw_null] - oid_skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17 + skip = "oid codegen fails on this" # https://github.com/facebookexperimental/object-introspection/issues/17 param_types = ["IncompleteType*"] setup = "return nullptr;" expect_json = '''[{ @@ -55,6 +55,7 @@ definitions = ''' }]''' [cases.unique_ptr] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299 param_types = ["const std::unique_ptr&"] setup = ''' auto raw_ptr = static_cast(::operator new(5)); @@ -63,6 +64,7 @@ definitions = ''' ''' expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]' [cases.unique_ptr_null] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/299 param_types = ["const std::unique_ptr&"] setup = ''' return std::unique_ptr( @@ -71,6 +73,7 @@ definitions = ''' expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]' [cases.shared_ptr] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300 param_types = ["const std::shared_ptr&"] setup = ''' auto raw_ptr = static_cast(::operator new(5)); @@ -78,6 +81,7 @@ definitions = ''' ''' expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]' [cases.shared_ptr_null] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/300 param_types = ["const std::shared_ptr"] setup = "return nullptr;" expect_json = '[{"staticSize":16, "dynamicSize":0, "NOT":"members"}]' diff --git a/test/integration/runner_common.cpp b/test/integration/runner_common.cpp index 2ad7aa7..cecf6fd 100644 --- a/test/integration/runner_common.cpp +++ b/test/integration/runner_common.cpp @@ -302,10 +302,10 @@ OidProc OidIntegration::runOidOnProcess(OidOpts opts, }; } -void OidIntegration::compare_json(const bpt::ptree& expected_json, - const bpt::ptree& actual_json, - const std::string& full_key, - bool expect_eq) { +void IntegrationBase::compare_json(const bpt::ptree& expected_json, + const bpt::ptree& actual_json, + const std::string& full_key, + bool expect_eq) { if (expected_json.empty()) { if (expect_eq) { ASSERT_EQ(expected_json.data(), actual_json.data()) @@ -357,6 +357,21 @@ void OidIntegration::compare_json(const bpt::ptree& expected_json, auto actual_it = actual_json.find(key); auto curr_key = full_key + "." + key; if (actual_it == actual_json.not_found()) { + // TODO: Remove these with the switch to treebuilderv2. This is a hack to + // make some old JSON output compatible with new JSON output. + if (key == "typeName") { + auto type_names = actual_json.find("typeNames"); + if (type_names != actual_json.not_found()) { + if (auto name = type_names->second.rbegin(); + name != type_names->second.rend()) { + compare_json(val, name->second, curr_key, expect_eq); + continue; + } + } + } else if (key == "dynamicSize" && val.get_value() == 0) { + continue; + } + ADD_FAILURE() << "Expected key not found in output: " << curr_key; continue; } diff --git a/test/integration/runner_common.h b/test/integration/runner_common.h index 8cec14b..f0cf4db 100644 --- a/test/integration/runner_common.h +++ b/test/integration/runner_common.h @@ -49,16 +49,6 @@ class IntegrationBase : public ::testing::Test { std::string stdout_; std::string stderr_; -}; - -class OidIntegration : public IntegrationBase { - protected: - std::string TmpDirStr() override; - - OidProc runOidOnProcess(OidOpts opts, - std::vector extra_args, - std::string configPrefix, - std::string configSuffix); /* * compare_json @@ -72,6 +62,16 @@ class OidIntegration : public IntegrationBase { bool expect_eq = true); }; +class OidIntegration : public IntegrationBase { + protected: + std::string TmpDirStr() override; + + OidProc runOidOnProcess(OidOpts opts, + std::vector extra_args, + std::string configPrefix, + std::string configSuffix); +}; + class OilIntegration : public IntegrationBase { protected: std::string TmpDirStr() override; diff --git a/test/integration/simple.toml b/test/integration/simple.toml index 2fe4405..71a215a 100644 --- a/test/integration/simple.toml +++ b/test/integration/simple.toml @@ -44,6 +44,5 @@ definitions = ''' setup = "return {};" expect_json = '''[{ "staticSize":8, - "dynamicSize":0, - "NOT":"members" + "dynamicSize":0 }]''' diff --git a/test/integration/sorted_vector_set.toml b/test/integration/sorted_vector_set.toml index 7a871f1..f8e15c0 100644 --- a/test/integration/sorted_vector_set.toml +++ b/test/integration/sorted_vector_set.toml @@ -5,6 +5,7 @@ definitions = ''' [cases] [cases.no_ints] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/320 param_types = ["const sorted_vector_set&"] setup = "return {};" expect_json = '''[{ @@ -22,6 +23,7 @@ definitions = ''' }]}]''' [cases.some_ints] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/320 param_types = ["const sorted_vector_set&"] setup = ''' sorted_vector_set is; diff --git a/test/integration/std_array.toml b/test/integration/std_array.toml index 5e6ae15..2a8cd4e 100644 --- a/test/integration/std_array.toml +++ b/test/integration/std_array.toml @@ -1,6 +1,7 @@ includes = ["array", "cstdint", "vector"] [cases] [cases.uint64_length_0] + oil_skip = 'needs updating for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/318 param_types = ["std::array&"] setup = "return {{}};" expect_json = ''' @@ -16,6 +17,7 @@ includes = ["array", "cstdint", "vector"] ] ''' [cases.uint64_length_1] + oil_skip = 'needs updating for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/318 param_types = ["std::array&"] setup = "return {{1}};" expect_json = ''' @@ -31,6 +33,7 @@ includes = ["array", "cstdint", "vector"] ] ''' [cases.uint64_length_8] + oil_skip = 'needs updating for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/318 param_types = ["std::array&"] setup = "return {{0,1,2,3,4,5,6,7}};" expect_json = ''' @@ -45,6 +48,7 @@ includes = ["array", "cstdint", "vector"] ] ''' [cases.vector_length_1] + oil_skip = 'needs updating for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/318 param_types = ["std::array, 1>&"] setup = "return {{std::initializer_list({1,2,3,4,5})}};" expect_json = ''' @@ -65,6 +69,7 @@ includes = ["array", "cstdint", "vector"] ] ''' [cases.vector_length_2] + oil_skip = 'needs updating for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/318 param_types = ["std::array, 2>&"] setup = "return {{std::initializer_list({1,2,3,4,5}), std::initializer_list({6,7,8,9})}};" expect_json = ''' diff --git a/test/integration/std_conditional.toml b/test/integration/std_conditional.toml index 00a9135..16b2f0f 100644 --- a/test/integration/std_conditional.toml +++ b/test/integration/std_conditional.toml @@ -16,6 +16,7 @@ public: ''' [cases] [cases.a] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/317 param_types = ["const C&"] setup = ''' C foo; diff --git a/test/integration/std_deque.toml b/test/integration/std_deque.toml index ec24b26..bf774ef 100644 --- a/test/integration/std_deque.toml +++ b/test/integration/std_deque.toml @@ -3,18 +3,22 @@ includes = ["deque"] [cases] [cases.int_empty] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/316 param_types = ["const std::deque&"] setup = "return {};" expect_json = '[{"staticSize":80, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]' [cases.int_some] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/316 param_types = ["const std::deque&"] setup = "return {{1,2,3}};" expect_json = '[{"staticSize":80, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}]' [cases.deque_int_empty] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/316 param_types = ["const std::deque>&"] setup = "return {};" expect_json = '[{"staticSize":80, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":80}]' [cases.deque_int_some] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/316 param_types = ["const std::deque>&"] setup = "return {{{1,2,3},{},{4,5}}};" expect_json = '''[{ diff --git a/test/integration/std_deque_del_allocator.toml b/test/integration/std_deque_del_allocator.toml index 96cda64..c3bae58 100644 --- a/test/integration/std_deque_del_allocator.toml +++ b/test/integration/std_deque_del_allocator.toml @@ -24,6 +24,7 @@ includes = ["deque"] [cases] [cases.a] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/316 param_types = ["const Foo&"] setup = ''' Foo foo; diff --git a/test/integration/std_list_del_allocator.toml b/test/integration/std_list_del_allocator.toml index f15678f..a85668d 100644 --- a/test/integration/std_list_del_allocator.toml +++ b/test/integration/std_list_del_allocator.toml @@ -27,6 +27,7 @@ includes = ["list"] [cases] [cases.a] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/315 param_types = ["const Foo&"] setup = ''' Foo foo; diff --git a/test/integration/std_map_custom_comparator.toml b/test/integration/std_map_custom_comparator.toml index 4cca520..79c1bff 100644 --- a/test/integration/std_map_custom_comparator.toml +++ b/test/integration/std_map_custom_comparator.toml @@ -36,6 +36,7 @@ includes = ["map", "functional"] [cases] [cases.a] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/314 param_types = ["const Foo&"] setup = ''' Foo foo; diff --git a/test/integration/std_multimap_custom_comparator.toml b/test/integration/std_multimap_custom_comparator.toml index 4d781cf..aedd49e 100644 --- a/test/integration/std_multimap_custom_comparator.toml +++ b/test/integration/std_multimap_custom_comparator.toml @@ -14,6 +14,7 @@ includes = ["map"] [cases] [cases.a] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/313 param_types = ["const Foo&"] setup = ''' Foo foo; diff --git a/test/integration/std_optional.toml b/test/integration/std_optional.toml index f88063d..67579e9 100644 --- a/test/integration/std_optional.toml +++ b/test/integration/std_optional.toml @@ -1,6 +1,7 @@ includes = ["optional", "cstdint", "vector"] [cases] [cases.uint64_empty] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/312 param_types = ["std::optional&"] setup = "return std::nullopt;" expect_json = ''' @@ -16,6 +17,7 @@ includes = ["optional", "cstdint", "vector"] ] ''' [cases.uint64_present] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/312 param_types = ["std::optional&"] setup = "return 64;" expect_json = ''' @@ -31,6 +33,7 @@ includes = ["optional", "cstdint", "vector"] ] ''' [cases.vector_empty] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/312 param_types = ["std::optional>&"] setup = "return std::nullopt;" expect_json = ''' @@ -46,6 +49,7 @@ includes = ["optional", "cstdint", "vector"] ] ''' [cases.vector_present] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/312 param_types = ["std::optional>&"] setup = "return {{{1,2,3,4,5}}};" expect_json = ''' diff --git a/test/integration/std_pair.toml b/test/integration/std_pair.toml index a509ecc..3006579 100644 --- a/test/integration/std_pair.toml +++ b/test/integration/std_pair.toml @@ -1,6 +1,7 @@ includes = ["vector", "utility", "cstdint"] [cases] [cases.uint64_uint64] + oil_skip = 'tests need updating for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/310 param_types = ["std::pair&"] setup = "return {{0, 1}};" expect_json = ''' @@ -14,6 +15,7 @@ includes = ["vector", "utility", "cstdint"] ] ''' [cases.uint64_uint32] + oil_skip = 'tests need updating for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/310 param_types = ["std::pair&"] setup = "return {{0, 1}};" # Should still have static size of 16 due to padding @@ -28,6 +30,7 @@ includes = ["vector", "utility", "cstdint"] ] ''' [cases.vector_vector] + oil_skip = 'tests need updating for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/310 param_types = ["std::pair, std::vector>&"] setup = "return {{std::initializer_list({0,1,2}), std::initializer_list({3,4,5,6})}};" expect_json = ''' diff --git a/test/integration/std_priority_queue.toml b/test/integration/std_priority_queue.toml index b50de9b..b4930ff 100644 --- a/test/integration/std_priority_queue.toml +++ b/test/integration/std_priority_queue.toml @@ -1,6 +1,7 @@ includes = ["queue"] [cases] [cases.int_empty] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/309 param_types = ["const std::priority_queue&"] setup = "return {};" expect_json = '''[{ @@ -13,6 +14,7 @@ includes = ["queue"] } ]}]''' [cases.int_some] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/309 param_types = ["const std::priority_queue&"] setup = "return std::priority_queue({}, {3,2,1});" expect_json = '''[{ @@ -25,6 +27,7 @@ includes = ["queue"] } ]}]''' [cases.adapter_deque_empty] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/309 param_types = ["const std::priority_queue>&"] setup = "return {};" expect_json = '''[{ @@ -37,6 +40,7 @@ includes = ["queue"] } ]}]''' [cases.adapter_deque_some] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/309 param_types = ["const std::priority_queue>&"] setup = "return std::priority_queue>({}, {3,2,1});" expect_json = '''[{ diff --git a/test/integration/std_queue.toml b/test/integration/std_queue.toml index 362d0e6..7e804c1 100644 --- a/test/integration/std_queue.toml +++ b/test/integration/std_queue.toml @@ -1,6 +1,7 @@ includes = ["queue"] [cases] [cases.int_empty] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/308 param_types = ["const std::queue&"] setup = "return {};" expect_json = '''[{ @@ -13,6 +14,7 @@ includes = ["queue"] } ]}]''' [cases.int_some] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/308 param_types = ["const std::queue&"] setup = "return std::queue({1,2,3});" expect_json = '''[{ @@ -25,6 +27,7 @@ includes = ["queue"] } ]}]''' [cases.queue_int_empty] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/308 param_types = ["const std::queue>&"] setup = "return {};" expect_json = '''[{ @@ -35,6 +38,7 @@ includes = ["queue"] } ]}]''' [cases.queue_int_some] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/308 param_types = ["const std::queue>&"] setup = ''' return std::queue>({ @@ -83,6 +87,7 @@ includes = ["queue"] } ]}]''' [cases.adapter_vector_empty] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/308 param_types = ["const std::queue>&"] setup = "return {};" expect_json = '''[{ @@ -95,6 +100,7 @@ includes = ["queue"] } ]}]''' [cases.adapter_vector_some] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/308 param_types = ["const std::queue>&"] setup = "return std::queue>({1,2,3});" expect_json = '''[{ diff --git a/test/integration/std_reference_wrapper.toml b/test/integration/std_reference_wrapper.toml index 601e3c8..bb3dd52 100644 --- a/test/integration/std_reference_wrapper.toml +++ b/test/integration/std_reference_wrapper.toml @@ -1,10 +1,12 @@ includes = ["functional"] [cases] [cases.int] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/307 param_types = ["const std::reference_wrapper&"] setup = "return std::ref(*new int(1));" expect_json = '[{"staticSize":8, "dynamicSize":4, "length":1, "capacity":1, "elementStaticSize":4}]' [cases.vector] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/307 param_types = ["const std::vector>&"] setup = "return {{std::ref(*new int(1)), std::ref(*new int(2)), std::ref(*new int(3))}};" expect_json = '''[{ diff --git a/test/integration/std_set_custom_comparator.toml b/test/integration/std_set_custom_comparator.toml index 60b4c19..d1c187f 100644 --- a/test/integration/std_set_custom_comparator.toml +++ b/test/integration/std_set_custom_comparator.toml @@ -31,6 +31,7 @@ includes = ["set", "functional"] [cases] [cases.a] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/306 param_types = ["const Foo&"] setup = ''' Foo foo; diff --git a/test/integration/std_smart_ptr.toml b/test/integration/std_smart_ptr.toml index 3a19758..2d9a463 100644 --- a/test/integration/std_smart_ptr.toml +++ b/test/integration/std_smart_ptr.toml @@ -7,6 +7,7 @@ definitions = ''' ''' [cases] [cases.unique_ptr_uint64_empty] + oil_skip = "std::unique_ptr is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/299 param_types = ["std::unique_ptr&"] setup = "return {nullptr};" expect_json = ''' @@ -22,6 +23,7 @@ definitions = ''' ] ''' [cases.unique_ptr_uint64_present] + oil_skip = "std::unique_ptr is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/299 param_types = ["std::unique_ptr&"] setup = "return {std::make_unique(64)};" expect_json = ''' @@ -36,6 +38,7 @@ definitions = ''' ] ''' [cases.unique_ptr_vector_empty] + oil_skip = "std::unique_ptr is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/299 param_types = ["std::unique_ptr>&"] setup = "return {nullptr};" expect_json = ''' @@ -50,6 +53,7 @@ definitions = ''' ] ''' [cases.unique_ptr_vector_present] + oil_skip = "std::unique_ptr is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/299 param_types = ["std::unique_ptr>&"] setup = "return {std::make_unique>(std::initializer_list({1,2,3,4,5}))};" expect_json = ''' @@ -72,6 +76,7 @@ definitions = ''' ] ''' [cases.unique_ptr_void_empty] + oil_skip = "std::unique_ptr is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/299 param_types = ["std::unique_ptr&"] setup = "return {std::unique_ptr(nullptr, &void_int_deleter)};" expect_json = ''' @@ -83,6 +88,7 @@ definitions = ''' ] ''' [cases.unique_ptr_void_present] + oil_skip = "std::unique_ptr is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/299 param_types = ["std::unique_ptr&"] setup = "return {std::unique_ptr(new int, &void_int_deleter)};" expect_json = ''' @@ -94,6 +100,7 @@ definitions = ''' ] ''' [cases.shared_ptr_uint64_empty] + oil_skip = "std::shared_ptr is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/300 param_types = ["std::shared_ptr&"] setup = "return {nullptr};" expect_json = ''' @@ -108,6 +115,7 @@ definitions = ''' ] ''' [cases.shared_ptr_uint64_present] + oil_skip = "std::shared_ptr is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/300 param_types = ["std::shared_ptr&"] setup = "return std::make_shared(64);" expect_json = ''' @@ -122,6 +130,7 @@ definitions = ''' ] ''' [cases.shared_ptr_vector_empty] + oil_skip = "std::shared_ptr is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/300 param_types = ["std::shared_ptr>&"] setup = "return {nullptr};" expect_json = ''' @@ -136,6 +145,7 @@ definitions = ''' ] ''' [cases.shared_ptr_vector_present] + oil_skip = "std::shared_ptr is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/300 param_types = ["std::shared_ptr>&"] setup = "return std::make_shared>(std::initializer_list({1,2,3,4,5}));" expect_json = ''' @@ -156,6 +166,7 @@ definitions = ''' ] ''' [cases.shared_ptr_void_empty] + oil_skip = "std::shared_ptr is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/300 param_types = ["std::shared_ptr&"] setup = "return {nullptr};" expect_json = ''' @@ -168,6 +179,7 @@ definitions = ''' ] ''' [cases.shared_ptr_void_present] + oil_skip = "std::shared_ptr is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/300 param_types = ["std::shared_ptr&"] setup = "return {std::shared_ptr(new int)};" expect_json = ''' @@ -192,6 +204,7 @@ definitions = ''' } ] ''' + expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]''' [cases.weak_ptr_int64_void_empty] param_types = ["std::weak_ptr&"] setup = "return std::weak_ptr();" @@ -205,6 +218,7 @@ definitions = ''' } ] ''' + expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]''' [cases.weak_ptr_int64_present] param_types = ["std::weak_ptr&"] setup = ''' @@ -222,6 +236,7 @@ definitions = ''' } ] ''' + expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]''' [cases.weak_ptr_int64_expired] param_types = ["std::weak_ptr&"] setup = ''' @@ -239,6 +254,7 @@ definitions = ''' } ] ''' + expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]''' [cases.weak_ptr_int64_present_chase] param_types = ["std::weak_ptr&"] cli_options = ["-fchase-raw-pointers"] @@ -257,6 +273,7 @@ definitions = ''' } ] ''' + expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]''' [cases.weak_ptr_int64_expired_chase] param_types = ["std::weak_ptr&"] cli_options = ["-fchase-raw-pointers"] @@ -273,3 +290,4 @@ definitions = ''' } ] ''' + expect_json_v2 = '''[{ "staticSize": 16, "exclusiveSize": 16, "members":[]}]''' diff --git a/test/integration/std_stack.toml b/test/integration/std_stack.toml index 8b01b74..3294fdf 100644 --- a/test/integration/std_stack.toml +++ b/test/integration/std_stack.toml @@ -1,6 +1,7 @@ includes = ["stack", "vector"] [cases] [cases.int_empty] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/305 param_types = ["const std::stack&"] setup = "return {};" expect_json = '''[{ @@ -13,6 +14,7 @@ includes = ["stack", "vector"] } ]}]''' [cases.int_some] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/305 param_types = ["const std::stack&"] setup = "return std::stack({1,2,3});" expect_json = '''[{ @@ -25,6 +27,7 @@ includes = ["stack", "vector"] } ]}]''' [cases.stack_int_empty] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/305 param_types = ["const std::stack>&"] setup = "return {};" expect_json = '''[{ @@ -35,6 +38,7 @@ includes = ["stack", "vector"] } ]}]''' [cases.stack_int_some] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/305 param_types = ["const std::stack>&"] setup = ''' return std::stack>({ @@ -83,6 +87,7 @@ includes = ["stack", "vector"] } ]}]''' [cases.adapter_vector_empty] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/305 param_types = ["const std::stack>&"] setup = "return {};" expect_json = '''[{ @@ -95,6 +100,7 @@ includes = ["stack", "vector"] } ]}]''' [cases.adapter_vector_some] + oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/305 param_types = ["const std::stack>&"] setup = "return std::stack>({1,2,3});" expect_json = '''[{ diff --git a/test/integration/std_string.toml b/test/integration/std_string.toml index 6e6f525..25f2424 100644 --- a/test/integration/std_string.toml +++ b/test/integration/std_string.toml @@ -1,6 +1,7 @@ includes = ["string"] [cases] [cases.empty] + oil_skip = 'needs updating for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/311 param_types = ["std::string&"] setup = "return {};" expect_json = ''' @@ -25,6 +26,7 @@ includes = ["string"] ] ''' [cases.sso] + oil_skip = 'needs updating for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/311 param_types = ["std::string&"] setup = 'return {"012345"};' expect_json = ''' @@ -49,6 +51,7 @@ includes = ["string"] ] ''' [cases.heap_allocated] + oil_skip = 'needs updating for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/311 param_types = ["std::string&"] setup = 'return {"abcdefghijklmnopqrstuvwxzy"};' expect_json = ''' diff --git a/test/integration/std_tuple.toml b/test/integration/std_tuple.toml index 21c43cb..4d63064 100644 --- a/test/integration/std_tuple.toml +++ b/test/integration/std_tuple.toml @@ -9,6 +9,7 @@ struct Bar { ''' [cases] [cases.uint64_uint64] + oil_skip = "std::tuple is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/304 param_types = ["Bar&"] setup = ''' Foo f; diff --git a/test/integration/std_unordered_map_custom_operator.toml b/test/integration/std_unordered_map_custom_operator.toml index 44c7934..a4cc7a4 100644 --- a/test/integration/std_unordered_map_custom_operator.toml +++ b/test/integration/std_unordered_map_custom_operator.toml @@ -33,6 +33,7 @@ includes = ["unordered_map"] [cases] [cases.a] + oil_skip = "std::unordered_map is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/303 param_types = ["const Foo&"] setup = ''' Foo foo; diff --git a/test/integration/std_unordered_set_custom_operator.toml b/test/integration/std_unordered_set_custom_operator.toml index b9e3bd9..9df345a 100644 --- a/test/integration/std_unordered_set_custom_operator.toml +++ b/test/integration/std_unordered_set_custom_operator.toml @@ -33,6 +33,7 @@ includes = ["unordered_set"] [cases] [cases.a] + oil_skip = "std::unordered_set is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/302 param_types = ["const Foo&"] setup = ''' Foo foo; diff --git a/test/integration/std_variant.toml b/test/integration/std_variant.toml index 038693b..d361b06 100644 --- a/test/integration/std_variant.toml +++ b/test/integration/std_variant.toml @@ -9,6 +9,7 @@ definitions = ''' [cases] [cases.char_int64_1] + oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298 param_types = ["const std::variant&"] setup = "return 'a';" expect_json = '''[{ @@ -22,6 +23,7 @@ definitions = ''' {"typeName":"char", "staticSize":1, "exclusiveSize":1, "dynamicSize":0} ]}]''' [cases.char_int64_2] + oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298 param_types = ["const std::variant&"] setup = "return 1234;" expect_json = '''[{ @@ -36,6 +38,7 @@ definitions = ''' ]}]''' [cases.vector_int_1] + oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298 param_types = ["const std::variant, int>&"] setup = "return std::vector{1,2,3};" expect_json = '''[{ @@ -57,6 +60,7 @@ definitions = ''' } ]}]''' [cases.vector_int_2] + oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298 param_types = ["const std::variant, int>&"] setup = "return 123;" expect_json = '''[{ @@ -74,6 +78,7 @@ definitions = ''' ]}]''' [cases.optional] + oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298 # This test case ensures that the alignment of std::variant is set # correctly, as otherwise the size of the std::optional would be wrong param_types = ["const std::optional>&"] @@ -100,6 +105,7 @@ definitions = ''' ]}]''' [cases.empty] + oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298 # https://en.cppreference.com/w/cpp/utility/variant/valueless_by_exception param_types = ["const std::variant&"] setup = ''' @@ -127,6 +133,7 @@ definitions = ''' # 0xff can be a valid index if there are at least 256 parameters, and that # the invalid index value is raised to 0xffff. [cases.256_params_256] + oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298 param_types = ["const std::variant&"] setup = "return 'a';" expect_json = '''[{ @@ -140,6 +147,7 @@ definitions = ''' {"typeName":"char", "staticSize":1, "exclusiveSize":1, "dynamicSize":0} ]}]''' [cases.256_params_empty] + oil_skip = "std::variant is not implemented for treebuilder v2" # https://github.com/facebookexperimental/object-introspection/issues/298 param_types = ["const std::variant&"] setup = ''' std::variant var{'a'}; diff --git a/test/integration/std_vector.toml b/test/integration/std_vector.toml index b23a464..473e8a4 100644 --- a/test/integration/std_vector.toml +++ b/test/integration/std_vector.toml @@ -4,10 +4,16 @@ includes = ["vector"] param_types = ["const std::vector&"] setup = "return {};" expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":4}]' + expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]' [cases.int_some] param_types = ["const std::vector&"] setup = "return {{1,2,3}};" expect_json = '[{"staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4}]' + expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity":3, "members":[ + {"staticSize":4, "exclusiveSize":4}, + {"staticSize":4, "exclusiveSize":4}, + {"staticSize":4, "exclusiveSize":4} + ]}]''' [cases.bool_empty] skip = true # https://github.com/facebookexperimental/object-introspection/issues/14 param_types = ["const std::vector&"] @@ -22,6 +28,7 @@ includes = ["vector"] param_types = ["const std::vector>&"] setup = "return {};" expect_json = '[{"staticSize":24, "dynamicSize":0, "length":0, "capacity":0, "elementStaticSize":24}]' + expect_json_v2 = '[{"staticSize":24, "exclusiveSize":24, "length":0, "capacity":0, "members":[]}]' [cases.vector_int_some] param_types = ["const std::vector>&"] setup = "return {{{1,2,3},{4},{5,6}}};" @@ -37,6 +44,11 @@ includes = ["vector"] {"staticSize":24, "dynamicSize":4, "exclusiveSize":28, "length":1, "capacity":1, "elementStaticSize":4}, {"staticSize":24, "dynamicSize":8, "exclusiveSize":32, "length":2, "capacity":2, "elementStaticSize":4} ]}]''' + expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[ + {"staticSize":24, "exclusiveSize":24, "length":3, "capacity": 3, "members":[]}, + {"staticSize":24, "exclusiveSize":24, "length":1, "capacity": 1, "members":[]}, + {"staticSize":24, "exclusiveSize":24, "length":2, "capacity": 2, "members":[]} + ]}]''' [cases.reserve] param_types = ["const std::vector&"] setup = ''' @@ -45,3 +57,8 @@ includes = ["vector"] return ret; ''' expect_json = '[{"staticSize":24, "dynamicSize":40, "exclusiveSize":64, "length":3, "capacity":10, "elementStaticSize":4}]' + expect_json_v2 = '''[{"staticSize":24, "exclusiveSize":52, "length":3, "capacity":10, "members":[ + {"staticSize":4, "exclusiveSize":4}, + {"staticSize":4, "exclusiveSize":4}, + {"staticSize":4, "exclusiveSize":4} + ]}]''' diff --git a/test/integration/std_vector_del_allocator.toml b/test/integration/std_vector_del_allocator.toml index 475074c..335f010 100644 --- a/test/integration/std_vector_del_allocator.toml +++ b/test/integration/std_vector_del_allocator.toml @@ -27,6 +27,7 @@ includes = ["vector"] [cases] [cases.a] + oil_skip = "oil gets the exclusive size of vector subfields wrong" # https://github.com/facebookexperimental/object-introspection/issues/301 param_types = ["const Foo&"] setup = ''' Foo foo; @@ -57,3 +58,22 @@ includes = ["vector"] ]} ]} ]}]''' + expect_json_v2 = '''[{ + "staticSize":48, + "exclusiveSize":0, + "members":[ + {"name":"v1", "staticSize":24, "exclusiveSize":24, "length":1, "capacity":1, "members":[ + {"name":"[]", "staticSize":4, "exclusiveSize":0, "members":[ + {"name":"a", "staticSize":4, "exclusiveSize":4, "members":[]} + ]} + ]}, + {"name":"v2", "staticSize":24, "exclusiveSize":24, "length":2, "capacity":2, "members":[ + {"name":"[]", "staticSize":4, "exclusiveSize":0, "members":[ + {"name":"b", "staticSize":4, "exclusiveSize":4, "members":[]} + ]}, + {"name":"[]", "staticSize":4, "exclusiveSize":0, "members":[ + {"name":"b", "staticSize":4, "exclusiveSize":4, "members":[]} + ]} + ]} + ] + }]''' diff --git a/test/integration/templates.toml b/test/integration/templates.toml index 3552859..a8e7b05 100644 --- a/test/integration/templates.toml +++ b/test/integration/templates.toml @@ -24,6 +24,7 @@ definitions = ''' [cases] [cases.int] + oil_skip = "primitives are named differently in treebuilderv2" # https://github.com/facebookexperimental/object-introspection/issues/286 param_types = ["const TemplatedClass1&"] setup = "return {};" expect_json = '''[{ @@ -48,7 +49,19 @@ definitions = ''' "capacity":0, "elementStaticSize":4 }]}]''' + expect_json_v2 = '''[{ + "typeName":"TemplatedClass1 > >", + "staticSize":24, + "exclusiveSize":0, + "members":[{ + "typeName":"std::vector>", + "staticSize":24, + "exclusiveSize":24, + "length":0, + "capacity":0 + }]}]''' [cases.two] + oil_skip = "OIL returns better primitive names" param_types = ["const TemplatedClass2&"] setup = "return {};" expect_json = '''[{ @@ -73,3 +86,13 @@ definitions = ''' "capacity":3, "elementStaticSize":4 }]}]''' + expect_json_v2 = '''[{ + "typeName":"TemplatedClassVal<3>", + "staticSize":12, + "exclusiveSize":0, + "members":[{ + "staticSize":12, + "exclusiveSize":0, + "length":3, + "capacity":3 + }]}]''' diff --git a/test/integration/thrift_isset.toml b/test/integration/thrift_isset.toml index 497a6d1..e32f416 100644 --- a/test/integration/thrift_isset.toml +++ b/test/integration/thrift_isset.toml @@ -77,6 +77,7 @@ namespace cpp2 { [cases] [cases.unpacked] + oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296 param_types = ["const cpp2::MyThriftStructUnpacked&"] setup = ''' cpp2::MyThriftStructUnpacked ret; @@ -96,6 +97,7 @@ namespace cpp2 { ]}]''' [cases.packed] + oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296 param_types = ["const cpp2::MyThriftStructPacked&"] setup = ''' cpp2::MyThriftStructPacked ret; @@ -126,6 +128,7 @@ namespace cpp2 { ]}]''' [cases.packed_non_atomic] + oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296 param_types = ["const cpp2::MyThriftStructPackedNonAtomic&"] setup = ''' cpp2::MyThriftStructPackedNonAtomic ret; @@ -146,6 +149,7 @@ namespace cpp2 { ]}]''' [cases.out_of_order] + oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296 param_types = ["const cpp2::MyThriftStructOutOfOrder&"] setup = ''' cpp2::MyThriftStructOutOfOrder ret; @@ -164,6 +168,7 @@ namespace cpp2 { ]}]''' [cases.required] + oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296 param_types = ["const cpp2::MyThriftStructRequired&"] setup = ''' cpp2::MyThriftStructRequired ret; @@ -186,6 +191,7 @@ namespace cpp2 { ]}]''' [cases.box] + oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296 param_types = ["const cpp2::MyThriftStructBoxed&"] setup = ''' cpp2::MyThriftStructBoxed ret; @@ -207,6 +213,7 @@ namespace cpp2 { ]}]''' [cases.no_capture] + oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296 param_types = ["const cpp2::MyThriftStructBoxed&"] setup = ''' cpp2::MyThriftStructBoxed ret; diff --git a/test/integration/thrift_isset_missing.toml b/test/integration/thrift_isset_missing.toml index 7ba581e..63b578f 100644 --- a/test/integration/thrift_isset_missing.toml +++ b/test/integration/thrift_isset_missing.toml @@ -58,6 +58,7 @@ raw_definitions = ''' ''' [cases] [cases.present] + oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296 param_types = ["const FakeThriftWithData&"] setup = ''' FakeThriftWithData ret; @@ -78,6 +79,7 @@ raw_definitions = ''' {"name":"__isset", "staticSize":3} ]}]''' [cases.missing] + oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296 param_types = ["const FakeThriftWithoutData&"] setup = ''' FakeThriftWithoutData ret; @@ -98,6 +100,7 @@ raw_definitions = ''' {"name":"__isset", "staticSize":3} ]}]''' [cases.mixed] + oil_skip = 'oil does not support thrift isset yet' # https://github.com/facebookexperimental/object-introspection/issues/296 param_types = ["const Mixed&"] setup = ''' Mixed ret; diff --git a/test/integration/thrift_namespaces.toml b/test/integration/thrift_namespaces.toml index cc0a28c..f2ce9f9 100644 --- a/test/integration/thrift_namespaces.toml +++ b/test/integration/thrift_namespaces.toml @@ -10,6 +10,7 @@ thrift_definitions = ''' [cases] [cases.a] + oil_skip = 'enum type template arguments do not match' # https://github.com/facebookexperimental/object-introspection/issues/297 param_types = ["const namespaceA::namespaceB::TTTTT&"] setup = ''' namespaceA::namespaceB::TTTTT ret; diff --git a/test/integration/thrift_unions.toml b/test/integration/thrift_unions.toml index cc1c401..c7aa82e 100644 --- a/test/integration/thrift_unions.toml +++ b/test/integration/thrift_unions.toml @@ -33,6 +33,13 @@ namespace cpp2 { {"typeName":"storage_type", "name":"value_", "staticSize":4}, {"typeName":"underlying_type_t", "name":"type_", "staticSize":4} ]}]''' + expect_json_v2 = '''[{ + "staticSize":8, + "exclusiveSize":0, + "members":[ + {"typeNames":["storage_type"], "name":"value_", "staticSize":4, "exclusiveSize":4}, + {"typeNames":["underlying_type_t", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4} + ]}]''' [cases.dynamic_int] param_types = ["const cpp2::DynamicUnion&"] setup = ''' @@ -47,6 +54,13 @@ namespace cpp2 { {"typeName":"storage_type", "name":"value_", "staticSize":24}, {"typeName":"underlying_type_t", "name":"type_", "staticSize":4} ]}]''' + expect_json_v2 = '''[{ + "staticSize":32, + "exclusiveSize":4, + "members":[ + {"typeNames":["storage_type"], "name":"value_", "staticSize":24, "exclusiveSize":24}, + {"typeNames":["underlying_type_t", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4} + ]}]''' [cases.dynamic_vec] param_types = ["const cpp2::DynamicUnion&"] setup = ''' @@ -61,3 +75,10 @@ namespace cpp2 { {"typeName":"storage_type", "name":"value_", "staticSize":24}, {"typeName":"underlying_type_t", "name":"type_", "staticSize":4} ]}]''' + expect_json_v2 = '''[{ + "staticSize":32, + "exclusiveSize":4, + "members":[ + {"typeNames":["storage_type"], "name":"value_", "staticSize":24, "exclusiveSize":24}, + {"typeNames":["underlying_type_t", "type", "int32_t"], "name":"type_", "staticSize":4, "exclusiveSize":4} + ]}]''' diff --git a/test/integration/typedefed_parent.toml b/test/integration/typedefed_parent.toml index 05e7a9e..72c4a57 100644 --- a/test/integration/typedefed_parent.toml +++ b/test/integration/typedefed_parent.toml @@ -27,6 +27,13 @@ definitions = ''' {"name":"a", "staticSize":24, "dynamicSize":12, "length":3, "capacity":3}, {"name":"b", "staticSize":24, "dynamicSize":20, "length":5, "capacity":5} ]}]''' + expect_json_v2 = '''[{ + "staticSize":48, + "exclusiveSize":0, + "members":[ + {"name":"a", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3}, + {"name":"b", "staticSize":24, "exclusiveSize":24, "length":5, "capacity":5} + ]}]''' [cases.multilevel_typedef_parent] param_types = ["const Bar_2&"] setup = ''' @@ -40,3 +47,10 @@ definitions = ''' {"name":"a", "staticSize":24, "dynamicSize":12, "length":3, "capacity":3}, {"name":"c", "staticSize":24, "dynamicSize":20, "length":5, "capacity":5} ]}]''' + expect_json_v2 = '''[{ + "staticSize":48, + "exclusiveSize":0, + "members":[ + {"name":"a", "staticSize":24, "exclusiveSize":24, "length":3, "capacity":3}, + {"name":"c", "staticSize":24, "exclusiveSize":24, "length":5, "capacity":5} + ]}]''' diff --git a/test/integration/typedefs.toml b/test/integration/typedefs.toml index 7df0d7e..8c30337 100644 --- a/test/integration/typedefs.toml +++ b/test/integration/typedefs.toml @@ -4,9 +4,15 @@ definitions = ''' using UsingUInt64 = uint64_t; using IntVector = std::vector; using Anonymous = struct { int n; }; + + template + struct Proxy { + T t; + }; ''' [cases] [cases.c_style] + oil_disable = "oil can't pick up top level typedefs" param_types = ["TdUInt64"] setup = "return {};" expect_json = '''[{ @@ -24,6 +30,7 @@ definitions = ''' } ]}]''' [cases.using] + oil_disable = "oil can't pick up top level typedefs" param_types = ["UsingUInt64"] setup = "return {};" expect_json = '''[{ @@ -41,6 +48,7 @@ definitions = ''' } ]}]''' [cases.container] + oil_disable = "oil can't pick up top level typedefs" param_types = ["const IntVector&"] setup = "return {};" expect_json = '''[{ @@ -61,7 +69,7 @@ definitions = ''' } ]}]''' [cases.anonymous] - oil_skip = "TreeBuilder crashes" # https://github.com/facebookexperimental/object-introspection/issues/232 + oil_disable = "oil can't pick up top level typedefs" param_types = ["const Anonymous&"] setup = "return {};" expect_json = '''[{ @@ -86,3 +94,4 @@ definitions = ''' } ]} ]}]''' + diff --git a/test/integration/unions.toml b/test/integration/unions.toml index de84707..42341de 100644 --- a/test/integration/unions.toml +++ b/test/integration/unions.toml @@ -45,14 +45,17 @@ definitions = ''' param_types = ["const MyUnion&"] setup = "return 123;" expect_json = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}]' + expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}]' [cases.vector] param_types = ["const MyUnion&"] setup = "return std::vector{1,2,3};" expect_json = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}]' + expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}]' [cases.unordered_map] param_types = ["const MyUnion&"] setup = 'return std::unordered_map{{"a", "b"}, {"c","d"}};' expect_json = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}]' + expect_json_v2 = '[{"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}]' [cases.alignment] # Wrap the union in a pair as a way of inferring its alignment @@ -63,6 +66,11 @@ definitions = ''' {"staticSize":1, "dynamicSize":0, "exclusiveSize":1}, {"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"} ]}]''' + expect_json_v2 = '''[ + {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[ + {"staticSize":1, "dynamicSize":0, "exclusiveSize":1}, + {"staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]} + ]}]''' [cases.tagged_int] param_types = ["const TaggedUnion&"] @@ -72,6 +80,11 @@ definitions = ''' {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}, {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "NOT":"members"} ]}]''' + expect_json_v2 = '''[ + {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[ + {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}, + {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "members":[]} + ]}]''' [cases.tagged_vector] param_types = ["const TaggedUnion&"] setup = "return TaggedUnion{MyUnion{std::vector{1,2,3}}, 1};" @@ -80,6 +93,11 @@ definitions = ''' {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}, {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "NOT":"members"} ]}]''' + expect_json_v2 = '''[ + {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[ + {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}, + {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "members":[]} + ]}]''' [cases.tagged_unordered_map] param_types = ["const TaggedUnion&"] setup = 'return TaggedUnion{MyUnion{std::unordered_map{{"a", "b"}, {"c","d"}}}, 2};' @@ -88,3 +106,8 @@ definitions = ''' {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "NOT":"members"}, {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "NOT":"members"} ]}]''' + expect_json_v2 = '''[ + {"staticSize":64, "dynamicSize":0, "exclusiveSize":7, "members":[ + {"name":"storage", "staticSize":56, "dynamicSize":0, "exclusiveSize":56, "members":[]}, + {"name":"tag", "staticSize":1, "dynamicSize":0, "exclusiveSize":1, "members":[]} + ]}]''' diff --git a/test/test_add_children.cpp b/test/test_add_children.cpp index 06e4907..1ccb19f 100644 --- a/test/test_add_children.cpp +++ b/test/test_add_children.cpp @@ -50,14 +50,16 @@ TEST_F(AddChildrenTest, SimpleStruct) { TEST_F(AddChildrenTest, InheritanceStatic) { // Should do nothing test("oid_test_case_inheritance_access_public", R"( -[2] Pointer +[4] Pointer [0] Class: Public (size: 8) Parent (offset: 0) [1] Class: Base (size: 4) Member: base_int (offset: 0) - Primitive: int32_t +[3] Typedef: int32_t +[2] Typedef: __int32_t + Primitive: int32_t Member: public_int (offset: 4) - Primitive: int32_t + [3] )"); } diff --git a/test/test_drgn_parser.cpp b/test/test_drgn_parser.cpp index ca7b29b..c3c70b9 100644 --- a/test/test_drgn_parser.cpp +++ b/test/test_drgn_parser.cpp @@ -144,14 +144,16 @@ TEST_F(DrgnParserTest, SimpleUnion) { TEST_F(DrgnParserTest, Inheritance) { test("oid_test_case_inheritance_access_public", R"( -[2] Pointer +[4] Pointer [0] Class: Public (size: 8) Parent (offset: 0) [1] Class: Base (size: 4) Member: base_int (offset: 0) - Primitive: int32_t +[3] Typedef: int32_t +[2] Typedef: __int32_t + Primitive: int32_t Member: public_int (offset: 4) - Primitive: int32_t + [3] )"); } diff --git a/test/test_name_gen.cpp b/test/test_name_gen.cpp index fcb596c..4114e5d 100644 --- a/test/test_name_gen.cpp +++ b/test/test_name_gen.cpp @@ -446,7 +446,7 @@ TEST(NameGenTest, AnonymousTypes) { EXPECT_EQ(myenum.name(), "__oi_anon_1"); EXPECT_EQ(mytypedef.name(), "__oi_anon_2"); - EXPECT_EQ(myclass.inputName(), ""); - EXPECT_EQ(myenum.inputName(), ""); + EXPECT_EQ(myclass.inputName(), "__oi_anon_0"); + EXPECT_EQ(myenum.inputName(), "__oi_anon_1"); EXPECT_EQ(mytypedef.inputName(), ""); } diff --git a/types/README.md b/types/README.md index 92fd307..340fc10 100644 --- a/types/README.md +++ b/types/README.md @@ -8,10 +8,6 @@ This document describes the format of the container definition files contained i The fully-qualified name of the container type. This is used to match against the names of types contained in an executable's debug information. -- `ctype` - - A reference to the enum `ContainerTypeEnum` in OI's source code. - - `header` The name of the C++ header file in which this container is defined. @@ -35,42 +31,32 @@ This document describes the format of the container definition files contained i C++ code for the definition of a `getSizeType` function for this container. -- `handler` +- `processor.type` - C++ code for the definition of a `TypeHandler` class for this container. See - further down for a description. + The static type that will be filled in by this particular processor. Multiple + processors are allowed and recommended. + +- `processor.func` + + C++ code for the function body of a function with this signature: + `void(result::Element& el, std::stack& ins, ParsedData d)` which is + supplied with the data of the paired static type. + +- `traversal_func` + + C++ code for the function body of a function with this signature: + `static types::st::Unit getSizeType(const T& container, ST returnArg)` + where `ST` defines the combined static type of each supplied processor. ## Changes introduced with Typed Data Segment - `decl` and `func` fields are ignored when using `-ftyped-data-segment` and the `handler` field is used instead. -### TypeHandler Classes -A `TypeHandler` class describes both what a type will write into the data segment -and how to write it. It consists of two major parts: -- `using type = ...;` - describe what it will write into the data segment. -- `static types::st::Unit getSizeType(...)` - a function which takes a - const reference to a container and a `::type` by value and fills in the type. - -Example: -```cpp -template -struct TypeHandler> { - using type = - types::st::Pair, types::st::VarInt>; - - static types::st::Unit getSizeType( - const std::string & container, - typename TypeHandler>::type returnArg) { - bool sso = ((uintptr_t)container.data() < - (uintptr_t)(&container + sizeof(std::string))) && - ((uintptr_t)container.data() >= (uintptr_t)&container); - - return returnArg.write(container.capacity()).write(container.size()); - } -}; -``` - +## Changes introduced with TreeBuilder V2 +- `decl`, `func`, and `handler` fields are ignored when using `-ftree-builder-v2`. + The `TypeHandler` is instead constructed from `traversal_func` and `processor` + entries. ## Changes introduced with TypeGraph - `typeName` and `matcher` fields have been merged into the single field `type_name`. @@ -81,6 +67,16 @@ struct TypeHandler> { - There is one option to specify template parameters which should not be measured: `stub_template_params`. The type graph code will automatically determine what type of stubbing to apply to each parameter in this list. ### Deprecated Options +- `ctype` + + A reference to the enum `ContainerTypeEnum` in OI's source code. This is no + longer required with Tree Builder v2. + +- `handler` + + C++ code for the definition of a `TypeHandler` class for this container. Now + generated from `traversal_func` and `processor` entries. + - `numTemplateParams` The first `numTemplateParams` template parameters for this container represent diff --git a/types/array_type.toml b/types/array_type.toml index 32479ed..ede8f1e 100644 --- a/types/array_type.toml +++ b/types/array_type.toml @@ -49,3 +49,34 @@ struct TypeHandler> { } }; """ + +traversal_func = """ + auto tail = returnArg.write(container.size()); + + for (auto & it: container) { + tail = tail.delegate([&it](auto ret) { + return TypeHandler::getSizeType(it, ret); + }); + } + + return tail.finish(); +""" + +[[codegen.processor]] +type = "types::st::List::type>" +func = """ +static constexpr std::array names{"TODO"}; +static constexpr inst::Field childField{ + sizeof(T0), + "[]", + names, + TypeHandler::fields, + TypeHandler::processors, +}; + +size_t size = std::get(d.val).length; +el.exclusive_size = N0 == 0 ? 1 : 0; +el.container_stats.emplace(result::Element::ContainerStats{ .capacity = size, .length = size }); +for (size_t i = 0; i < size; i++) + ins.emplace(childField); +""" diff --git a/types/cxx11_string_type.toml b/types/cxx11_string_type.toml index 6b14254..5919c30 100644 --- a/types/cxx11_string_type.toml +++ b/types/cxx11_string_type.toml @@ -53,3 +53,34 @@ struct TypeHandler> { } }; """ + +traversal_func = """ + bool sso = ((uintptr_t)container.data() < + (uintptr_t)(&container + sizeof(std::__cxx11::basic_string))) && + ((uintptr_t)container.data() >= (uintptr_t)&container); + + return returnArg.write(container.capacity()) + .write(sso) + .write(container.size()); +""" + +[[codegen.processor]] +type = "types::st::VarInt" +func = """ +uint64_t capacity = std::get(d.val).value; +el.container_stats.emplace(result::Element::ContainerStats { .capacity = capacity }); +""" + +[[codegen.processor]] +type = "types::st::VarInt" +func = """ +bool sso = std::get(d.val).value; +if (!sso) + el.exclusive_size += el.container_stats->capacity * sizeof(T0); +""" + +[[codegen.processor]] +type = "types::st::VarInt" +func = """ +el.container_stats->length = std::get(d.val).value; +""" diff --git a/types/map_seq_type.toml b/types/map_seq_type.toml index efa107c..67cfa1d 100644 --- a/types/map_seq_type.toml +++ b/types/map_seq_type.toml @@ -66,3 +66,73 @@ struct TypeHandler> { } }; """ + +traversal_func = ''' + auto tail = returnArg.write((uintptr_t)&container) + .write(container.capacity()) + .write(container.size()); + + for (const auto& kv : container) { + tail = tail.delegate([&kv](auto ret) { + auto next = ret.delegate([&kv](typename TypeHandler::type ret) { + return OIInternal::getSizeType(kv.first, ret); + }); + return OIInternal::getSizeType(kv.second, next); + }); + } + + return tail.finish(); +''' + +[[codegen.processor]] +type = "types::st::VarInt" +func = "el.pointer = std::get(d.val).value;" + +[[codegen.processor]] +type = "types::st::VarInt" +func = ''' +el.container_stats.emplace(result::Element::ContainerStats{ .capacity = std::get(d.val).value }); +''' + +[[codegen.processor]] +type = '''types::st::List::type, + typename TypeHandler::type +>>''' +func = ''' +using element_type = std::pair; + +static constexpr std::array names{"TODO"}; + +static constexpr std::array entryFields{ + inst::Field{ + sizeof(T0), + "key", + names, + TypeHandler::fields, + TypeHandler::processors, + }, + inst::Field{ + sizeof(T1), + "value", + names, + TypeHandler::fields, + TypeHandler::processors, + }, +}; +static constexpr auto entryField = inst::Field { + sizeof(element_type), + sizeof(element_type) - sizeof(T0) - sizeof(T1), + "[]", + names, + entryFields, + std::array{}, +}; + +auto list = std::get(d.val); +el.container_stats->length = list.length; +el.exclusive_size += (el.container_stats->capacity - el.container_stats->length) * sizeof(element_type); + +for (size_t i = 0; i < list.length; i++) + ins.emplace(entryField); +''' diff --git a/types/pair_type.toml b/types/pair_type.toml index d82a719..5aa840f 100644 --- a/types/pair_type.toml +++ b/types/pair_type.toml @@ -45,3 +45,36 @@ struct TypeHandler> { } }; """ + +traversal_func = """ + return OIInternal::getSizeType( + container.second, + returnArg.delegate([&container](auto ret) { + return OIInternal::getSizeType(container.first, ret); + }) + ); +""" + +[[codegen.processor]] +type = "types::st::Pair::type, typename TypeHandler::type>" +func = """ +static constexpr std::array names{"TODO"}; +static constexpr auto firstField = inst::Field{ + sizeof(T0), + "first", + names, + TypeHandler::fields, + TypeHandler::processors, +}; +static constexpr auto secondField = inst::Field{ + sizeof(T1), + "second", + names, + TypeHandler::fields, + TypeHandler::processors, +}; + +el.exclusive_size = sizeof(std::pair) - sizeof(T0) - sizeof(T1); +ins.emplace(secondField); +ins.emplace(firstField); +""" diff --git a/types/seq_type.toml b/types/seq_type.toml index 5005af6..fe05866 100644 --- a/types/seq_type.toml +++ b/types/seq_type.toml @@ -1,10 +1,10 @@ [info] type_name = "std::vector" stub_template_params = [1] -ctype = "SEQ_TYPE" header = "vector" # Old: +ctype = "SEQ_TYPE" typeName = "std::vector<" ns = ["namespace std"] numTemplateParams = 1 @@ -63,3 +63,51 @@ struct TypeHandler> { } }; """ + +traversal_func = """ +auto tail = returnArg.write((uintptr_t)&container) + .write(container.capacity()) + .write(container.size()); + +// The double ampersand is needed otherwise this loop doesn't work with +// vector +for (auto&& it : container) { + tail = tail.delegate([&it](auto ret) { + return OIInternal::getSizeType(it, ret); + }); +} + +return tail.finish(); +""" + +[[codegen.processor]] +type = "types::st::VarInt" +func = """ +el.pointer = std::get(d.val).value; +""" + +[[codegen.processor]] +type = "types::st::VarInt" +func = """ +el.container_stats.emplace(result::Element::ContainerStats{ .capacity = std::get(d.val).value }); +""" + +[[codegen.processor]] +type = "types::st::List::type>" +func = """ +static constexpr std::array names{"TODO"}; +static constexpr auto childField = inst::Field{ + sizeof(T0), + "[]", + names, + TypeHandler::fields, + TypeHandler::processors, +}; + +auto list = std::get(d.val); +el.container_stats->length = list.length; +el.exclusive_size += (el.container_stats->capacity - el.container_stats->length) * sizeof(T0); + +for (size_t i = 0; i < list.length; i++) + ins.emplace(childField); +""" diff --git a/types/sorted_vec_set_type.toml b/types/sorted_vec_set_type.toml index 1f9f690..2bfe95d 100644 --- a/types/sorted_vec_set_type.toml +++ b/types/sorted_vec_set_type.toml @@ -1,7 +1,7 @@ [info] type_name = "folly::sorted_vector_set" -ctype = "SORTED_VEC_SET_TYPE" header = "folly/sorted_vector_types.h" +ctype = "SORTED_VEC_SET_TYPE" stub_template_params = [1,2] underlying_container_index = 4 diff --git a/types/thrift_isset_type.toml b/types/thrift_isset_type.toml index 3c1e8b7..edbc118 100644 --- a/types/thrift_isset_type.toml +++ b/types/thrift_isset_type.toml @@ -19,3 +19,7 @@ func = """ handler = """ // DummyHandler %1% """ + +traversal_func = """ +return returnArg; +""" diff --git a/types/weak_ptr_type.toml b/types/weak_ptr_type.toml index 9c02b8a..83eaf56 100644 --- a/types/weak_ptr_type.toml +++ b/types/weak_ptr_type.toml @@ -36,3 +36,6 @@ struct TypeHandler> { } }; """ + +traversal_func = "return returnArg;" +