codegen: carry decl and func with containerinfo

This commit is contained in:
Jake Hillion 2023-03-21 10:52:28 -07:00 committed by Jake Hillion
parent 10f47510d1
commit 76f525f43d
11 changed files with 176 additions and 192 deletions

View File

@ -25,7 +25,54 @@ extern "C" {
constexpr int oidMagicId = 0x01DE8;
struct ContainerInfo;
#define LIST_OF_CONTAINER_TYPES \
X(UNKNOWN_TYPE) \
X(ARRAY_TYPE) \
X(SMALL_VEC_TYPE) \
X(SET_TYPE) \
X(UNORDERED_SET_TYPE) \
X(SEQ_TYPE) \
X(LIST_TYPE) \
X(STD_MAP_TYPE) \
X(STD_UNORDERED_MAP_TYPE) \
X(MAP_SEQ_TYPE) \
X(BY_MULTI_QRT_TYPE) \
X(F14_MAP) \
X(F14_SET) \
X(FEED_QUICK_HASH_SET) \
X(FEED_QUICK_HASH_MAP) \
X(RADIX_TREE_TYPE) \
X(PAIR_TYPE) \
X(STRING_TYPE) \
X(FOLLY_IOBUF_TYPE) \
X(FOLLY_IOBUFQUEUE_TYPE) \
X(FB_STRING_TYPE) \
X(UNIQ_PTR_TYPE) \
X(SHRD_PTR_TYPE) \
X(FB_HASH_MAP_TYPE) \
X(FB_HASH_SET_TYPE) \
X(FOLLY_OPTIONAL_TYPE) \
X(OPTIONAL_TYPE) \
X(TRY_TYPE) \
X(REF_WRAPPER_TYPE) \
X(SORTED_VEC_SET_TYPE) \
X(REPEATED_FIELD_TYPE) \
X(CAFFE2_BLOB_TYPE) \
X(MULTI_MAP_TYPE) \
X(FOLLY_SMALL_HEAP_VECTOR_MAP) \
X(CONTAINER_ADAPTER_TYPE) \
X(MICROLIST_TYPE) \
X(ENUM_MAP_TYPE) \
X(BOOST_BIMAP_TYPE) \
X(STD_VARIANT_TYPE) \
X(THRIFT_ISSET_TYPE) \
X(WEAK_PTR_TYPE)
enum ContainerTypeEnum {
#define X(name) name,
LIST_OF_CONTAINER_TYPES
#undef X
};
struct RootInfo {
std::string varName;
@ -47,8 +94,9 @@ struct DrgnClassMemberInfo {
struct TypeHierarchy {
std::map<struct drgn_type*, std::vector<DrgnClassMemberInfo>> classMembersMap;
std::map<struct drgn_type*,
std::pair<ContainerInfo, std::vector<struct drgn_qualified_type>>>
std::map<
struct drgn_type*,
std::pair<ContainerTypeEnum, std::vector<struct drgn_qualified_type>>>
containerTypeMap;
std::map<struct drgn_type*, struct drgn_type*> typedefMap;
std::map<std::string, size_t> sizeMap;

View File

@ -20,6 +20,8 @@
#include <map>
namespace fs = std::filesystem;
ContainerTypeEnum containerTypeEnumFromStr(std::string& str) {
static const std::map<std::string, ContainerTypeEnum> nameMap = {
#define X(name) {#name, name},
@ -128,6 +130,30 @@ std::unique_ptr<ContainerInfo> ContainerInfo::loadFromFile(
std::optional<size_t> underlyingContainerIndex =
(*info)["underlyingContainerIndex"].value<size_t>();
toml::table* codegen = container["codegen"].as_table();
if (!codegen) {
LOG(ERROR) << "a container info file requires an `codegen` table";
return nullptr;
}
std::string decl;
if (std::optional<std::string> str =
(*codegen)["decl"].value<std::string>()) {
decl = std::move(*str);
} else {
LOG(ERROR) << "`codegen.decl` is a required field";
return nullptr;
}
std::string func;
if (std::optional<std::string> str =
(*codegen)["func"].value<std::string>()) {
func = std::move(*str);
} else {
LOG(ERROR) << "`codegen.func` is a required field";
return nullptr;
}
return std::unique_ptr<ContainerInfo>(new ContainerInfo{
std::move(typeName),
std::move(matcher),
@ -138,5 +164,10 @@ std::unique_ptr<ContainerInfo> ContainerInfo::loadFromFile(
std::move(replaceTemplateParamIndex),
allocatorIndex,
underlyingContainerIndex,
{
std::move(decl),
std::move(func),
},
});
}

View File

@ -17,63 +17,45 @@
#include <filesystem>
#include <optional>
#include <regex>
#include <set>
#include <string>
#include <vector>
namespace fs = std::filesystem;
#include "Common.h"
#define LIST_OF_CONTAINER_TYPES \
X(UNKNOWN_TYPE) \
X(ARRAY_TYPE) \
X(SMALL_VEC_TYPE) \
X(SET_TYPE) \
X(UNORDERED_SET_TYPE) \
X(SEQ_TYPE) \
X(LIST_TYPE) \
X(STD_MAP_TYPE) \
X(STD_UNORDERED_MAP_TYPE) \
X(MAP_SEQ_TYPE) \
X(BY_MULTI_QRT_TYPE) \
X(F14_MAP) \
X(F14_SET) \
X(FEED_QUICK_HASH_SET) \
X(FEED_QUICK_HASH_MAP) \
X(RADIX_TREE_TYPE) \
X(PAIR_TYPE) \
X(STRING_TYPE) \
X(FOLLY_IOBUF_TYPE) \
X(FOLLY_IOBUFQUEUE_TYPE) \
X(FB_STRING_TYPE) \
X(UNIQ_PTR_TYPE) \
X(SHRD_PTR_TYPE) \
X(FB_HASH_MAP_TYPE) \
X(FB_HASH_SET_TYPE) \
X(FOLLY_OPTIONAL_TYPE) \
X(OPTIONAL_TYPE) \
X(TRY_TYPE) \
X(REF_WRAPPER_TYPE) \
X(SORTED_VEC_SET_TYPE) \
X(REPEATED_FIELD_TYPE) \
X(CAFFE2_BLOB_TYPE) \
X(MULTI_MAP_TYPE) \
X(FOLLY_SMALL_HEAP_VECTOR_MAP) \
X(CONTAINER_ADAPTER_TYPE) \
X(MICROLIST_TYPE) \
X(ENUM_MAP_TYPE) \
X(BOOST_BIMAP_TYPE) \
X(STD_VARIANT_TYPE) \
X(THRIFT_ISSET_TYPE) \
X(WEAK_PTR_TYPE)
enum ContainerTypeEnum {
#define X(name) name,
LIST_OF_CONTAINER_TYPES
#undef X
};
ContainerTypeEnum containerTypeEnumFromStr(std::string& str);
const char* containerTypeEnumToStr(ContainerTypeEnum ty);
struct ContainerInfo {
struct Codegen {
std::string decl;
std::string func;
};
ContainerInfo(const ContainerInfo&) = delete;
ContainerInfo& operator=(const ContainerInfo& other) = delete;
ContainerInfo() = default;
ContainerInfo(std::string typeName_, std::regex matcher_,
std::optional<size_t> numTemplateParams_,
ContainerTypeEnum ctype_, std::string header_,
std::vector<std::string> ns_,
std::vector<size_t> replaceTemplateParamIndex_,
std::optional<size_t> allocatorIndex_,
std::optional<size_t> underlyingContainerIndex_,
ContainerInfo::Codegen codegen_)
: typeName(std::move(typeName_)),
matcher(std::move(matcher_)),
numTemplateParams(numTemplateParams_),
ctype(ctype_),
header(std::move(header_)),
ns(std::move(ns_)),
replaceTemplateParamIndex(std::move(replaceTemplateParamIndex_)),
allocatorIndex(allocatorIndex_),
underlyingContainerIndex(underlyingContainerIndex_),
codegen(std::move(codegen_)) {
}
std::string typeName;
std::regex matcher;
std::optional<size_t> numTemplateParams;
@ -86,9 +68,16 @@ struct ContainerInfo {
// adapter
std::optional<size_t> underlyingContainerIndex{};
static std::unique_ptr<ContainerInfo> loadFromFile(const fs::path& path);
Codegen codegen;
static std::unique_ptr<ContainerInfo> loadFromFile(
const std::filesystem::path& path);
bool operator<(const ContainerInfo& rhs) const {
return (typeName < rhs.typeName);
}
};
using ContainerInfoRefSet =
std::set<std::reference_wrapper<const ContainerInfo>,
std::less<ContainerInfo>>;

View File

@ -16,7 +16,6 @@
#include "FuncGen.h"
#include <glog/logging.h>
#include <toml++/toml.h>
#include <boost/format.hpp>
#include <map>
@ -322,27 +321,14 @@ void FuncGen::DefineTopLevelGetSizeSmartPtr(std::string& testCode,
}
bool FuncGen::DeclareGetSizeFuncs(std::string& testCode,
const std::set<ContainerInfo>& containerInfo,
const ContainerInfoRefSet& containerInfo,
bool chaseRawPointers) {
for (auto& cInfo : containerInfo) {
for (const ContainerInfo& cInfo : containerInfo) {
std::string ctype = cInfo.typeName;
ctype = ctype.substr(0, ctype.find("<", 0));
if (!typeToFuncMap.contains(cInfo.ctype)) {
LOG(ERROR) << "attempted to use container `"
<< containerTypeEnumToStr(cInfo.ctype)
<< "` for which a declaration was not provided";
return false;
}
auto& func = typeToDeclMap[cInfo.ctype];
boost::format fmt;
fmt = boost::format(func) % ctype;
/*if (cInfo.ctype == STRING_TYPE) {
fmt = boost::format(func);
} else {
fmt = boost::format(func) % ctype;
}*/
auto& decl = cInfo.codegen.decl;
boost::format fmt = boost::format(decl) % ctype;
testCode.append(fmt.str());
}
@ -359,28 +345,14 @@ bool FuncGen::DeclareGetSizeFuncs(std::string& testCode,
}
bool FuncGen::DefineGetSizeFuncs(std::string& testCode,
const std::set<ContainerInfo>& containerInfo,
const ContainerInfoRefSet& containerInfo,
bool chaseRawPointers) {
for (auto& cInfo : containerInfo) {
for (const ContainerInfo& cInfo : containerInfo) {
std::string ctype = cInfo.typeName;
ctype = ctype.substr(0, ctype.find("<", 0));
if (!typeToFuncMap.contains(cInfo.ctype)) {
LOG(ERROR) << "attempted to use container `"
<< containerTypeEnumToStr(cInfo.ctype)
<< "` for which a definition was not provided";
return false;
}
auto& func = typeToFuncMap[cInfo.ctype];
boost::format fmt;
fmt = boost::format(func) % ctype;
/*if (cInfo.ctype == STRING_TYPE) {
fmt = boost::format(func);
} else {
fmt = boost::format(func) % ctype;
}*/
auto& func = cInfo.codegen.func;
boost::format fmt = boost::format(func) % ctype;
testCode.append(fmt.str());
}
@ -422,38 +394,3 @@ void FuncGen::DeclareGetContainer(std::string& testCode) {
)";
testCode.append(func);
}
bool FuncGen::RegisterContainer(ContainerTypeEnum ctype, const fs::path& path) {
toml::table container;
try {
container = toml::parse_file(std::string(path));
} catch (const toml::parse_error& ex) {
LOG(ERROR) << "FuncGen::RegisterContainer: " << path << " : "
<< ex.description();
return false;
}
toml::table* codegen = container["codegen"].as_table();
if (!codegen) {
LOG(ERROR) << "a container info file requires an `codegen` table";
return false;
}
if (std::optional<std::string> str =
(*codegen)["decl"].value<std::string>()) {
typeToDeclMap.emplace(ctype, std::move(*str));
} else {
LOG(ERROR) << "`codegen.decl` is a required field";
return false;
}
if (std::optional<std::string> str =
(*codegen)["func"].value<std::string>()) {
typeToFuncMap.emplace(ctype, std::move(*str));
} else {
LOG(ERROR) << "`codegen.func` is a required field";
return false;
}
return true;
}

View File

@ -25,8 +25,6 @@ namespace fs = std::filesystem;
class FuncGen {
public:
bool RegisterContainer(ContainerTypeEnum, const fs::path& path);
static void DeclareStoreData(std::string& testCode);
static void DefineStoreData(std::string& testCode);
@ -40,10 +38,10 @@ class FuncGen {
static void DefineEncodeDataSize(std::string& testCode);
bool DeclareGetSizeFuncs(std::string& testCode,
const std::set<ContainerInfo>& containerInfo,
const ContainerInfoRefSet& containerInfo,
bool chaseRawPointers);
bool DefineGetSizeFuncs(std::string& testCode,
const std::set<ContainerInfo>& containerInfo,
const ContainerInfoRefSet& containerInfo,
bool chaseRawPointers);
static void DeclareGetContainer(std::string& testCode);
@ -67,8 +65,4 @@ class FuncGen {
static void DefineGetSizeTypedValueFunc(std::string& testCode,
const std::string& ctype);
private:
std::map<ContainerTypeEnum, std::string> typeToDeclMap;
std::map<ContainerTypeEnum, std::string> typeToFuncMap;
};

View File

@ -106,7 +106,7 @@ OICodeGen::OICodeGen(const Config& c, SymbolService& s)
bool OICodeGen::registerContainer(const fs::path& path) {
VLOG(1) << "registering container, path: " << path;
auto info = ContainerInfo::loadFromFile(path);
if (!info || !funcGen.RegisterContainer(info->ctype, path)) {
if (!info) {
return false;
}
@ -159,7 +159,8 @@ std::optional<const std::string_view> OICodeGen::fullyQualifiedName(
return typeNamePair->second.contents;
}
std::optional<ContainerInfo> OICodeGen::getContainerInfo(drgn_type* type) {
std::optional<std::reference_wrapper<const ContainerInfo>>
OICodeGen::getContainerInfo(drgn_type* type) {
auto name = fullyQualifiedName(type);
if (!name.has_value()) {
return std::nullopt;
@ -168,9 +169,9 @@ std::optional<ContainerInfo> OICodeGen::getContainerInfo(drgn_type* type) {
std::string nameStr = std::string(*name);
for (auto it = containerInfoList.rbegin(); it != containerInfoList.rend();
++it) {
const auto& info = *it;
if (std::regex_search(nameStr, info->matcher)) {
return *info;
const ContainerInfo& info = **it;
if (std::regex_search(nameStr, info.matcher)) {
return info;
}
}
return std::nullopt;
@ -359,11 +360,12 @@ void OICodeGen::replaceTemplateParameters(
drgn_type* type, TemplateParamList& template_params,
std::vector<std::string>& template_params_strings,
const std::string& nameWithoutTemplate) {
auto containerInfo = getContainerInfo(type);
if (!containerInfo.has_value()) {
auto optContainerInfo = getContainerInfo(type);
if (!optContainerInfo.has_value()) {
LOG(ERROR) << "Unknown container type: " << nameWithoutTemplate;
return;
}
const ContainerInfo& containerInfo = *optContainerInfo;
// Some containers will need special handling
if (nameWithoutTemplate == "bimap<") {
@ -373,12 +375,12 @@ void OICodeGen::replaceTemplateParameters(
removeTemplateParamAtIndex(template_params_strings, 3);
removeTemplateParamAtIndex(template_params_strings, 2);
} else {
for (auto const& index : containerInfo->replaceTemplateParamIndex) {
for (auto const& index : containerInfo.replaceTemplateParamIndex) {
replaceTemplateOperator(template_params, template_params_strings, index);
}
if (containerInfo->allocatorIndex) {
if (containerInfo.allocatorIndex) {
removeTemplateParamAtIndex(template_params_strings,
*containerInfo->allocatorIndex);
*containerInfo.allocatorIndex);
}
}
}
@ -672,22 +674,23 @@ bool OICodeGen::getContainerTemplateParams(drgn_type* type, bool& ifStub) {
}
typeName = transformTypeName(type, typeName);
auto containerInfo = getContainerInfo(type);
if (!containerInfo.has_value()) {
auto optContainerInfo = getContainerInfo(type);
if (!optContainerInfo.has_value()) {
LOG(ERROR) << "Unknown container type: " << typeName;
return false;
}
const ContainerInfo& containerInfo = *optContainerInfo;
std::vector<size_t> paramIdxs;
if (containerInfo->underlyingContainerIndex.has_value()) {
if (containerInfo->numTemplateParams.has_value()) {
if (containerInfo.underlyingContainerIndex.has_value()) {
if (containerInfo.numTemplateParams.has_value()) {
LOG(ERROR) << "Container adapters should not enumerate their template "
"parameters";
return false;
}
paramIdxs.push_back(*containerInfo->underlyingContainerIndex);
paramIdxs.push_back(*containerInfo.underlyingContainerIndex);
} else {
auto numTemplateParams = containerInfo->numTemplateParams;
auto numTemplateParams = containerInfo.numTemplateParams;
if (!numTemplateParams.has_value()) {
if (!drgn_type_has_template_parameters(type)) {
LOG(ERROR) << "Failed to find template params";
@ -703,7 +706,7 @@ bool OICodeGen::getContainerTemplateParams(drgn_type* type, bool& ifStub) {
}
}
return enumerateTemplateParamIdxs(type, *containerInfo, paramIdxs, ifStub);
return enumerateTemplateParamIdxs(type, containerInfo, paramIdxs, ifStub);
}
bool OICodeGen::enumerateTemplateParamIdxs(drgn_type* type,
@ -776,8 +779,8 @@ bool OICodeGen::enumerateTemplateParamIdxs(drgn_type* type,
auto& templateTypes =
containerTypeMapDrgn
.emplace(type,
std::pair(containerInfo, std::vector<drgn_qualified_type>()))
.emplace(type, std::pair(std::ref(containerInfo),
std::vector<drgn_qualified_type>()))
.first->second.second;
for (auto i : paramIdxs) {
@ -3011,7 +3014,7 @@ bool OICodeGen::generateJitCode(std::string& code) {
}
std::set<std::string> includedHeaders = config.defaultHeaders;
for (auto& e : containerTypesFuncDef) {
for (const ContainerInfo& e : containerTypesFuncDef) {
includedHeaders.insert(e.header);
}
@ -3070,7 +3073,7 @@ bool OICodeGen::generateJitCode(std::string& code) {
definitionsCode.append("// namespace uses\n");
std::set<std::string> usedNamespaces = config.defaultNamespaces;
for (const auto& v : containerTypesFuncDef) {
for (const ContainerInfo& v : containerTypesFuncDef) {
for (auto& e : v.ns) {
usedNamespaces.insert(std::string(e));
}
@ -3705,9 +3708,18 @@ void OICodeGen::setRootType(drgn_qualified_type rt) {
}
TypeHierarchy OICodeGen::getTypeHierarchy() {
std::map<
struct drgn_type*,
std::pair<ContainerTypeEnum, std::vector<struct drgn_qualified_type>>>
containerTypeMap;
for (auto const& [k, v] : containerTypeMapDrgn) {
const ContainerInfo& cinfo = v.first;
containerTypeMap[k] = {cinfo.ctype, v.second};
}
return {
.classMembersMap = getClassMembersMap(),
.containerTypeMap = containerTypeMapDrgn,
.containerTypeMap = std::move(containerTypeMap),
.typedefMap = typedefTypes,
.sizeMap = sizeMap,
.knownDummyTypeList = knownDummyTypeList,

View File

@ -114,8 +114,9 @@ class OICodeGen {
Config config{};
FuncGen funcGen;
using ContainerTypeMap =
std::pair<ContainerInfo, std::vector<drgn_qualified_type>>;
using ContainerTypeMapEntry =
std::pair<std::reference_wrapper<const ContainerInfo>,
std::vector<drgn_qualified_type>>;
using TemplateParamList =
std::vector<std::pair<drgn_qualified_type, std::string>>;
@ -126,7 +127,7 @@ class OICodeGen {
std::string linkageName;
std::map<drgn_type*, std::string> unnamedUnion;
std::map<std::string, size_t> sizeMap;
std::map<drgn_type*, ContainerTypeMap> containerTypeMapDrgn;
std::map<drgn_type*, ContainerTypeMapEntry> containerTypeMapDrgn;
std::vector<std::unique_ptr<ContainerInfo>> containerInfoList;
std::vector<drgn_type*> enumTypes;
std::vector<std::string> knownTypes;
@ -155,7 +156,7 @@ class OICodeGen {
std::set<drgn_type*> thriftIssetStructTypes;
std::vector<drgn_type*> topoSortedStructTypes;
std::set<ContainerInfo> containerTypesFuncDef;
ContainerInfoRefSet containerTypesFuncDef;
std::map<std::string, PaddingInfo> paddedStructs;
@ -191,7 +192,8 @@ class OICodeGen {
static SortedTypeDefMap getSortedTypeDefMap(
const std::map<drgn_type*, drgn_type*>& typedefTypeMap);
std::optional<ContainerInfo> getContainerInfo(drgn_type* type);
std::optional<std::reference_wrapper<const ContainerInfo>> getContainerInfo(
drgn_type* type);
void printAllTypes();
void printAllTypeNames();

View File

@ -66,32 +66,6 @@ void serialize(Archive& ar, PaddingInfo& p, const unsigned int version) {
INSTANCIATE_SERIALIZE(PaddingInfo)
template <class Archive>
void serialize(Archive& ar, ContainerInfo& info, const unsigned int version) {
verify_version<ContainerInfo>(version);
ar& info.typeName;
// Unfortunately boost serialization doesn't support `std::optional`,
// so we have to do this ourselves
size_t numTemplateParams = 0;
if (Archive::is_saving::value) {
numTemplateParams =
info.numTemplateParams.value_or(std::numeric_limits<size_t>::max());
}
ar& numTemplateParams;
if (Archive::is_loading::value) {
if (numTemplateParams == std::numeric_limits<size_t>::max()) {
info.numTemplateParams = std::nullopt;
} else {
info.numTemplateParams = numTemplateParams;
}
}
ar& info.ctype;
ar& info.header;
ar& info.ns;
}
INSTANCIATE_SERIALIZE(ContainerInfo)
template <class Archive>
void serialize(Archive& ar, struct drgn_location_description& location,
const unsigned int version) {

View File

@ -25,7 +25,6 @@
#include <boost/serialization/vector.hpp>
#include "Common.h"
#include "ContainerInfo.h"
#include "PaddingHunter.h"
#include "SymbolService.h"
@ -39,7 +38,6 @@
BOOST_CLASS_VERSION(Type, version)
DEFINE_TYPE_VERSION(PaddingInfo, 120, 3)
DEFINE_TYPE_VERSION(ContainerInfo, 200, 5)
DEFINE_TYPE_VERSION(struct drgn_location_description, 32, 2)
DEFINE_TYPE_VERSION(struct drgn_object_locator, 72, 2)
DEFINE_TYPE_VERSION(FuncDesc::Arg, 128, 2)
@ -51,7 +49,7 @@ DEFINE_TYPE_VERSION(struct drgn_type, 152, 4)
DEFINE_TYPE_VERSION(DrgnClassMemberInfo, 64, 3)
DEFINE_TYPE_VERSION(struct drgn_qualified_type, 16, 2)
DEFINE_TYPE_VERSION(RootInfo, 48, 2)
DEFINE_TYPE_VERSION(TypeHierarchy, 384, 6)
DEFINE_TYPE_VERSION(TypeHierarchy, 384, 7)
#undef DEFINE_TYPE_VERSION
@ -62,7 +60,6 @@ namespace boost::serialization {
void serialize(Archive&, Type&, const unsigned int)
DECL_SERIALIZE(PaddingInfo);
DECL_SERIALIZE(ContainerInfo);
DECL_SERIALIZE(FuncDesc::Arg);
DECL_SERIALIZE(FuncDesc);

View File

@ -571,8 +571,8 @@ void TreeBuilder::processContainer(const Variable& variable, Node& node) {
node.typeName + "'");
}
auto& [containerInfo, templateTypes] = entry->second;
kind = containerInfo.ctype;
auto& [containerKind, templateTypes] = entry->second;
kind = containerKind;
for (const auto& tt : templateTypes) {
elementTypes.push_back(tt);
}

View File

@ -76,7 +76,7 @@ void printClassMembersMap(
void printContainerTypeMap(
const std::map<
struct drgn_type*,
std::pair<ContainerInfo, std::vector<struct drgn_qualified_type>>>&
std::pair<ContainerTypeEnum, std::vector<struct drgn_qualified_type>>>&
containerTypeMap) {
printf("{");
bool isFirstItem = true;