JakeHillion/object-introspection
1
0
Fork 0
mirror of https://github.com/JakeHillion/object-introspection.git synced 2024-11-12 21:56:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

add command line feature addition/removal

Browse Source
This commit is contained in:
Jake Hillion 2023-04-17 11:13:59 -07:00 committed by Jake Hillion
parent 5971643101
commit 641a128b39
23 changed files with 428 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/compile-time-oil/Makefile
View File

@ -12,7 +12,7 @@ OilVectorOfStrings.o: OilVectorOfStrings.cpp
oilgen:
rm -f oilgen
(cd ../../ && make oid-devel)
(cd ../../ && make oid)
ln -s ../../build/oilgen oilgen
OilVectorOfStrings: oilgen OilVectorOfStrings.o

1
src/CMakeLists.txt
View File

@ -23,6 +23,7 @@ target_link_libraries(symbol_service
add_library(codegen
ContainerInfo.cpp
Features.cpp
FuncGen.cpp
OICodeGen.cpp
)

48
src/Features.cpp Normal file
View File

@ -0,0 +1,48 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "Features.h"
#include <map>
namespace ObjectIntrospection {
Feature featureFromStr(std::string_view str) {
static const std::map<std::string_view, Feature> nameMap = {
#define X(name, str) {str, Feature::name},
OI_FEATURE_LIST
#undef X
};
if (auto search = nameMap.find(str); search != nameMap.end()) {
return search->second;
}
return Feature::UnknownFeature;
}
const char* featureToStr(Feature f) {
switch (f) {
#define X(name, str) \
case Feature::name: \
return str;
OI_FEATURE_LIST
#undef X
default:
return "UnknownFeature";
}
}
} // namespace ObjectIntrospection

46
src/Features.h Normal file
View File

@ -0,0 +1,46 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <array>
#include <string_view>
#define OI_FEATURE_LIST \
X(ChaseRawPointers, "chase-raw-pointers") \
X(PackStructs, "pack-structs") \
X(GenPaddingStats, "gen-padding-stats") \
X(CaptureThriftIsset, "capture-thrift-isset") \
X(PolymorphicInheritance, "polymorphic-inheritance")
namespace ObjectIntrospection {
enum class Feature {
UnknownFeature,
#define X(name, _) name,
OI_FEATURE_LIST
#undef X
};
Feature featureFromStr(std::string_view);
const char* featureToStr(Feature);
constexpr std::array allFeatures = {
#define X(name, _) Feature::name,
OI_FEATURE_LIST
#undef X
};
} // namespace ObjectIntrospection

63
src/OICodeGen.cpp
View File

@ -19,6 +19,7 @@
#include <glog/logging.h>
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/join.hpp>
#include <boost/algorithm/string/regex.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <boost/format.hpp>
@ -61,6 +62,13 @@ std::unique_ptr<OICodeGen> OICodeGen::buildFromConfig(const Config& c,
OICodeGen::OICodeGen(const Config& c, SymbolService& s)
: config{c}, symbols{s} {
chaseRawPointers = config.features.contains(Feature::ChaseRawPointers);
packStructs = config.features.contains(Feature::PackStructs);
genPaddingStats = config.features.contains(Feature::GenPaddingStats);
captureThriftIsset = config.features.contains(Feature::CaptureThriftIsset);
polymorphicInheritance =
config.features.contains(Feature::PolymorphicInheritance);
// TODO: Should folly::Range just be added as a container?
auto typesToStub = std::array{
"SharedMutex",
@ -985,7 +993,7 @@ bool OICodeGen::recordChildren(drgn_type* type) {
* types in the program to build the reverse mapping.
*/
bool OICodeGen::enumerateChildClasses() {
if (!config.polymorphicInheritance) {
if (!polymorphicInheritance) {
return true;
}
@ -1314,7 +1322,7 @@ bool OICodeGen::isEmptyClassOrFunctionType(drgn_type* type,
* one or more virtual member functions or virtual base classes).
*/
bool OICodeGen::isDynamic(drgn_type* type) const {
if (!config.polymorphicInheritance || !drgn_type_has_virtuality(type)) {
if (!polymorphicInheritance || !drgn_type_has_virtuality(type)) {
return false;
}
@ -1992,7 +2000,7 @@ bool OICodeGen::getDrgnTypeNameInt(drgn_type* type, std::string& outName) {
name.assign(drgn_type_name(type));
} else if (drgn_type_kind(type) == DRGN_TYPE_POINTER) {
drgn_type* underlyingType = getPtrUnderlyingType(type);
if (config.chaseRawPointers &&
if (chaseRawPointers &&
drgn_type_kind(underlyingType) != DRGN_TYPE_FUNCTION) {
// For pointers, figure out name for the underlying type then add
// appropriate number of '*'
@ -2262,7 +2270,7 @@ bool OICodeGen::generateStructDef(drgn_type* e, std::string& code) {
std::string structDefinition;
if (paddingInfo.paddingSize != 0 && config.genPaddingStats) {
if (paddingInfo.paddingSize != 0 && genPaddingStats) {
structDefinition.append("/* offset | size */ ");
}
@ -2278,8 +2286,7 @@ bool OICodeGen::generateStructDef(drgn_type* e, std::string& code) {
std::to_string(*alignment / CHAR_BIT) + ")");
}
if (config.packStructs &&
(kind == DRGN_TYPE_STRUCT || kind == DRGN_TYPE_CLASS) &&
if (packStructs && (kind == DRGN_TYPE_STRUCT || kind == DRGN_TYPE_CLASS) &&
violatesAlignmentRequirement && paddingInfo.paddingSize == 0) {
structDefinition.append(" __attribute__((__packed__))");
}
@ -2298,7 +2305,7 @@ bool OICodeGen::generateStructDef(drgn_type* e, std::string& code) {
structDefinition.append("};\n");
if (config.genPaddingStats) {
if (genPaddingStats) {
auto paddedStructFound = paddedStructs.find(*tmpStr);
if (paddedStructFound == paddedStructs.end()) {
@ -2490,7 +2497,7 @@ std::optional<uint64_t> OICodeGen::generateMember(
currOffsetBits = 0;
VLOG(1) << "Member size: " << memberSize;
} else {
addSizeComment(config.genPaddingStats, code, currOffsetBits, memberSize);
addSizeComment(genPaddingStats, code, currOffsetBits, memberSize);
currOffsetBits = currOffsetBits + memberSize;
}
@ -2683,12 +2690,12 @@ bool OICodeGen::generateStructMembers(
bool isThriftIssetStruct =
typeName.starts_with("isset_bitset<") && memberName == "__isset";
if (config.captureThriftIsset && isThriftIssetStruct &&
if (captureThriftIsset && isThriftIssetStruct &&
memberIndex == members.size() - 1) {
thriftIssetStructTypes.insert(e);
}
if (config.genPaddingStats) {
if (genPaddingStats) {
paddingInfo.isThriftStruct = isThriftIssetStruct;
/*
@ -3232,12 +3239,12 @@ bool OICodeGen::generateJitCode(std::string& code) {
functionsCode.append("namespace OIInternal {\nnamespace {\n");
functionsCode.append("// functions -----\n");
if (!funcGen.DeclareGetSizeFuncs(functionsCode, containerTypesFuncDef,
config.chaseRawPointers)) {
chaseRawPointers)) {
LOG(ERROR) << "declaring get size for containers failed";
return false;
}
if (config.chaseRawPointers) {
if (chaseRawPointers) {
functionsCode.append(R"(
template<typename T>
void getSizeType(const T* t, size_t& returnArg);
@ -3258,7 +3265,7 @@ bool OICodeGen::generateJitCode(std::string& code) {
}
}
if (config.useDataSegment || config.chaseRawPointers) {
if (config.useDataSegment || chaseRawPointers) {
funcGen.DeclareStoreData(functionsCode);
}
@ -3270,12 +3277,12 @@ bool OICodeGen::generateJitCode(std::string& code) {
}
if (!funcGen.DefineGetSizeFuncs(functionsCode, containerTypesFuncDef,
config.chaseRawPointers)) {
chaseRawPointers)) {
LOG(ERROR) << "defining get size for containers failed";
return false;
}
if (config.chaseRawPointers) {
if (chaseRawPointers) {
functionsCode.append(R"(
template<typename T>
void getSizeType(const T* s_ptr, size_t& returnArg)
@ -3744,22 +3751,22 @@ std::string OICodeGen::Config::toString() const {
ignoreMembers += ';';
}
return (useDataSegment ? "" : "Dont") + "UseDataSegment,"s +
(chaseRawPointers ? "" : "Dont") + "ChaseRawPointers,"s +
(packStructs ? "" : "Dont") + "PackStructs,"s +
(genPaddingStats ? "" : "Dont") + "GenPaddingStats,"s +
(captureThriftIsset ? "" : "Dont") + "CaptureThriftIsset,"s +
(polymorphicInheritance ? "" : "Dont") + "PolymorphicInheritance,"s +
ignoreMembers;
return boost::algorithm::join(toOptions(), ",") + "," + ignoreMembers;
}
std::vector<std::string> OICodeGen::Config::toOptions() const {
return {"", // useDataSegment is always true?
(chaseRawPointers ? "-n" : ""),
(packStructs ? "" : "-z"),
(genPaddingStats ? "" : "-w"),
(captureThriftIsset ? "-T" : ""),
(polymorphicInheritance ? "-P" : "")};
std::vector<std::string> options;
options.reserve(allFeatures.size());
for (const auto f : allFeatures) {
if (features.contains(f)) {
options.emplace_back(std::string("-f") + featureToStr(f));
} else {
options.emplace_back(std::string("-F") + featureToStr(f));
}
}
return options;
}
void OICodeGen::initializeCodeGen() {

15
src/OICodeGen.h
View File

@ -28,6 +28,7 @@ struct irequest;
#include "Common.h"
#include "ContainerInfo.h"
#include "Features.h"
#include "FuncGen.h"
#include "PaddingHunter.h"
@ -35,6 +36,7 @@ extern "C" {
#include <drgn.h>
}
using namespace ObjectIntrospection;
namespace fs = std::filesystem;
struct ParentMember {
@ -54,11 +56,8 @@ class OICodeGen {
* uninitialized field" warning if they missed any.
*/
bool useDataSegment;
bool chaseRawPointers;
bool packStructs;
bool genPaddingStats;
bool captureThriftIsset;
bool polymorphicInheritance;
std::set<Feature> features{};
std::set<fs::path> containerConfigPaths{};
std::set<std::string> defaultHeaders{};
@ -114,6 +113,12 @@ class OICodeGen {
Config config{};
FuncGen funcGen;
bool chaseRawPointers;
bool packStructs;
bool genPaddingStats;
bool captureThriftIsset;
bool polymorphicInheritance;
using ContainerTypeMapEntry =
std::pair<std::reference_wrapper<const ContainerInfo>,
std::vector<drgn_qualified_type>>;

93
src/OID.cpp
View File

@ -22,15 +22,18 @@
#include <cstdlib>
#include <filesystem>
#include <iostream>
#include <map>
extern "C" {
#include <getopt.h>
#include <libgen.h>
}
#include "Features.h"
#include "Metrics.h"
#include "OIDebugger.h"
#include "OIOpts.h"
#include "OIUtils.h"
#include "PaddingHunter.h"
#include "TimeUtils.h"
#include "TreeBuilder.h"
@ -156,6 +159,18 @@ constexpr static OIOpts opts{
"Follow runtime polymorphic inheritance hierarchies"},
OIOpt{'m', "mode", required_argument, "[prod]",
"Allows to specify a mode of operation/group of settings"},
OIOpt{'f', "enable-feature", required_argument, nullptr,
"Enable a specific feature: ["
#define X(name, str) str ","
OI_FEATURE_LIST
#undef X
"]"},
OIOpt{'F', "disable-feature", required_argument, nullptr,
"Disable a specific feature: ["
#define X(name, str) str ","
OI_FEATURE_LIST
#undef X
"]"},
};
void usage() {
@ -266,11 +281,11 @@ struct Config {
} // namespace Oid
static ExitStatus::ExitStatus runScript(const std::string& fileName,
std::istream& script,
const Oid::Config& oidConfig,
const OICodeGen::Config& codeGenConfig,
const TreeBuilder::Config& tbConfig) {
static ExitStatus::ExitStatus runScript(
const std::string& fileName, std::istream& script,
const Oid::Config& oidConfig, const OICodeGen::Config& codeGenConfig,
const OICompiler::Config& compilerConfig,
const TreeBuilder::Config& tbConfig) {
if (!fileName.empty()) {
VLOG(1) << "SCR FILE: " << fileName;
}
@ -279,11 +294,11 @@ static ExitStatus::ExitStatus runScript(const std::string& fileName,
std::shared_ptr<OIDebugger> oid; // share oid with the global signal handler
if (oidConfig.pid != 0) {
oid = std::make_shared<OIDebugger>(oidConfig.pid, oidConfig.configFile,
codeGenConfig, tbConfig);
oid = std::make_shared<OIDebugger>(oidConfig.pid, codeGenConfig,
compilerConfig, tbConfig);
} else {
oid = std::make_shared<OIDebugger>(
oidConfig.debugInfoFile, oidConfig.configFile, codeGenConfig, tbConfig);
oid = std::make_shared<OIDebugger>(oidConfig.debugInfoFile, codeGenConfig,
compilerConfig, tbConfig);
}
weak_oid = oid; // set the weak_ptr for signal handlers
@ -463,12 +478,13 @@ int main(int argc, char* argv[]) {
std::string configGenOption;
std::optional<fs::path> jsonPath{std::nullopt};
std::map<Feature, bool> features = {
{Feature::PackStructs, true},
{Feature::GenPaddingStats, true},
};
bool logAllStructs = true;
bool chaseRawPointers = false;
bool packStructs = true;
bool dumpDataSegment = false;
bool captureThriftIsset = false;
bool polymorphicInheritance = false;
Metrics::Tracing _("main");
#ifndef OSS_ENABLE
@ -485,13 +501,25 @@ int main(int argc, char* argv[]) {
while ((c = getopt_long(argc, argv, opts.shortOpts(), opts.longOpts(),
nullptr)) != -1) {
switch (c) {
case 'F':
[[fallthrough]];
case 'f':
if (auto f = featureFromStr(optarg); f != Feature::UnknownFeature) {
features[f] = c == 'f'; // '-f' enables, '-F' disables
} else {
LOG(ERROR) << "Invalid feature: " << optarg << " specified!";
usage();
return ExitStatus::UsageError;
}
break;
case 'm': {
if (strcmp("prod", optarg) == 0) {
// change default settings for prod
oidConfig.hardDisableDrgn = true;
oidConfig.cacheRemoteDownload = true;
oidConfig.cacheBasePath = "/tmp/oid-cache";
chaseRawPointers = true;
features[Feature::ChaseRawPointers] = true;
} else {
LOG(ERROR) << "Invalid mode: " << optarg << " specified!";
usage();
@ -616,13 +644,13 @@ int main(int argc, char* argv[]) {
oidConfig.removeMappings = true;
break;
case 'n':
chaseRawPointers = true;
features[Feature::ChaseRawPointers] = true;
break;
case 'a':
logAllStructs = true;
break;
case 'z':
packStructs = false;
features[Feature::PackStructs] = false;
break;
case 'B':
dumpDataSegment = true;
@ -637,16 +665,16 @@ int main(int argc, char* argv[]) {
oidConfig.timeout_s = atoi(optarg);
break;
case 'w':
oidConfig.genPaddingStats = false;
features[Feature::GenPaddingStats] = false;
break;
case 'J':
jsonPath = optarg != nullptr ? optarg : "oid_out.json";
break;
case 'T':
captureThriftIsset = true;
features[Feature::CaptureThriftIsset] = true;
break;
case 'P':
polymorphicInheritance = true;
features[Feature::PolymorphicInheritance] = true;
break;
case 'h':
default:
@ -694,38 +722,43 @@ int main(int argc, char* argv[]) {
scriptSource = "entry:unknown_function:arg0";
}
OICompiler::Config compilerConfig{};
OICodeGen::Config codeGenConfig{
.useDataSegment = true,
.chaseRawPointers = chaseRawPointers,
.packStructs = packStructs,
.genPaddingStats = oidConfig.genPaddingStats,
.captureThriftIsset = captureThriftIsset,
.polymorphicInheritance = polymorphicInheritance,
.features = {}, // fill in after processing the config file
};
TreeBuilder::Config tbConfig{
.features = {}, // fill in after processing the config file
.logAllStructs = logAllStructs,
.chaseRawPointers = chaseRawPointers,
.genPaddingStats = oidConfig.genPaddingStats,
.dumpDataSegment = dumpDataSegment,
.jsonPath = jsonPath,
};
auto featureSet = OIUtils::processConfigFile(oidConfig.configFile, features,
compilerConfig, codeGenConfig);
if (!featureSet) {
return ExitStatus::UsageError;
}
codeGenConfig.features = *featureSet;
tbConfig.features = *featureSet;
if (!scriptFile.empty()) {
if (!std::filesystem::exists(scriptFile)) {
LOG(ERROR) << "Non-existent script file: " << scriptFile;
return ExitStatus::FileNotFoundError;
}
std::ifstream script(scriptFile);
auto status =
runScript(scriptFile, script, oidConfig, codeGenConfig, tbConfig);
auto status = runScript(scriptFile, script, oidConfig, codeGenConfig,
compilerConfig, tbConfig);
if (status != ExitStatus::Success) {
return status;
}
} else if (!scriptSource.empty()) {
std::istringstream script(scriptSource);
auto status =
runScript(scriptFile, script, oidConfig, codeGenConfig, tbConfig);
auto status = runScript(scriptFile, script, oidConfig, codeGenConfig,
compilerConfig, tbConfig);
if (status != ExitStatus::Success) {
return status;
}

31
src/OIDebugger.cpp
View File

@ -272,7 +272,7 @@ bool OIDebugger::segmentInit(void) {
VLOG(1) << "segConfig size " << sizeof(segConfig);
if (segmentConfigFile.fail()) {
LOG(ERROR) << "init: error in writing configFile" << configFilePath
LOG(ERROR) << "init: error in writing configFile" << segConfigFilePath
<< strerror(errno);
}
VLOG(1) << "About to flush segment config file";
@ -1868,28 +1868,21 @@ bool OIDebugger::removeTrap(pid_t pid, const trapInfo& t) {
return true;
}
OIDebugger::OIDebugger(std::string configFile, OICodeGen::Config genConfig,
OIDebugger::OIDebugger(OICodeGen::Config genConfig, OICompiler::Config ccConfig,
TreeBuilder::Config tbConfig)
: configFilePath{configFile},
: compilerConfig{std::move(ccConfig)},
generatorConfig{std::move(genConfig)},
treeBuilderConfig{std::move(tbConfig)} {
if (configFilePath.empty()) {
configFilePath = fs::current_path() / "oid.toml";
}
VLOG(1) << "config file: " << configFilePath;
debug = true;
OIUtils::processConfigFile(configFilePath, compilerConfig, generatorConfig);
cache.generatorConfig = generatorConfig;
VLOG(1) << "CodeGen config: " << generatorConfig.toString();
}
OIDebugger::OIDebugger(pid_t pid, std::string configFile,
OICodeGen::Config genConfig,
OIDebugger::OIDebugger(pid_t pid, OICodeGen::Config genConfig,
OICompiler::Config ccConfig,
TreeBuilder::Config tbConfig)
: OIDebugger(std::move(configFile), std::move(genConfig),
: OIDebugger(std::move(genConfig), std::move(ccConfig),
std::move(tbConfig)) {
traceePid = pid;
symbols = std::make_shared<SymbolService>(traceePid);
@ -1898,10 +1891,10 @@ OIDebugger::OIDebugger(pid_t pid, std::string configFile,
cache.symbols = symbols;
}
OIDebugger::OIDebugger(fs::path debugInfo, std::string configFile,
OICodeGen::Config genConfig,
OIDebugger::OIDebugger(fs::path debugInfo, OICodeGen::Config genConfig,
OICompiler::Config ccConfig,
TreeBuilder::Config tbConfig)
: OIDebugger(std::move(configFile), std::move(genConfig),
: OIDebugger(std::move(genConfig), std::move(ccConfig),
std::move(tbConfig)) {
symbols = std::make_shared<SymbolService>(std::move(debugInfo));
cache.symbols = symbols;
@ -2848,7 +2841,7 @@ bool OIDebugger::processTargetData() {
const auto& [rootType, typeHierarchy, paddingInfos] = typeInfo->second;
VLOG(1) << "Root type addr: " << (void*)rootType.type.type;
if (treeBuilderConfig.genPaddingStats) {
if (treeBuilderConfig.features.contains(Feature::GenPaddingStats)) {
paddingHunter.localPaddedStructs = paddingInfos;
typeTree.setPaddedStructs(&paddingHunter.localPaddedStructs);
}
@ -2872,7 +2865,7 @@ bool OIDebugger::processTargetData() {
continue;
}
if (treeBuilderConfig.genPaddingStats) {
if (treeBuilderConfig.features.contains(Feature::GenPaddingStats)) {
paddingHunter.processLocalPaddingInfo();
}
}
@ -2886,7 +2879,7 @@ bool OIDebugger::processTargetData() {
typeTree.dumpJson();
}
if (treeBuilderConfig.genPaddingStats) {
if (treeBuilderConfig.features.contains(Feature::GenPaddingStats)) {
paddingHunter.outputPaddingInfo();
}

9
src/OIDebugger.h
View File

@ -32,11 +32,13 @@
namespace fs = std::filesystem;
class OIDebugger {
OIDebugger(std::string, OICodeGen::Config, TreeBuilder::Config);
OIDebugger(OICodeGen::Config, OICompiler::Config, TreeBuilder::Config);
public:
OIDebugger(pid_t, std::string, OICodeGen::Config, TreeBuilder::Config);
OIDebugger(fs::path, std::string, OICodeGen::Config, TreeBuilder::Config);
OIDebugger(pid_t, OICodeGen::Config, OICompiler::Config, TreeBuilder::Config);
OIDebugger(fs::path, OICodeGen::Config, OICompiler::Config,
TreeBuilder::Config);
bool segmentInit(void);
bool stopTarget(void);
bool interruptTarget(void);
@ -142,7 +144,6 @@ class OIDebugger {
}
private:
std::string configFilePath;
bool debug = false;
bool enableJitLogging = false;
pid_t traceePid{};

11
src/OIGenerator.cpp
View File

@ -184,13 +184,20 @@ int OIGenerator::generate(fs::path& primaryObject, SymbolService& symbols) {
std::vector<std::tuple<drgn_qualified_type, std::string>> oilTypes =
findOilTypesAndNames(prog);
std::map<Feature, bool> featuresMap = {
{Feature::PackStructs, true},
};
OICodeGen::Config generatorConfig{};
OICompiler::Config compilerConfig{};
if (!OIUtils::processConfigFile(configFilePath, compilerConfig,
generatorConfig)) {
auto features = OIUtils::processConfigFile(configFilePath, featuresMap,
compilerConfig, generatorConfig);
if (!features) {
LOG(ERROR) << "failed to process config file";
return -1;
}
generatorConfig.features = *features;
generatorConfig.useDataSegment = false;
size_t failures = 0;

18
src/OILibraryImpl.cpp
View File

@ -80,16 +80,22 @@ void OILibraryImpl::initCompiler() {
symbols = std::make_shared<SymbolService>(getpid());
compilerConfig.generateJitDebugInfo = _self->opts.generateJitDebugInfo;
generatorConfig.useDataSegment = false;
generatorConfig.chaseRawPointers = _self->opts.chaseRawPointers;
generatorConfig.packStructs = true;
generatorConfig.genPaddingStats = false;
}
bool OILibraryImpl::processConfigFile() {
return OIUtils::processConfigFile(_self->opts.configFilePath, compilerConfig,
generatorConfig);
auto features = OIUtils::processConfigFile(
_self->opts.configFilePath,
{
{Feature::ChaseRawPointers, _self->opts.chaseRawPointers},
{Feature::PackStructs, true},
},
compilerConfig, generatorConfig);
if (!features) {
return false;
}
generatorConfig.features = *features;
return true;
}
template <class T, class F>

46
src/OIUtils.cpp
View File

@ -13,6 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "OIUtils.h"
#include <glog/logging.h>
#include <toml++/toml.h>
@ -22,18 +24,16 @@
#include <boost/property_tree/ptree.hpp>
#include <filesystem>
#include "OICodeGen.h"
#include "OICompiler.h"
namespace fs = std::filesystem;
namespace OIUtils {
using namespace ObjectIntrospection;
using namespace std::literals;
bool processConfigFile(const std::string& configFilePath,
OICompiler::Config& compilerConfig,
OICodeGen::Config& generatorConfig) {
std::optional<std::set<Feature>> processConfigFile(
const std::string& configFilePath, std::map<Feature, bool> featureMap,
OICompiler::Config& compilerConfig, OICodeGen::Config& generatorConfig) {
fs::path configDirectory = fs::path(configFilePath).remove_filename();
toml::table config;
@ -42,7 +42,29 @@ bool processConfigFile(const std::string& configFilePath,
} catch (const toml::parse_error& ex) {
LOG(ERROR) << "processConfigFileToml: " << configFilePath << " : "
<< ex.description();
return false;
return {};
}
if (toml::array* features = config["features"].as_array()) {
for (auto&& el : *features) {
auto* featureStr = el.as_string();
if (!featureStr) {
LOG(ERROR) << "enabled features must be strings";
return {};
}
if (auto f = featureFromStr(featureStr->get());
f != Feature::UnknownFeature) {
// Inserts element(s) into the container, if the container doesn't
// already contain an element with an equivalent key. Hence prefer
// command line enabling/disabling.
featureMap.insert({f, true});
} else {
LOG(ERROR) << "unrecognised feature: " << featureStr->get()
<< " specified in config";
return {};
}
}
}
if (toml::table* types = config["types"].as_table()) {
@ -111,7 +133,7 @@ bool processConfigFile(const std::string& configFilePath,
auto* type = (*ignore)["type"].as_string();
if (!type) {
LOG(ERROR) << "Config entry 'ignore' must specify a type";
return false;
return {};
}
auto* members = (*ignore)["members"].as_array();
@ -129,7 +151,13 @@ bool processConfigFile(const std::string& configFilePath,
}
}
return true;
std::set<Feature> featuresSet;
for (auto [k, v] : featureMap) {
if (v) {
featuresSet.insert(k);
}
}
return featuresSet;
}
} // namespace OIUtils

10
src/OIUtils.h
View File

@ -15,13 +15,17 @@
*/
#pragma once
#include <optional>
#include <set>
#include "Features.h"
#include "OICodeGen.h"
#include "OICompiler.h"
namespace OIUtils {
bool processConfigFile(const std::string& configFilePath,
OICompiler::Config& compilerConfig,
OICodeGen::Config& generatorConfig);
std::optional<std::set<Feature>> processConfigFile(
const std::string& configFilePath, std::map<Feature, bool> featureMap,
OICompiler::Config& compilerConfig, OICodeGen::Config& generatorConfig);
} // namespace OIUtils

7
src/TreeBuilder.cpp
View File

@ -53,6 +53,9 @@ enum class TrackPointerTag : uint64_t {
TreeBuilder::TreeBuilder(Config c) : config{std::move(c)} {
buffer = std::make_unique<msgpack::sbuffer>();
chaseRawPointers = config.features.contains(Feature::ChaseRawPointers);
genPaddingStats = config.features.contains(Feature::GenPaddingStats);
auto testdbPath = "/tmp/testdb_" + std::to_string(getpid());
if (auto status = rocksdb::DestroyDB(testdbPath, {}); !status.ok()) {
LOG(FATAL) << "RocksDB error while destroying database: "
@ -414,7 +417,7 @@ TreeBuilder::Node TreeBuilder::process(NodeID id, Variable variable) {
if (!variable.isStubbed) {
switch (drgn_type_kind(variable.type)) {
case DRGN_TYPE_POINTER:
if (config.chaseRawPointers) {
if (chaseRawPointers) {
// Pointers to incomplete types are stubbed out
// See OICodeGen::enumeratePointerType
if (th->knownDummyTypeList.contains(variable.type)) {
@ -529,7 +532,7 @@ TreeBuilder::Node TreeBuilder::process(NodeID id, Variable variable) {
break;
}
if (config.genPaddingStats) {
if (genPaddingStats) {
auto entry = paddedStructs->find(node.typeName);
if (entry != paddedStructs->end()) {
entry->second.instancesCnt++;

8
src/TreeBuilder.h
View File

@ -18,11 +18,13 @@
#include <memory>
#include <msgpack/sbuffer_decl.hpp>
#include <optional>
#include <set>
#include <string>
#include <unordered_set>
#include <vector>
#include "Common.h"
#include "Features.h"
// The rocksdb includes are extremely heavy and bloat compile times,
// so we just forward-declare `DB` to avoid making other compile units
@ -39,9 +41,8 @@ class TreeBuilder {
struct Config {
// Don't set default values for the config so the user gets
// an "unitialized field" warning if he missed any.
std::set<ObjectIntrospection::Feature> features;
bool logAllStructs;
bool chaseRawPointers;
bool genPaddingStats;
bool dumpDataSegment;
std::optional<std::string> jsonPath;
};
@ -66,6 +67,9 @@ class TreeBuilder {
const std::vector<uint64_t>* oidData = nullptr;
std::map<std::string, PaddingInfo>* paddedStructs = nullptr;
bool genPaddingStats;
bool chaseRawPointers;
/*
* The RocksDB output needs versioning so they are imported correctly in
* Scuba. Version 1 had no concept of versioning and no header.

17
test/integration/gen_tests.py
View File

@ -270,13 +270,16 @@ def add_oid_integration_test(f, config, case_name, case):
cli_options = (
"{" + ", ".join(f'"{option}"' for option in case.get("cli_options", ())) + "}"
)
config_extra = case.get("config", "")
config_prefix = case.get("config_prefix", "")
config_suffix = case.get("config_suffix", "")
f.write(
f"\n"
f"TEST_F(OidIntegration, {case_str}) {{\n"
f"{generate_skip(case, 'oid')}"
f' std::string configOptions = R"--({config_extra})--";\n'
f' std::string configPrefix = R"--({config_prefix})--";\n'
f' std::string configSuffix = R"--({config_suffix})--";\n'
f" ba::io_context ctx;\n"
f" auto [target, oid] = runOidOnProcess(\n"
f" {{\n"
@ -285,7 +288,7 @@ def add_oid_integration_test(f, config, case_name, case):
f' .scriptSource = "{probe_str}",\n'
f" }},\n"
f" {cli_options},\n"
f" std::move(configOptions));\n"
f" std::move(configPrefix), std::move(configSuffix));\n"
f" ASSERT_EQ(exit_code(oid), {exit_code});\n"
f" EXPECT_EQ(target.proc.running(), true);\n"
)
@ -341,18 +344,20 @@ def add_oil_integration_test(f, config, case_name, case):
if case.get("oil_disable", False):
return
config_extra = case.get("config", "")
config_prefix = case.get("config_prefix", "")
config_suffix = case.get("config_suffix", "")
f.write(
f"\n"
f"TEST_F(OilIntegration, {case_str}) {{\n"
f"{generate_skip(case, 'oil')}"
f' std::string configOptions = R"--({config_extra})--";\n'
f' std::string configPrefix = R"--({config_prefix})--";\n'
f' std::string configSuffix = R"--({config_suffix})--";\n'
f" ba::io_context ctx;\n"
f" auto target = runOilTarget({{\n"
f" .ctx = ctx,\n"
f' .targetArgs = "oil {case_str}",\n'
f" }}, std::move(configOptions));\n\n"
f" }}, std::move(configPrefix), std::move(configSuffix));\n\n"
f" ASSERT_EQ(exit_code(target), {exit_code});\n"
f"\n"
f" bpt::ptree result_json;\n"

2
test/integration/ignored.toml
View File

@ -18,7 +18,7 @@ definitions = '''
"The 3rd member of the struct Bar"
};
"""
config = """
config_suffix = """
[[codegen.ignore]]
type = "Foo"
members = ["a"]

20
test/integration/inheritance_polymorphic.toml
View File

@ -136,3 +136,23 @@ definitions = '''
{"name":"vec_b", "staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4},
{"name":"int_c", "staticSize":4, "dynamicSize":0}
]}]'''
[cases.feature_flag]
oil_skip = "Polymorphic inheritance disabled in OIL"
cli_options = ["-fpolymorphic-inheritance"]
param_types = ["const B&"]
arg_types = ["C"]
setup = '''
C c;
c.vec_b = {1,2,3};
return c;
'''
expect_json = '''[{
"typeName":"C",
"staticSize":48,
"dynamicSize":12,
"members":[
{"staticSize":8, "dynamicSize":0},
{"name":"int_a", "staticSize":4, "dynamicSize":0},
{"name":"vec_b", "staticSize":24, "dynamicSize":12, "length":3, "capacity":3, "elementStaticSize":4},
{"name":"int_c", "staticSize":4, "dynamicSize":0}
]}]'''

44
test/integration/pointers.toml
View File

@ -238,3 +238,47 @@ definitions = '''
{"staticSize":8, "dynamicSize":0, "pointer":0},
{"staticSize":8, "dynamicSize":0, "NOT": {"pointer":0}}
]}]'''
[cases.feature_flag]
oil_disable = "oil can't chase raw pointers safely"
param_types = ["const std::vector<int*>&"]
setup = "return {{new int(1), nullptr, new int(3)}};"
cli_options = ["-fchase-raw-pointers"]
expect_json = '''[{
"staticSize":24,
"dynamicSize":32,
"length":3,
"capacity":3,
"elementStaticSize":8,
"members":[
{"staticSize":8, "dynamicSize":4, "NOT": {"pointer":0}},
{"staticSize":8, "dynamicSize":0, "pointer":0},
{"staticSize":8, "dynamicSize":4, "NOT": {"pointer":0}}
]}]'''
[cases.feature_flag_disabled]
param_types = ["const PrimitivePtrs&"]
setup = "return PrimitivePtrs{0, new int(0), new int(0)};"
cli_options = ["-fchase-raw-pointers", "-Fchase-raw-pointers"]
expect_json = '''[{
"staticSize":24,
"dynamicSize":0,
"members":[
{"name":"a", "staticSize":4, "dynamicSize":0},
{"name":"b", "staticSize":8, "dynamicSize":0},
{"name":"c", "staticSize":8, "dynamicSize":0}
]}]'''
[cases.feature_config]
oil_disable = "oil can't chase raw pointers safely"
param_types = ["const std::vector<int*>&"]
setup = "return {{new int(1), nullptr, new int(3)}};"
config_prefix = 'features = ["chase-raw-pointers"]'
expect_json = '''[{
"staticSize":24,
"dynamicSize":32,
"length":3,
"capacity":3,
"elementStaticSize":8,
"members":[
{"staticSize":8, "dynamicSize":4, "NOT": {"pointer":0}},
{"staticSize":8, "dynamicSize":0, "pointer":0},
{"staticSize":8, "dynamicSize":4, "NOT": {"pointer":0}}
]}]'''

27
test/integration/runner_common.cpp
View File

@ -122,9 +122,10 @@ int IntegrationBase::exit_code(Proc& proc) {
return proc.proc.exit_code();
}
fs::path IntegrationBase::createCustomConfig(const std::string& extraConfig) {
fs::path IntegrationBase::createCustomConfig(const std::string& prefix,
const std::string& suffix) {
// If no extra config provided, return the config path unaltered.
if (extraConfig.empty()) {
if (prefix.empty() && suffix.empty()) {
return configFile;
}
@ -157,9 +158,17 @@ fs::path IntegrationBase::createCustomConfig(const std::string& extraConfig) {
}
std::ofstream customConfig(customConfigFile, std::ios_base::app);
if (!prefix.empty()) {
customConfig << "\n\n# Test custom config start\n\n";
customConfig << prefix;
customConfig << "\n\n# Test custom config end\n\n";
}
customConfig << config;
customConfig << "\n\n# Test custom config\n\n";
customConfig << extraConfig;
if (!suffix.empty()) {
customConfig << "\n\n# Test custom config start\n\n";
customConfig << suffix;
customConfig << "\n\n# Test custom config end\n\n";
}
return customConfigFile;
}
@ -170,7 +179,8 @@ std::string OidIntegration::TmpDirStr() {
OidProc OidIntegration::runOidOnProcess(OidOpts opts,
std::vector<std::string> extra_args,
std::string extra_config) {
std::string configPrefix,
std::string configSuffix) {
// Binary paths are populated by CMake
std::string targetExe =
std::string(TARGET_EXE_PATH) + " " + opts.targetArgs + " 1000";
@ -195,7 +205,7 @@ OidProc OidIntegration::runOidOnProcess(OidOpts opts,
std::ofstream touch(segconfigPath);
}
fs::path thisConfig = createCustomConfig(extra_config);
fs::path thisConfig = createCustomConfig(configPrefix, configSuffix);
// Keep PID as the last argument to make it easier for users to directly copy
// and modify the command from the verbose mode output.
@ -337,8 +347,9 @@ std::string OilIntegration::TmpDirStr() {
return std::string("/tmp/oil-integration-XXXXXX");
}
Proc OilIntegration::runOilTarget(OidOpts opts, std::string extra_config) {
fs::path thisConfig = createCustomConfig(extra_config);
Proc OilIntegration::runOilTarget(OidOpts opts, std::string configPrefix,
std::string configSuffix) {
fs::path thisConfig = createCustomConfig(configPrefix, configSuffix);
std::string targetExe = std::string(TARGET_EXE_PATH) + " " + opts.targetArgs +
" " + thisConfig.string();

8
test/integration/runner_common.h
View File

@ -38,7 +38,8 @@ class IntegrationBase : public ::testing::Test {
void TearDown() override;
void SetUp() override;
int exit_code(Proc& proc);
std::filesystem::path createCustomConfig(const std::string& extra);
std::filesystem::path createCustomConfig(const std::string& prefix,
const std::string& suffix);
std::filesystem::path workingDir;
@ -51,7 +52,7 @@ class OidIntegration : public IntegrationBase {
std::string TmpDirStr() override;
OidProc runOidOnProcess(OidOpts opts, std::vector<std::string> extra_args,
std::string extra_config);
std::string configPrefix, std::string configSuffix);
/*
* compare_json
@ -69,5 +70,6 @@ class OilIntegration : public IntegrationBase {
protected:
std::string TmpDirStr() override;
Proc runOilTarget(OidOpts opts, std::string extra_config);
Proc runOilTarget(OidOpts opts, std::string configPrefix,
std::string configSuffix);
};

21
test/integration/thrift_isset.toml
View File

@ -225,3 +225,24 @@ namespace cpp2 {
{"name":"__fbthrift_field_e", "staticSize":4, "NOT":"isset"},
{"name":"__isset", "staticSize":3}
]}]'''
[cases.feature_flag]
param_types = ["const cpp2::MyThriftStructBoxed&"]
setup = '''
cpp2::MyThriftStructBoxed ret;
ret.d_ref() = 1;
ret.e_ref() = 1;
return ret;
'''
cli_options = ["-fcapture-thrift-isset"]
expect_json = '''[{
"staticSize":32,
"dynamicSize":0,
"members":[
{"name":"__fbthrift_field_a", "staticSize":8, "NOT":"isset"},
{"name":"__fbthrift_field_b", "staticSize":8, "NOT":"isset"},
{"name":"__fbthrift_field_c", "staticSize":4, "isset":false},
{"name":"__fbthrift_field_d", "staticSize":4, "isset":true},
{"name":"__fbthrift_field_e", "staticSize":4, "isset":true},
{"name":"__isset", "staticSize":3}
]}]'''

17
tools/OITB.cpp
View File

@ -30,6 +30,8 @@
namespace fs = std::filesystem;
using namespace ObjectIntrospection;
constexpr static OIOpts opts{
OIOpt{'h', "help", no_argument, nullptr, "Print this message and exit"},
OIOpt{'a', "log-all-structs", no_argument, nullptr,
@ -118,8 +120,10 @@ static std::ostream&
operator<<(std::ostream& out, TreeBuilder::Config tbc) {
out << "TreeBuilde::Config = [";
out << "\n logAllStructs = " << tbc.logAllStructs;
out << "\n chaseRawPointers = " << tbc.chaseRawPointers;
out << "\n genPaddingStats = " << tbc.genPaddingStats;
out << "\n chaseRawPointers = "
<< tbc.features.contains(Feature::ChaseRawPointers);
out << "\n genPaddingStats = "
<< tbc.features.contains(Feature::GenPaddingStats);
out << "\n dumpDataSegment = " << tbc.dumpDataSegment;
out << "\n jsonPath = " << (tbc.jsonPath ? *tbc.jsonPath : "NONE");
out << "\n]\n";
@ -134,9 +138,8 @@ int main(int argc, char* argv[]) {
/* Reflects `oid`'s defaults for TreeBuilder::Config */
TreeBuilder::Config tbConfig{
.features = {Feature::PackStructs, Feature::GenPaddingStats},
.logAllStructs = true,
.chaseRawPointers = false,
.genPaddingStats = true,
.dumpDataSegment = false,
.jsonPath = std::nullopt,
};
@ -154,10 +157,10 @@ int main(int argc, char* argv[]) {
true; // Weird that we're setting it to true, again...
break;
case 'n':
tbConfig.chaseRawPointers = true;
tbConfig.features.insert(Feature::ChaseRawPointers);
break;
case 'w':
tbConfig.genPaddingStats = false;
tbConfig.features.erase(Feature::GenPaddingStats);
break;
case 'J':
tbConfig.jsonPath = optarg ? optarg : "oid_out.json";
@ -186,7 +189,7 @@ int main(int argc, char* argv[]) {
TreeBuilder typeTree(tbConfig);
if (tbConfig.genPaddingStats) {
if (tbConfig.features.contains(Feature::GenPaddingStats)) {
LOG(INFO) << "Setting-up PaddingHunter...";
typeTree.setPaddedStructs(&paddingInfos);
}