JakeHillion/object-introspection
1
0
Fork 0
mirror of https://github.com/JakeHillion/object-introspection.git synced 2024-09-19 11:09:05 +01:00
Code Issues Packages Projects Releases Wiki Activity

improve caching infrastructure

Browse Source
This commit is contained in:
Jake Hillion 2023-11-15 06:47:13 -08:00
parent f9cb0115e1
commit 4174f62cf8
28 changed files with 909 additions and 575 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CMakeLists.txt
View File

@ -272,7 +272,7 @@ ADD_FLEX_BISON_DEPENDENCY(Lexer Parser)
add_library(oid_parser STATIC ${BISON_Parser_OUTPUTS} ${FLEX_Lexer_OUTPUTS})
target_compile_options(oid_parser PRIVATE "-w") # Disable warnings for generated code
target_link_libraries(oid_parser glog::glog)
target_link_libraries(oid_parser glog::glog folly_headers)
### Core OI
@ -283,7 +283,6 @@ add_library(oicore
oi/Config.cpp
oi/Descs.cpp
oi/Metrics.cpp
oi/OICache.cpp
oi/OICompiler.cpp
oi/PaddingHunter.cpp
oi/Serialize.cpp
@ -295,7 +294,9 @@ target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
llvm_map_components_to_libnames(llvm_libs core native mcjit x86disassembler)
target_link_libraries(oicore
cache
codegen
memory_file
${Boost_LIBRARIES}
Boost::headers

17
oi/CMakeLists.txt
View File

@ -21,11 +21,17 @@ target_link_libraries(symbol_service
Boost::headers
${Boost_LIBRARIES}
folly_headers
glog::glog
dw
)
add_library(memory_file
support/MemoryFile.cpp
)
target_link_libraries(memory_file glog::glog Boost::headers)
add_library(features Features.cpp)
target_link_libraries(features glog::glog)
@ -55,6 +61,17 @@ target_link_libraries(codegen
glog::glog
)
add_library(cache
Cache.cpp
Serialize.cpp
cache/Entity.cpp
cache/LocalCache.cpp
)
target_link_libraries(cache
Boost::serialization
folly_headers
)
add_library(exporters_json exporters/Json.cpp)
target_include_directories(exporters_json PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_link_libraries(exporters_json oil)

55
oi/Cache.cpp Normal file
View File

@ -0,0 +1,55 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "Cache.h"
namespace oi::detail {
using cache::Builder;
using cache::Entity;
using cache::Source;
void Cache::registerSource(std::unique_ptr<cache::Source> s) {
sources.emplace_back(std::move(s));
}
void Cache::registerBuilder(std::unique_ptr<cache::Builder> b) {
builders.emplace_back(std::move(b));
}
bool Cache::read(const Source::Request& req, Entity& en) {
for (auto& s : sources) {
if (s->read(req, en))
return true;
}
return false;
}
bool Cache::write(const Source::Request& req, const Entity& en) {
for (auto& s : sources) {
if (s->write(req, en))
return true;
}
return false;
}
Builder::Response Cache::build(const Builder::Request& req) {
for (auto& b : builders) {
if (auto ret = b->build(req); ret != Builder::Response::Failure)
return ret;
}
return Builder::Response::Failure;
}
} // namespace oi::detail

43
oi/Cache.h Normal file
View File

@ -0,0 +1,43 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <filesystem>
#include <memory>
#include <optional>
#include <vector>
#include "oi/cache/Builder.h"
#include "oi/cache/Source.h"
namespace oi::detail {
class Cache {
public:
void registerSource(std::unique_ptr<cache::Source>);
void registerBuilder(std::unique_ptr<cache::Builder>);
bool read(const cache::Source::Request&, cache::Entity&);
bool write(const cache::Source::Request&, const cache::Entity&);
cache::Builder::Response build(const cache::Builder::Request&);
private:
std::vector<std::unique_ptr<cache::Source>> sources;
std::vector<std::unique_ptr<cache::Builder>> builders;
};
} // namespace oi::detail

13
oi/EnumBitset.h
View File

@ -61,6 +61,8 @@ class EnumBitset {
private:
BitsetType bitset;
friend struct std::hash<EnumBitset<T, N>>;
};
template <typename T, size_t N>
@ -70,3 +72,14 @@ EnumBitset<T, N> operator&(const EnumBitset<T, N>& lhs,
out &= rhs;
return out;
}
namespace std {
template <typename T, size_t N>
struct hash<EnumBitset<T, N>> {
size_t operator()(const EnumBitset<T, N>& bs) const {
return hash<bitset<N>>{}(bs.bitset);
}
};
} // namespace std

257
oi/OICache.cpp
View File

@ -1,257 +0,0 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "oi/OICache.h"
#include <glog/logging.h>
#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <fstream>
#include "oi/Descs.h"
#include "oi/OICodeGen.h"
#include "oi/Portability.h"
#include "oi/Serialize.h"
#if OI_PORTABILITY_META_INTERNAL()
#include "object-introspection/internal/GobsService.h"
#include "object-introspection/internal/ManifoldCache.h"
#endif
namespace oi::detail {
static std::optional<std::reference_wrapper<const std::string>> getEntName(
SymbolService& symbols, const irequest& req, OICache::Entity ent) {
if (ent == OICache::Entity::FuncDescs ||
ent == OICache::Entity::GlobalDescs) {
return req.func;
} else {
if (req.type == "global") {
const auto& globalDesc = symbols.findGlobalDesc(req.func);
if (!globalDesc) {
return std::nullopt;
}
return globalDesc->typeName;
} else {
const auto& funcDesc = symbols.findFuncDesc(req);
if (!funcDesc) {
return std::nullopt;
}
const auto& arg = funcDesc->getArgument(req.arg);
if (!arg) {
return std::nullopt;
}
return arg->typeName;
}
}
}
std::optional<fs::path> OICache::getPath(const irequest& req,
Entity ent) const {
auto hash = [](const std::string& str) {
return std::to_string(std::hash<std::string>{}(str));
};
auto ext = extensions[static_cast<size_t>(ent)];
const auto& entName = getEntName(*symbols, req, ent);
if (!entName.has_value()) {
return std::nullopt;
}
return basePath / (hash(*entName) + ext);
}
template <typename T>
bool OICache::load(const irequest& req, Entity ent, T& data) {
if (!isEnabled())
return false;
try {
auto buildID = symbols->locateBuildID();
if (!buildID) {
LOG(ERROR) << "Failed to locate buildID";
return false;
}
auto cachePath = getPath(req, ent);
if (!cachePath.has_value()) {
LOG(ERROR) << "Failed to get cache path for " << req.type << ':'
<< req.func << ':' << req.arg << '/'
<< static_cast<size_t>(ent);
return false;
}
LOG(INFO) << "Loading cache " << *cachePath;
std::ifstream ifs(*cachePath);
boost::archive::text_iarchive ia(ifs);
std::string cacheBuildId;
ia >> cacheBuildId;
if (cacheBuildId != *buildID) {
LOG(ERROR) << "The cache's build id '" << cacheBuildId
<< "' doesn't match the target's build id '" << *buildID
<< "'";
return false;
}
ia >> data;
return true;
} catch (const std::exception& e) {
LOG(WARNING) << "Failed to load from cache: " << e.what();
return false;
}
}
template <typename T>
bool OICache::store(const irequest& req, Entity ent, const T& data) {
if (!isEnabled())
return false;
try {
auto buildID = symbols->locateBuildID();
if (!buildID) {
LOG(ERROR) << "Failed to locate buildID";
return false;
}
auto cachePath = getPath(req, ent);
if (!cachePath.has_value()) {
LOG(ERROR) << "Failed to get cache path for " << req.type << ':'
<< req.func << ':' << req.arg << '/'
<< static_cast<size_t>(ent);
return false;
}
LOG(INFO) << "Storing cache " << *cachePath;
std::ofstream ofs(*cachePath);
boost::archive::text_oarchive oa(ofs);
oa << *buildID;
oa << data;
return true;
} catch (const std::exception& e) {
LOG(WARNING) << "Failed to write to cache: " << e.what();
return false;
}
}
#define INSTANTIATE_ARCHIVE(...) \
template bool OICache::load(const irequest&, Entity, __VA_ARGS__&); \
template bool OICache::store(const irequest&, Entity, const __VA_ARGS__&);
INSTANTIATE_ARCHIVE(std::pair<RootInfo, TypeHierarchy>)
INSTANTIATE_ARCHIVE(std::unordered_map<std::string, std::shared_ptr<FuncDesc>>)
INSTANTIATE_ARCHIVE(
std::unordered_map<std::string, std::shared_ptr<GlobalDesc>>)
INSTANTIATE_ARCHIVE(std::map<std::string, PaddingInfo>)
#undef INSTANTIATE_ARCHIVE
// Upload all contents of cache for this request
bool OICache::upload([[maybe_unused]] const irequest& req) {
#if OI_PORTABILITY_META_INTERNAL()
if (!isEnabled() || downloadedRemote || !enableUpload)
return true;
std::vector<std::filesystem::path> files;
for (size_t i = 0; i < static_cast<size_t>(OICache::Entity::MAX); i++) {
auto cachePath = getPath(req, static_cast<OICache::Entity>(i));
if (!cachePath.has_value()) {
LOG(ERROR) << "Failed to get cache path for " << req.type << ':'
<< req.func << ':' << req.arg << '/' << static_cast<size_t>(i);
return false;
}
files.push_back(*cachePath);
}
auto hash = generateRemoteHash(req);
if (hash.empty()) {
LOG(ERROR) << "failed to generate remote lookup hash";
return false;
}
return ObjectIntrospection::ManifoldCache::upload(hash, files);
#else
if (isEnabled() && !downloadedRemote && enableUpload) {
// We tried to download when support is not enabled!
LOG(ERROR) << "Tried to upload artifacts when support is not enabled!";
return false;
}
return true;
#endif
}
// Try to fetch contents of cache
bool OICache::download([[maybe_unused]] const irequest& req) {
#if OI_PORTABILITY_META_INTERNAL()
if (!isEnabled() || !enableDownload)
return true;
auto hash = generateRemoteHash(req);
if (hash.empty()) {
LOG(ERROR) << "failed to generate remote lookup hash";
return false;
}
if (basePath.filename() != hash) {
// Use a subdirectory per hash shard to avoid conflicts
basePath /= hash;
if (fs::exists(basePath.parent_path()) && !fs::exists(basePath))
fs::create_directory(basePath);
}
if (ObjectIntrospection::ManifoldCache::download(hash, basePath)) {
downloadedRemote = true;
return true;
}
if (abortOnLoadFail) {
LOG(ERROR) << "We weren't able to pull artifacts when requested, "
"aborting run!";
// If we aren't uploading, quit early as we requested a download and
// weren't able to get it.
return false;
}
return true;
#else
if (isEnabled() && enableDownload) {
// We tried to download when support is not enabled!
LOG(ERROR) << "Tried to download artifacts when support is not enabled!";
return false;
}
return true;
#endif
}
std::string OICache::generateRemoteHash(const irequest& req) {
auto buildID = symbols->locateBuildID();
if (!buildID) {
LOG(ERROR) << "Failed to locate buildID";
return "";
}
std::string remote_cache_id = *buildID + "/" + req.func + "/" + req.arg +
"/" + generatorConfig.toString();
#if OI_PORTABILITY_META_INTERNAL()
auto version_pair = ObjectIntrospection::GobsService::getOidRpmVersions();
remote_cache_id += "/" + version_pair.first + "/" + version_pair.second;
#endif
LOG(INFO) << "generating remote hash from: " << remote_cache_id;
return std::to_string(std::hash<std::string>{}(remote_cache_id));
}
} // namespace oi::detail

76
oi/OICache.h
View File

@ -1,76 +0,0 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <array>
#include <filesystem>
#include <memory>
#include <optional>
#include "oi/OICodeGen.h"
#include "oi/OIParser.h"
#include "oi/SymbolService.h"
namespace oi::detail {
class OICache {
public:
OICache(const OICodeGen::Config& generatorConfig)
: generatorConfig(generatorConfig) {
}
std::filesystem::path basePath{};
std::shared_ptr<SymbolService> symbols{};
bool downloadedRemote = false;
bool enableUpload = false;
bool enableDownload = false;
bool abortOnLoadFail = false;
// We need the generator config to download the cache
// with the matching configuration.
const OICodeGen::Config& generatorConfig;
// Entity is used to index the `extensions` array
// So we must keep the Entity enum and `extensions` array in sync!
enum class Entity {
Source,
Object,
FuncDescs,
GlobalDescs,
TypeHierarchy,
PaddingInfo,
MAX
};
static constexpr std::array<const char*, static_cast<size_t>(Entity::MAX)>
extensions{".cc", ".o", ".fd", ".gd", ".th", ".pd"};
bool isEnabled() const {
return !basePath.empty();
}
std::optional<std::filesystem::path> getPath(const irequest&, Entity) const;
template <typename T>
bool store(const irequest&, Entity, const T&);
template <typename T>
bool load(const irequest&, Entity, T&);
bool upload(const irequest& req);
bool download(const irequest& req);
private:
std::string generateRemoteHash(const irequest&);
};
} // namespace oi::detail

30
oi/OICompiler.cpp
View File

@ -41,9 +41,13 @@
#include <array>
#include <boost/range/combine.hpp>
#include <boost/scope_exit.hpp>
#include <fstream>
#include <iterator>
#include <span>
#include "oi/Headers.h"
#include "oi/Metrics.h"
#include "oi/support/MemoryFile.h"
extern "C" {
#include <llvm-c/Disassembler.h>
@ -474,7 +478,7 @@ static void debugDisAsm(
bool OICompiler::compile(const std::string& code,
const fs::path& sourcePath,
const fs::path& objectPath) {
std::vector<uint8_t>& objectBuf) {
metrics::Tracing _("compile");
/*
@ -507,7 +511,8 @@ bool OICompiler::compile(const std::string& code,
compInv->getFrontendOpts().Inputs.push_back(
FrontendInputFile(sourcePath.string(), InputKind{Language::CXX}));
compInv->getFrontendOpts().OutputFile = objectPath.string();
MemoryFile fileBuf{"out.o"};
compInv->getFrontendOpts().OutputFile = fileBuf.path().string();
compInv->getFrontendOpts().ProgramAction = clang::frontend::EmitObj;
auto& headerSearchOptions = compInv->getHeaderSearchOpts();
@ -561,7 +566,6 @@ bool OICompiler::compile(const std::string& code,
}
}
compInv->getFrontendOpts().OutputFile = objectPath;
compInv->getTargetOpts().Triple =
llvm::Triple::normalize(llvm::sys::getProcessTriple());
if (config.usePIC) {
@ -614,12 +618,15 @@ bool OICompiler::compile(const std::string& code,
}
*/
std::ifstream ifs{fileBuf.path().string(), std::ios::binary};
objectBuf = std::vector<uint8_t>(std::istreambuf_iterator<char>(ifs),
std::istreambuf_iterator<char>());
return true;
}
std::optional<OICompiler::RelocResult> OICompiler::applyRelocs(
uintptr_t baseRelocAddress,
const std::set<fs::path>& objectFiles,
const std::vector<std::vector<uint8_t>>& objectFiles,
const std::unordered_map<std::string, uintptr_t>& syntheticSymbols) {
metrics::Tracing relocationTracing("relocation");
@ -627,16 +634,19 @@ std::optional<OICompiler::RelocResult> OICompiler::applyRelocs(
RuntimeDyld dyld(*memMgr, *memMgr);
/* Load all the object files into the MemoryManager */
for (const auto& objPath : objectFiles) {
VLOG(1) << "Loading object file " << objPath;
auto objFile = ObjectFile::createObjectFile(objPath.c_str());
for (const auto& objContent : objectFiles) {
VLOG(1) << "Loading object content from memory";
llvm::MemoryBufferRef objBuf{
llvm::StringRef((const char*)objContent.data(), objContent.size()),
"test.o"};
auto objFile = ObjectFile::createObjectFile(objBuf);
if (!objFile) {
raw_os_ostream(LOG(ERROR)) << "Failed to load object file " << objPath
<< ": " << objFile.takeError();
raw_os_ostream(LOG(ERROR))
<< "Failed to load object file:" << objFile.takeError();
return std::nullopt;
}
dyld.loadObject(*objFile->getBinary());
dyld.loadObject(**objFile);
if (dyld.hasError()) {
LOG(ERROR) << "load object failed: " << dyld.getErrorString().data();
return std::nullopt;

8
oi/OICompiler.h
View File

@ -132,12 +132,12 @@ class OICompiler {
* Compile the given @param code and write the result in @param objectPath.
*
* @param code the C++ source code to compile
* @param sourcePath path/name of the code to compile (not used)
* @param objectPath path where to write the resulting Object file
* @param sourcePath path/name of the code to compile for debug info.
* @param objectBuf memory location to store the resultant code.
*
* @return true if the compilation succeeded, false otherwise.
*/
bool compile(const std::string&, const fs::path&, const fs::path&);
bool compile(const std::string&, const fs::path&, std::vector<uint8_t>&);
/**
* Load the @param objectFiles in memory and apply relocation at
@ -158,7 +158,7 @@ class OICompiler {
*/
std::optional<RelocResult> applyRelocs(
uintptr_t,
const std::set<fs::path>&,
const std::vector<std::vector<uint8_t>>&,
const std::unordered_map<std::string, uintptr_t>&);
/**

54
oi/OID.cpp
View File

@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <folly/init/Init.h>
#include <gflags/gflags.h>
#include <glog/logging.h>
@ -39,10 +40,9 @@ extern "C" {
#include "oi/Portability.h"
#include "oi/TimeUtils.h"
#include "oi/TreeBuilder.h"
#if OI_PORTABILITY_META_INTERNAL()
#include <folly/init/Init.h>
#endif
#include "oi/cache/GobsBuilder.h"
#include "oi/cache/LocalCache.h"
#include "oi/cache/ManifoldCache.h"
namespace oi::detail {
@ -334,15 +334,21 @@ static ExitStatus::ExitStatus runScript(
}
weak_oid = oid; // set the weak_ptr for signal handlers
Cache cache;
if (!oidConfig.cacheBasePath.empty()) {
oid->setCacheBasePath(oidConfig.cacheBasePath);
cache.registerSource(
std::make_unique<cache::LocalCache>(oidConfig.cacheBasePath));
}
if constexpr (kIsMetaInternal) {
auto manifold = std::make_unique<cache::ManifoldCache>();
manifold->setReadable(oidConfig.remoteCacheDownload);
manifold->setWriteable(oidConfig.remoteCacheUpload);
cache.registerSource(std::move(manifold));
oid->setCacheRemoteEnabled(oidConfig.cacheRemoteUpload,
oidConfig.cacheRemoteDownload);
if (!oid->validateCache()) {
return ExitStatus::UsageError;
cache.registerBuilder(std::make_unique<cache::GobsBuilder>());
}
oid->setCache(std::move(cache));
oid->setCustomCodeFile(oidConfig.customCodeFile);
oid->setHardDisableDrgn(oidConfig.hardDisableDrgn);
oid->setStrict(oidConfig.strict);
@ -404,8 +410,7 @@ static ExitStatus::ExitStatus runScript(
}
VLOG(1) << "init took " << std::dec << time_ns(time_hr::now() - initStart)
<< " nsecs\n"
<< "Compilation Started";
<< " nsecs\n";
auto compileStart = time_hr::now();
@ -489,11 +494,11 @@ static ExitStatus::ExitStatus runScript(
}
}
// Upload cache artifacts if present
if (!oid->uploadCache()) {
LOG(ERROR) << "cache upload requested and failed";
return ExitStatus::CacheUploadError;
}
// TODO: Upload cache artifacts if present
// if (!oid->uploadCache()) {
// LOG(ERROR) << "cache upload requested and failed";
// return ExitStatus::CacheUploadError;
// }
std::cout << "SUCCESS " << fileName << std::endl;
VLOG(1) << "Entire process took " << time_ns(time_hr::now() - progStart)
@ -524,14 +529,14 @@ int main(int argc, char* argv[]) {
metrics::Tracing _("main");
#if OI_PORTABILITY_META_INTERNAL()
folly::InitOptions init;
init.useGFlags(false);
init.removeFlags(false);
folly::init(&argc, &argv, init);
#else
google::InitGoogleLogging(argv[0]);
#endif
if constexpr (kIsMetaInternal) {
folly::InitOptions init;
init.useGFlags(false);
init.removeFlags(false);
folly::init(&argc, &argv, init);
} else {
google::InitGoogleLogging(argv[0]);
}
google::SetStderrLogging(google::WARNING);
int c = 0;
@ -555,7 +560,6 @@ int main(int argc, char* argv[]) {
// change default settings for prod
oidConfig.hardDisableDrgn = true;
oidConfig.cacheRemoteDownload = true;
oidConfig.cacheBasePath = "/tmp/oid-cache";
features[Feature::ChaseRawPointers] = true;
} else if (strcmp("strict", optarg) == 0) {
oidConfig.strict = true;

180
oi/OIDebugger.cpp
View File

@ -44,6 +44,7 @@ extern "C" {
#include <glog/logging.h>
#include "oi/Cache.h"
#include "oi/CodeGen.h"
#include "oi/Config.h"
#include "oi/ContainerInfo.h"
@ -1886,8 +1887,7 @@ bool OIDebugger::removeTrap(pid_t pid, const trapInfo& t) {
OIDebugger::OIDebugger(const OICodeGen::Config& genConfig,
OICompiler::Config ccConfig,
TreeBuilder::Config tbConfig)
: cache{genConfig},
compilerConfig{std::move(ccConfig)},
: compilerConfig{std::move(ccConfig)},
generatorConfig{genConfig},
treeBuilderConfig{std::move(tbConfig)} {
debug = true;
@ -1904,7 +1904,6 @@ OIDebugger::OIDebugger(pid_t pid,
symbols = std::make_shared<SymbolService>(traceePid);
setDataSegmentSize(dataSegSize);
createSegmentConfigFile();
cache.symbols = symbols;
}
OIDebugger::OIDebugger(fs::path debugInfo,
@ -1913,7 +1912,6 @@ OIDebugger::OIDebugger(fs::path debugInfo,
TreeBuilder::Config tbConfig)
: OIDebugger(genConfig, std::move(ccConfig), std::move(tbConfig)) {
symbols = std::make_shared<SymbolService>(std::move(debugInfo));
cache.symbols = symbols;
}
/*
@ -2163,8 +2161,8 @@ bool OIDebugger::compileCode() {
assert(pdata.numReqs() == 1);
const auto& preq = pdata.getReq();
OICompiler compiler{symbols, compilerConfig};
std::set<fs::path> objectFiles{};
std::optional<OICompiler> compiler;
std::vector<std::vector<uint8_t>> objectContents;
/*
* Global probes don't have multiple arguments, but calling `getReqForArg(X)`
@ -2176,106 +2174,89 @@ bool OIDebugger::compileCode() {
for (size_t i = 0; i < argCount; i++) {
const auto& req = preq.getReqForArg(i);
if (cache.isEnabled()) {
// try to download cache artifacts if present
if (!downloadCache()) {
#if OI_PORTABILITY_META_INTERNAL()
// Send a request to the GOBS service
char buf[PATH_MAX];
const std::string procpath =
"/proc/" + std::to_string(traceePid) + "/exe";
size_t buf_size;
if ((buf_size = readlink(procpath.c_str(), buf, PATH_MAX)) == -1) {
LOG(ERROR)
<< "Failed to get binary path for tracee, could not call GOBS: "
<< strerror(errno);
} else {
LOG(INFO) << "Attempting to get cache request from gobs";
ObjectIntrospection::GobsService::requestCache(
procpath,
std::string(buf, buf_size),
req.toString(),
generatorConfig.toOptions());
}
#endif
LOG(ERROR) << "No cache file found, exiting!";
return false;
}
cache::Source::Request cacheReq{
.features = generatorConfig.features,
.probe = req,
.traceePid = traceePid ? traceePid : -1,
.buildId = symbols->locateBuildID(),
};
if (req.type == "global") {
decltype(symbols->globalDescs) gds;
if (cache.load(req, OICache::Entity::GlobalDescs, gds)) {
symbols->globalDescs.merge(std::move(gds));
}
} else {
decltype(symbols->funcDescs) fds;
if (cache.load(req, OICache::Entity::FuncDescs, fds)) {
symbols->funcDescs.merge(std::move(fds));
}
cache::ObjectCode object;
cache::TypeHierarchy th;
cache::PaddingInfo pad;
bool skipCodeGen = cache_.read(cacheReq, object) &&
cache_.read(cacheReq, th) && cache_.read(cacheReq, pad);
// Attempt to execute on a remote builder before building locally
if (!skipCodeGen) {
cache::Builder::Request builderReq{
.location = cacheReq,
};
switch (cache_.build(builderReq)) {
case cache::Builder::Response::Success:
LOG(INFO) << "remote builder success";
skipCodeGen = cache_.read(cacheReq, object) &&
cache_.read(cacheReq, th) && cache_.read(cacheReq, pad);
break;
case cache::Builder::Response::Processing:
LOG(INFO) << "remote builder processing, please come back later";
return false;
case cache::Builder::Response::Failure:
break; // proceed to local generation
}
}
auto sourcePath = cache.getPath(req, OICache::Entity::Source);
auto objectPath = cache.getPath(req, OICache::Entity::Object);
auto typeHierarchyPath = cache.getPath(req, OICache::Entity::TypeHierarchy);
auto paddingInfoPath = cache.getPath(req, OICache::Entity::PaddingInfo);
if (skipCodeGen) {
LOG(INFO)
<< "Successfully loaded from the cache, skipping code generation.";
typeInfos.emplace(req, std::make_tuple(th.root, th.th, pad.info));
objectContents.emplace_back(std::move(object.code));
continue;
}
if (!sourcePath || !objectPath || !typeHierarchyPath || !paddingInfoPath) {
LOG(ERROR) << "Failed to get all cache paths, aborting!";
// 1. Code generate
// 2. Compile
// 3. Save into cache
VLOG(2) << "Compiling probe for '" << req.arg;
auto genCode = generateCode(req);
if (!genCode.has_value()) {
LOG(ERROR) << "Failed to generate code";
return false;
}
cache::SourceCode code{std::move(*genCode)};
if (!compiler.has_value())
compiler.emplace(symbols, compilerConfig);
fs::path sourcePath{};
if (!compiler->compile(code.code, sourcePath, object.code)) {
LOG(ERROR) << "Failed to compile code";
return false;
}
bool skipCodeGen = cache.isEnabled() && fs::exists(*objectPath) &&
fs::exists(*typeHierarchyPath) &&
fs::exists(*paddingInfoPath);
if (skipCodeGen) {
std::pair<RootInfo, TypeHierarchy> th;
skipCodeGen =
skipCodeGen && cache.load(req, OICache::Entity::TypeHierarchy, th);
bool cacheSuccess = true;
cacheSuccess &= cache_.write(cacheReq, code);
cacheSuccess &= cache_.write(cacheReq, object);
std::map<std::string, PaddingInfo> pad;
skipCodeGen =
skipCodeGen && cache.load(req, OICache::Entity::PaddingInfo, pad);
if (skipCodeGen) {
typeInfos.emplace(req, std::make_tuple(th.first, th.second, pad));
}
if (req.type == "global") {
cacheSuccess &=
cache_.write(cacheReq, cache::GlobalDescs(symbols->globalDescs));
} else {
cacheSuccess &=
cache_.write(cacheReq, cache::FuncDescs(symbols->funcDescs));
}
if (!skipCodeGen) {
VLOG(2) << "Compiling probe for '" << req.arg
<< "' into: " << *objectPath;
const auto& [rootType, typeHierarchy, paddingInfo] = typeInfos.at(req);
cacheSuccess &=
cache_.write(cacheReq, cache::TypeHierarchy(rootType, typeHierarchy));
cacheSuccess &= cache_.write(cacheReq, cache::PaddingInfo(paddingInfo));
auto code = generateCode(req);
if (!code.has_value()) {
LOG(ERROR) << "Failed to generate code";
return false;
}
if (!cacheSuccess)
LOG(ERROR) << "Failed to write all objects to cache";
bool doCompile = !cache.isEnabled() || !fs::exists(*objectPath);
if (doCompile) {
if (!compiler.compile(*code, *sourcePath, *objectPath)) {
LOG(ERROR) << "Failed to compile code";
return false;
}
}
}
if (cache.isEnabled() && !skipCodeGen) {
if (req.type == "global") {
cache.store(req, OICache::Entity::GlobalDescs, symbols->globalDescs);
} else {
cache.store(req, OICache::Entity::FuncDescs, symbols->funcDescs);
}
const auto& [rootType, typeHierarchy, paddingInfo] = typeInfos.at(req);
cache.store(req,
OICache::Entity::TypeHierarchy,
std::make_pair(rootType, typeHierarchy));
cache.store(req, OICache::Entity::PaddingInfo, paddingInfo);
}
objectFiles.insert(*objectPath);
objectContents.emplace_back(std::move(object.code));
}
if (traceePid) { // we attach to a process
@ -2287,11 +2268,10 @@ bool OIDebugger::compileCode() {
};
VLOG(2) << "Relocating...";
for (const auto& o : objectFiles) {
VLOG(2) << " * " << o;
}
auto relocRes = compiler.applyRelocs(
segConfig.jitCodeStart, objectFiles, syntheticSymbols);
if (!compiler.has_value())
compiler.emplace(symbols, compilerConfig);
auto relocRes = compiler->applyRelocs(segConfig.jitCodeStart,
objectContents, syntheticSymbols);
if (!relocRes.has_value()) {
LOG(ERROR) << "Failed to relocate object code";
return false;
@ -2950,10 +2930,6 @@ std::optional<std::string> OIDebugger::generateCode(const irequest& req) {
codegen2.codegenFromDrgn(root->type.type, code);
}
if (auto sourcePath = cache.getPath(req, OICache::Entity::Source)) {
std::ofstream(*sourcePath) << code;
}
if (!customCodeFile.empty()) {
auto ifs = std::ifstream(customCodeFile);
code.assign(std::istreambuf_iterator<char>(ifs),

53
oi/OIDebugger.h
View File

@ -20,7 +20,7 @@
#include <filesystem>
#include <fstream>
#include "oi/OICache.h"
#include "oi/Cache.h"
#include "oi/OICodeGen.h"
#include "oi/OICompiler.h"
#include "oi/OIParser.h"
@ -77,60 +77,15 @@ class OIDebugger {
return oidShouldExit;
};
void setCacheBasePath(std::filesystem::path basePath) {
if (std::filesystem::exists(basePath.parent_path()) &&
!std::filesystem::exists(basePath)) {
// Create cachedir if parent directory exists
// TODO if returning false here, throw an error
std::filesystem::create_directory(basePath);
}
cache.basePath = std::move(basePath);
}
void setCacheRemoteEnabled(bool upload, bool download) {
cache.enableUpload = upload;
cache.enableDownload = download;
cache.abortOnLoadFail = download && !upload;
}
bool validateCache() {
if ((cache.enableUpload || cache.enableDownload) && !cache.isEnabled()) {
LOG(ERROR) << "Cache download/upload option specified when cache is "
"disabled - aborting!";
return false;
}
return true;
}
void setHardDisableDrgn(bool val) {
symbols->setHardDisableDrgn(val);
}
void setStrict(bool val) {
treeBuilderConfig.strict = val;
}
bool uploadCache() {
return std::all_of(
std::begin(pdata), std::end(pdata), [this](const auto& req) {
return std::all_of(
std::begin(req.args),
std::end(req.args),
[this, &req](const auto& arg) {
return cache.upload(irequest{req.type, req.func, arg});
});
});
void setCache(Cache cache) {
cache_ = std::move(cache);
}
bool downloadCache() {
return std::all_of(
std::begin(pdata), std::end(pdata), [this](const auto& req) {
return std::all_of(
std::begin(req.args),
std::end(req.args),
[this, &req](const auto& arg) {
return cache.download(irequest{req.type, req.func, arg});
});
});
};
std::pair<RootInfo, TypeHierarchy> getTreeBuilderTyping() {
assert(pdata.numReqs() == 1);
@ -179,7 +134,7 @@ class OIDebugger {
const int replayInstSize = 512;
bool trapsRemoved{false};
std::shared_ptr<SymbolService> symbols;
OICache cache;
Cache cache_;
/*
* Map address of valid INT3 instruction to metadata for that interrupt.

10
oi/OIGenerator.cpp
View File

@ -153,14 +153,18 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig,
OICompiler compiler{{}, compilerConfig};
std::vector<uint8_t> objectCode;
if (!compiler.compile(code, sourcePath, objectCode)) {
return {};
}
// TODO: Revert to outputPath and remove printing when typegraph is done.
fs::path tmpObject = outputPath;
tmpObject.replace_extension(
"." + std::to_string(std::hash<std::string>{}(linkageName)) + ".o");
std::ofstream objectFile{tmpObject, std::ios::out | std::ios::binary};
objectFile.write(reinterpret_cast<const char*>(objectCode.data()),
objectCode.size());
if (!compiler.compile(code, sourcePath, tmpObject)) {
return {};
}
return tmpObject;
}

24
oi/OILibraryImpl.cpp
View File

@ -61,24 +61,6 @@ OILibraryImpl::LocalTextSegment::~LocalTextSegment() {
<< "segment unmap failed";
}
OILibraryImpl::MemoryFile::MemoryFile(const char* name) {
fd_ = memfd_create(name, 0);
if (fd_ == -1)
throw std::runtime_error(std::string("memfd creation failed: ") +
std::strerror(errno));
}
OILibraryImpl::MemoryFile::~MemoryFile() {
if (fd_ == -1)
return;
PLOG_IF(ERROR, close(fd_) == -1) << "memfd close failed";
}
std::filesystem::path OILibraryImpl::MemoryFile::path() {
return {(boost::format("/dev/fd/%1%") % fd_).str()};
}
OILibraryImpl::OILibraryImpl(void* atomicHole,
std::unordered_set<oi::Feature> fs,
GeneratorOptions opts)
@ -132,13 +114,13 @@ std::pair<void*, const exporters::inst::Inst&> OILibraryImpl::compileCode() {
outputFile << code;
}
auto object = MemoryFile("oil_object_code");
OICompiler compiler{symbols, compilerConfig_};
if (!compiler.compile(code, sourcePath, object.path()))
std::vector<uint8_t> object;
if (!compiler.compile(code, sourcePath, object))
throw std::runtime_error("oil jit compilation failed!");
auto relocRes = compiler.applyRelocs(
reinterpret_cast<uint64_t>(textSeg.data().data()), {object.path()}, {});
reinterpret_cast<uint64_t>(textSeg.data().data()), {object}, {});
if (!relocRes)
throw std::runtime_error("oil jit relocation failed!");

6
oi/OIParser.h
View File

@ -14,6 +14,9 @@
* limitations under the License.
*/
#pragma once
#include <folly/hash/Hash.h>
#include <cassert>
#include <list>
#include <memory>
@ -41,8 +44,7 @@ namespace std {
template <>
struct hash<irequest> {
std::size_t operator()(const irequest& req) const noexcept {
auto h = hash<std::string>();
return h(req.type) ^ h(req.func) ^ h(req.arg);
return folly::hash::hash_combine(req.type, req.func, req.arg);
}
};

33
oi/Serialize.h
View File

@ -55,23 +55,26 @@ DEFINE_TYPE_VERSION(TypeHierarchy, 384, 7)
namespace boost::serialization {
#define DECL_SERIALIZE(Type) \
template <class Archive> \
void serialize(Archive&, Type&, const unsigned int)
template <class Archive>
void serialize(Archive&, PaddingInfo&, unsigned int);
DECL_SERIALIZE(PaddingInfo);
template <class Archive>
void serialize(Archive&, FuncDesc::Arg&, unsigned int);
template <class Archive>
void serialize(Archive&, FuncDesc&, unsigned int);
template <class Archive>
void serialize(Archive&, GlobalDesc&, unsigned int);
DECL_SERIALIZE(FuncDesc::Arg);
DECL_SERIALIZE(FuncDesc);
DECL_SERIALIZE(GlobalDesc);
template <class Archive>
void serialize(Archive&, drgn_type&, unsigned int);
template <class Archive>
void serialize(Archive&, drgn_qualified_type&, unsigned int);
template <class Archive>
void serialize(Archive&, RootInfo&, unsigned int);
DECL_SERIALIZE(struct drgn_type);
DECL_SERIALIZE(struct drgn_qualified_type);
DECL_SERIALIZE(RootInfo);
DECL_SERIALIZE(DrgnClassMemberInfo);
DECL_SERIALIZE(TypeHierarchy);
#undef DECL_SERIALIZE
template <class Archive>
void serialize(Archive&, DrgnClassMemberInfo&, unsigned int);
template <class Archive>
void serialize(Archive&, TypeHierarchy&, unsigned int);
} // namespace boost::serialization

44
oi/cache/Builder.h vendored Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "oi/Features.h"
#include "oi/cache/Source.h"
namespace oi::detail::cache {
// Builder
//
// A builder which takes a request and satisfies it into a cache, usually
// remote. It is expected that you first check for presence in a Source, then
// call a remote Builder if an entry is not found.
class Builder {
public:
struct Request {
Source::Request location;
};
enum class Response {
Failure,
Processing,
Success,
};
virtual ~Builder() = default;
virtual Response build(const Request&);
};
} // namespace oi::detail::cache

78
oi/cache/Entity.cpp vendored Normal file
View File

@ -0,0 +1,78 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "Entity.h"
#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include "oi/Serialize.h"
namespace oi::detail::cache {
void SourceCode::store(std::ostream& os) const {
os << code;
}
void SourceCode::load(std::istream& is) {
is >> code;
}
void ObjectCode::store(std::ostream& os) const {
std::copy(code.begin(), code.end(), std::ostreambuf_iterator(os));
}
void ObjectCode::load(std::istream& is) {
std::copy(std::istreambuf_iterator<char>(is),
std::istreambuf_iterator<char>(), std::back_inserter(code));
}
void FuncDescs::store(std::ostream& os) const {
boost::archive::text_oarchive oa{os};
oa << descs;
}
void FuncDescs::load(std::istream& is) {
boost::archive::text_iarchive ia{is};
ia >> descs;
}
void GlobalDescs::store(std::ostream& os) const {
boost::archive::text_oarchive oa{os};
oa << descs;
}
void GlobalDescs::load(std::istream& is) {
boost::archive::text_iarchive ia{is};
ia >> descs;
}
void TypeHierarchy::store(std::ostream& os) const {
boost::archive::text_oarchive oa{os};
oa << root;
oa << th;
}
void TypeHierarchy::load(std::istream& is) {
boost::archive::text_iarchive ia{is};
ia >> root;
ia >> th;
}
void PaddingInfo::store(std::ostream& os) const {
boost::archive::text_oarchive oa{os};
oa << info;
}
void PaddingInfo::load(std::istream& is) {
boost::archive::text_iarchive ia{is};
ia >> info;
}
} // namespace oi::detail::cache

145
oi/cache/Entity.h vendored Normal file
View File

@ -0,0 +1,145 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <map>
#include <string>
#include <unordered_map>
#include <vector>
#include "oi/Descs.h"
#include "oi/PaddingHunter.h"
#include "oi/TypeHierarchy.h"
namespace oi::detail::cache {
enum class EntityType {
SourceCode,
ObjectCode,
FuncDescs,
GlobalDescs,
TypeHierarchy,
PaddingInfo,
};
class Entity {
public:
virtual ~Entity() = default;
virtual EntityType type() const = 0;
virtual void store(std::ostream&) const = 0;
virtual void load(std::istream&) = 0;
};
struct SourceCode final : public Entity {
SourceCode() {
}
SourceCode(std::string code_) : code{std::move(code_)} {
}
std::string code;
EntityType type() const override {
return EntityType::SourceCode;
}
void store(std::ostream&) const override;
void load(std::istream&) override;
};
struct ObjectCode final : public Entity {
ObjectCode() {
}
ObjectCode(std::vector<uint8_t> code_) : code{std::move(code_)} {
}
std::vector<uint8_t> code;
EntityType type() const override {
return EntityType::ObjectCode;
}
void store(std::ostream&) const override;
void load(std::istream&) override;
};
struct FuncDescs final : public Entity {
FuncDescs() {
}
FuncDescs(std::unordered_map<std::string, std::shared_ptr<FuncDesc>> descs_)
: descs{std::move(descs_)} {
}
std::unordered_map<std::string, std::shared_ptr<FuncDesc>> descs;
EntityType type() const override {
return EntityType::FuncDescs;
}
void store(std::ostream&) const override;
void load(std::istream&) override;
};
struct GlobalDescs final : public Entity {
GlobalDescs() {
}
GlobalDescs(
std::unordered_map<std::string, std::shared_ptr<GlobalDesc>> descs_)
: descs{std::move(descs_)} {
}
std::unordered_map<std::string, std::shared_ptr<GlobalDesc>> descs;
EntityType type() const override {
return EntityType::GlobalDescs;
}
void store(std::ostream&) const override;
void load(std::istream&) override;
};
struct TypeHierarchy final : public Entity {
TypeHierarchy() {
}
TypeHierarchy(RootInfo root_, ::TypeHierarchy th_)
: root{std::move(root_)}, th{std::move(th_)} {
}
::RootInfo root;
::TypeHierarchy th;
EntityType type() const override {
return EntityType::TypeHierarchy;
}
void store(std::ostream&) const override;
void load(std::istream&) override;
};
struct PaddingInfo final : public Entity {
PaddingInfo() {
}
PaddingInfo(std::map<std::string, ::PaddingInfo> info_)
: info{std::move(info_)} {
}
std::map<std::string, ::PaddingInfo> info;
EntityType type() const override {
return EntityType::PaddingInfo;
}
void store(std::ostream&) const override;
void load(std::istream&) override;
};
} // namespace oi::detail::cache

34
oi/cache/GobsBuilder.h vendored Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "oi/cache/Builder.h"
namespace oi::detail::cache {
class GobsBuilderImpl;
class GobsBuilder : public Builder {
public:
GobsBuilder();
Response build(const Request&) override;
private:
std::unique_ptr<GobsBuilderImpl> impl_;
};
} // namespace oi::detail::cache

81
oi/cache/LocalCache.cpp vendored Normal file
View File

@ -0,0 +1,81 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "LocalCache.h"
#include <fstream>
#include <stdexcept> // TODO: remove
#include <string_view>
namespace oi::detail::cache {
namespace {
std::string genHash(const Source::Request& req);
std::string_view getExtension(const Entity& en);
} // namespace
bool LocalCache::read(const Source::Request& req, Entity& en) {
std::string name = genHash(req);
name.append(getExtension(en));
std::filesystem::path path = cache_dir_ / name;
if (!std::filesystem::exists(path))
return false;
std::ifstream ifs{path};
en.load(ifs);
return true;
}
bool LocalCache::write(const Source::Request& req, const Entity& en) {
std::string name = genHash(req);
name.append(getExtension(en));
std::filesystem::path path = cache_dir_ / name;
std::filesystem::create_directories(cache_dir_);
std::ofstream of{path};
en.store(of);
return true;
}
namespace {
std::string genHash(const Source::Request& req) {
return std::to_string(std::hash<Source::Request>{}(req));
}
std::string_view getExtension(const Entity& en) {
switch (en.type()) {
case EntityType::SourceCode:
return ".cc";
case EntityType::ObjectCode:
return ".o";
case EntityType::FuncDescs:
return ".fd";
case EntityType::GlobalDescs:
return ".gd";
case EntityType::TypeHierarchy:
return ".th";
case EntityType::PaddingInfo:
return ".pd";
}
}
} // namespace
} // namespace oi::detail::cache

36
oi/cache/LocalCache.h vendored Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "oi/cache/Source.h"
namespace oi::detail::cache {
class LocalCache : public Source {
public:
LocalCache(std::filesystem::path cache_dir)
: cache_dir_{std::move(cache_dir)} {
}
~LocalCache() override = default;
bool read(const Request& req, Entity& en) override;
bool write(const Request& req, const Entity& en) override;
private:
std::filesystem::path cache_dir_;
};
} // namespace oi::detail::cache

38
oi/cache/ManifoldCache.h vendored Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "oi/cache/Source.h"
namespace oi::detail::cache {
class ManifoldCacheImpl;
class ManifoldCache : public Source {
public:
ManifoldCache();
bool read(const Request& req, Entity& en) override;
bool write(const Request& req, const Entity& en) override;
void setReadable(bool);
void setWriteable(bool);
private:
std::unique_ptr<ManifoldCacheImpl> impl_;
};
} // namespace oi::detail::cache

74
oi/cache/Source.h vendored Normal file
View File

@ -0,0 +1,74 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <folly/hash/Hash.h>
#include <filesystem>
#include <optional>
#include <string>
#include "oi/Features.h"
#include "oi/OIParser.h"
#include "oi/cache/Entity.h"
namespace oi::detail::cache {
// Source
//
// A source of cached Entities. May be implemented as a local or remote
// cache. Not expected to build a cache if one is not found.
class Source {
public:
struct Request {
FeatureSet features;
irequest probe;
pid_t traceePid = -1;
std::optional<std::string> buildId;
};
virtual ~Source() = default;
// Read and fill an Entity from the cache
//
// Return true if the operation did not fail. This includes if this Source is
// intentionally write-only.
virtual bool read(const Request&, Entity&) {
return true;
}
// Write a filled Entity to the cache
//
// Return true if the operation did not fail. This includes if this Source is
// intentionally read-only.
virtual bool write(const Request&, const Entity&) {
return true;
}
};
} // namespace oi::detail::cache
namespace std {
template <>
struct hash<oi::detail::cache::Source::Request> {
size_t operator()(const oi::detail::cache::Source::Request& req) const {
return folly::hash::hash_combine(req.features, req.probe, req.buildId);
}
};
} // namespace std

42
oi/support/MemoryFile.cpp Normal file
View File

@ -0,0 +1,42 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "MemoryFile.h"
#include <glog/logging.h>
#include <sys/mman.h>
#include <boost/format.hpp>
#include <cstring>
#include <stdexcept>
#include <string>
MemoryFile::MemoryFile(const char* name) {
fd_ = memfd_create(name, 0);
if (fd_ == -1)
throw std::runtime_error(std::string("memfd creation failed: ") +
std::strerror(errno));
}
MemoryFile::~MemoryFile() {
if (fd_ == -1)
return;
PLOG_IF(ERROR, close(fd_) == -1) << "memfd close failed";
}
std::filesystem::path MemoryFile::path() {
return {(boost::format("/dev/fd/%1%") % fd_).str()};
}

29
oi/support/MemoryFile.h Normal file
View File

@ -0,0 +1,29 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <filesystem>
class MemoryFile {
public:
MemoryFile(const char* name);
~MemoryFile();
std::filesystem::path path();
private:
int fd_ = -1;
};

1
test/integration/runner_common.cpp
View File

@ -207,6 +207,7 @@ OidProc OidIntegration::runOidOnProcess(OidOpts opts,
"--dump-json"s,
"--script-source"s, opts.scriptSource,
"--mode=strict"s,
"--cache-path=."s,
};
// clang-format on

18
test/test_compiler.cpp
View File

@ -43,11 +43,11 @@ TEST(CompilerTest, CompileAndRelocate) {
EXPECT_NE(mkdtemp(const_cast<char*>(tmpdir.c_str())), nullptr);
auto sourcePath = tmpdir / "src.cpp";
auto objectPath = tmpdir / "obj.o";
std::vector<uint8_t> objectContent;
OICompiler compiler{symbols, {}};
EXPECT_TRUE(compiler.compile(code, sourcePath, objectPath));
EXPECT_TRUE(compiler.compile(code, sourcePath, objectContent));
const size_t relocSlabSize = 4096;
void* relocSlab = mmap(nullptr,
@ -59,7 +59,7 @@ TEST(CompilerTest, CompileAndRelocate) {
EXPECT_NE(relocSlab, nullptr);
auto relocResult =
compiler.applyRelocs((uintptr_t)relocSlab, {objectPath}, {});
compiler.applyRelocs((uintptr_t)relocSlab, {objectContent}, {});
EXPECT_TRUE(relocResult.has_value());
auto& [_, segs, jitSymbols] = relocResult.value();
@ -121,14 +121,14 @@ TEST(CompilerTest, CompileAndRelocateMultipleObjs) {
EXPECT_NE(mkdtemp(const_cast<char*>(tmpdir.c_str())), nullptr);
auto sourceXPath = tmpdir / "srcX.cpp";
auto objectXPath = tmpdir / "objX.o";
std::vector<uint8_t> objectXContent;
auto sourceYPath = tmpdir / "srcY.cpp";
auto objectYPath = tmpdir / "objY.o";
std::vector<uint8_t> objectYContent;
OICompiler compiler{symbols, {}};
EXPECT_TRUE(compiler.compile(codeX, sourceXPath, objectXPath));
EXPECT_TRUE(compiler.compile(codeY, sourceYPath, objectYPath));
EXPECT_TRUE(compiler.compile(codeX, sourceXPath, objectXContent));
EXPECT_TRUE(compiler.compile(codeY, sourceYPath, objectYContent));
const size_t relocSlabSize = 8192;
void* relocSlab = mmap(nullptr,
@ -139,8 +139,8 @@ TEST(CompilerTest, CompileAndRelocateMultipleObjs) {
0);
EXPECT_NE(relocSlab, nullptr);
auto relocResult = compiler.applyRelocs(
(uintptr_t)relocSlab, {objectXPath, objectYPath}, {});
auto relocResult = compiler.applyRelocs((uintptr_t)relocSlab,
{objectXContent, objectYContent}, {});
EXPECT_TRUE(relocResult.has_value());
auto& [_, segs, jitSymbols] = relocResult.value();