From 4174f62cf8526dbec48ffbbda7571e9dbcceb281 Mon Sep 17 00:00:00 2001 From: Jake Hillion Date: Wed, 15 Nov 2023 06:47:13 -0800 Subject: [PATCH] improve caching infrastructure --- CMakeLists.txt | 5 +- oi/CMakeLists.txt | 17 ++ oi/Cache.cpp | 55 ++++++ oi/Cache.h | 43 +++++ oi/EnumBitset.h | 13 ++ oi/OICache.cpp | 257 ----------------------------- oi/OICache.h | 76 --------- oi/OICompiler.cpp | 30 ++-- oi/OICompiler.h | 8 +- oi/OID.cpp | 54 +++--- oi/OIDebugger.cpp | 180 +++++++++----------- oi/OIDebugger.h | 53 +----- oi/OIGenerator.cpp | 10 +- oi/OILibraryImpl.cpp | 24 +-- oi/OIParser.h | 6 +- oi/Serialize.h | 33 ++-- oi/cache/Builder.h | 44 +++++ oi/cache/Entity.cpp | 78 +++++++++ oi/cache/Entity.h | 145 ++++++++++++++++ oi/cache/GobsBuilder.h | 34 ++++ oi/cache/LocalCache.cpp | 81 +++++++++ oi/cache/LocalCache.h | 36 ++++ oi/cache/ManifoldCache.h | 38 +++++ oi/cache/Source.h | 74 +++++++++ oi/support/MemoryFile.cpp | 42 +++++ oi/support/MemoryFile.h | 29 ++++ test/integration/runner_common.cpp | 1 + test/test_compiler.cpp | 18 +- 28 files changed, 909 insertions(+), 575 deletions(-) create mode 100644 oi/Cache.cpp create mode 100644 oi/Cache.h delete mode 100644 oi/OICache.cpp delete mode 100644 oi/OICache.h create mode 100644 oi/cache/Builder.h create mode 100644 oi/cache/Entity.cpp create mode 100644 oi/cache/Entity.h create mode 100644 oi/cache/GobsBuilder.h create mode 100644 oi/cache/LocalCache.cpp create mode 100644 oi/cache/LocalCache.h create mode 100644 oi/cache/ManifoldCache.h create mode 100644 oi/cache/Source.h create mode 100644 oi/support/MemoryFile.cpp create mode 100644 oi/support/MemoryFile.h diff --git a/CMakeLists.txt b/CMakeLists.txt index d7f9ee7..effcfad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -272,7 +272,7 @@ ADD_FLEX_BISON_DEPENDENCY(Lexer Parser) add_library(oid_parser STATIC ${BISON_Parser_OUTPUTS} ${FLEX_Lexer_OUTPUTS}) target_compile_options(oid_parser PRIVATE "-w") # Disable warnings for generated code -target_link_libraries(oid_parser glog::glog) +target_link_libraries(oid_parser glog::glog folly_headers) ### Core OI @@ -283,7 +283,6 @@ add_library(oicore oi/Config.cpp oi/Descs.cpp oi/Metrics.cpp - oi/OICache.cpp oi/OICompiler.cpp oi/PaddingHunter.cpp oi/Serialize.cpp @@ -295,7 +294,9 @@ target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) llvm_map_components_to_libnames(llvm_libs core native mcjit x86disassembler) target_link_libraries(oicore + cache codegen + memory_file ${Boost_LIBRARIES} Boost::headers diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt index 3754f41..83d6afb 100644 --- a/oi/CMakeLists.txt +++ b/oi/CMakeLists.txt @@ -21,11 +21,17 @@ target_link_libraries(symbol_service Boost::headers ${Boost_LIBRARIES} + folly_headers glog::glog dw ) +add_library(memory_file + support/MemoryFile.cpp +) +target_link_libraries(memory_file glog::glog Boost::headers) + add_library(features Features.cpp) target_link_libraries(features glog::glog) @@ -55,6 +61,17 @@ target_link_libraries(codegen glog::glog ) +add_library(cache + Cache.cpp + Serialize.cpp + cache/Entity.cpp + cache/LocalCache.cpp +) +target_link_libraries(cache + Boost::serialization + folly_headers +) + add_library(exporters_json exporters/Json.cpp) target_include_directories(exporters_json PUBLIC ${CMAKE_SOURCE_DIR}/include) target_link_libraries(exporters_json oil) diff --git a/oi/Cache.cpp b/oi/Cache.cpp new file mode 100644 index 0000000..9599d5e --- /dev/null +++ b/oi/Cache.cpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "Cache.h" + +namespace oi::detail { + +using cache::Builder; +using cache::Entity; +using cache::Source; + +void Cache::registerSource(std::unique_ptr s) { + sources.emplace_back(std::move(s)); +} +void Cache::registerBuilder(std::unique_ptr b) { + builders.emplace_back(std::move(b)); +} + +bool Cache::read(const Source::Request& req, Entity& en) { + for (auto& s : sources) { + if (s->read(req, en)) + return true; + } + return false; +} + +bool Cache::write(const Source::Request& req, const Entity& en) { + for (auto& s : sources) { + if (s->write(req, en)) + return true; + } + return false; +} + +Builder::Response Cache::build(const Builder::Request& req) { + for (auto& b : builders) { + if (auto ret = b->build(req); ret != Builder::Response::Failure) + return ret; + } + return Builder::Response::Failure; +} + +} // namespace oi::detail diff --git a/oi/Cache.h b/oi/Cache.h new file mode 100644 index 0000000..ada5b24 --- /dev/null +++ b/oi/Cache.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include + +#include "oi/cache/Builder.h" +#include "oi/cache/Source.h" + +namespace oi::detail { + +class Cache { + public: + void registerSource(std::unique_ptr); + void registerBuilder(std::unique_ptr); + + bool read(const cache::Source::Request&, cache::Entity&); + bool write(const cache::Source::Request&, const cache::Entity&); + + cache::Builder::Response build(const cache::Builder::Request&); + + private: + std::vector> sources; + std::vector> builders; +}; + +} // namespace oi::detail diff --git a/oi/EnumBitset.h b/oi/EnumBitset.h index d9828ee..6d7e434 100644 --- a/oi/EnumBitset.h +++ b/oi/EnumBitset.h @@ -61,6 +61,8 @@ class EnumBitset { private: BitsetType bitset; + + friend struct std::hash>; }; template @@ -70,3 +72,14 @@ EnumBitset operator&(const EnumBitset& lhs, out &= rhs; return out; } + +namespace std { + +template +struct hash> { + size_t operator()(const EnumBitset& bs) const { + return hash>{}(bs.bitset); + } +}; + +} // namespace std diff --git a/oi/OICache.cpp b/oi/OICache.cpp deleted file mode 100644 index 4f2d8d7..0000000 --- a/oi/OICache.cpp +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "oi/OICache.h" - -#include - -#include -#include -#include - -#include "oi/Descs.h" -#include "oi/OICodeGen.h" -#include "oi/Portability.h" -#include "oi/Serialize.h" - -#if OI_PORTABILITY_META_INTERNAL() -#include "object-introspection/internal/GobsService.h" -#include "object-introspection/internal/ManifoldCache.h" -#endif - -namespace oi::detail { - -static std::optional> getEntName( - SymbolService& symbols, const irequest& req, OICache::Entity ent) { - if (ent == OICache::Entity::FuncDescs || - ent == OICache::Entity::GlobalDescs) { - return req.func; - } else { - if (req.type == "global") { - const auto& globalDesc = symbols.findGlobalDesc(req.func); - if (!globalDesc) { - return std::nullopt; - } - - return globalDesc->typeName; - } else { - const auto& funcDesc = symbols.findFuncDesc(req); - if (!funcDesc) { - return std::nullopt; - } - - const auto& arg = funcDesc->getArgument(req.arg); - if (!arg) { - return std::nullopt; - } - - return arg->typeName; - } - } -} - -std::optional OICache::getPath(const irequest& req, - Entity ent) const { - auto hash = [](const std::string& str) { - return std::to_string(std::hash{}(str)); - }; - - auto ext = extensions[static_cast(ent)]; - - const auto& entName = getEntName(*symbols, req, ent); - if (!entName.has_value()) { - return std::nullopt; - } - - return basePath / (hash(*entName) + ext); -} - -template -bool OICache::load(const irequest& req, Entity ent, T& data) { - if (!isEnabled()) - return false; - try { - auto buildID = symbols->locateBuildID(); - if (!buildID) { - LOG(ERROR) << "Failed to locate buildID"; - return false; - } - - auto cachePath = getPath(req, ent); - if (!cachePath.has_value()) { - LOG(ERROR) << "Failed to get cache path for " << req.type << ':' - << req.func << ':' << req.arg << '/' - << static_cast(ent); - return false; - } - - LOG(INFO) << "Loading cache " << *cachePath; - std::ifstream ifs(*cachePath); - boost::archive::text_iarchive ia(ifs); - - std::string cacheBuildId; - ia >> cacheBuildId; - if (cacheBuildId != *buildID) { - LOG(ERROR) << "The cache's build id '" << cacheBuildId - << "' doesn't match the target's build id '" << *buildID - << "'"; - return false; - } - - ia >> data; - return true; - } catch (const std::exception& e) { - LOG(WARNING) << "Failed to load from cache: " << e.what(); - return false; - } -} - -template -bool OICache::store(const irequest& req, Entity ent, const T& data) { - if (!isEnabled()) - return false; - try { - auto buildID = symbols->locateBuildID(); - if (!buildID) { - LOG(ERROR) << "Failed to locate buildID"; - return false; - } - - auto cachePath = getPath(req, ent); - if (!cachePath.has_value()) { - LOG(ERROR) << "Failed to get cache path for " << req.type << ':' - << req.func << ':' << req.arg << '/' - << static_cast(ent); - return false; - } - - LOG(INFO) << "Storing cache " << *cachePath; - std::ofstream ofs(*cachePath); - boost::archive::text_oarchive oa(ofs); - - oa << *buildID; - oa << data; - return true; - } catch (const std::exception& e) { - LOG(WARNING) << "Failed to write to cache: " << e.what(); - return false; - } -} - -#define INSTANTIATE_ARCHIVE(...) \ - template bool OICache::load(const irequest&, Entity, __VA_ARGS__&); \ - template bool OICache::store(const irequest&, Entity, const __VA_ARGS__&); - -INSTANTIATE_ARCHIVE(std::pair) -INSTANTIATE_ARCHIVE(std::unordered_map>) -INSTANTIATE_ARCHIVE( - std::unordered_map>) -INSTANTIATE_ARCHIVE(std::map) - -#undef INSTANTIATE_ARCHIVE - -// Upload all contents of cache for this request -bool OICache::upload([[maybe_unused]] const irequest& req) { -#if OI_PORTABILITY_META_INTERNAL() - if (!isEnabled() || downloadedRemote || !enableUpload) - return true; - std::vector files; - - for (size_t i = 0; i < static_cast(OICache::Entity::MAX); i++) { - auto cachePath = getPath(req, static_cast(i)); - if (!cachePath.has_value()) { - LOG(ERROR) << "Failed to get cache path for " << req.type << ':' - << req.func << ':' << req.arg << '/' << static_cast(i); - return false; - } - files.push_back(*cachePath); - } - - auto hash = generateRemoteHash(req); - if (hash.empty()) { - LOG(ERROR) << "failed to generate remote lookup hash"; - return false; - } - - return ObjectIntrospection::ManifoldCache::upload(hash, files); -#else - if (isEnabled() && !downloadedRemote && enableUpload) { - // We tried to download when support is not enabled! - LOG(ERROR) << "Tried to upload artifacts when support is not enabled!"; - return false; - } - return true; -#endif -} - -// Try to fetch contents of cache -bool OICache::download([[maybe_unused]] const irequest& req) { -#if OI_PORTABILITY_META_INTERNAL() - if (!isEnabled() || !enableDownload) - return true; - - auto hash = generateRemoteHash(req); - if (hash.empty()) { - LOG(ERROR) << "failed to generate remote lookup hash"; - return false; - } - - if (basePath.filename() != hash) { - // Use a subdirectory per hash shard to avoid conflicts - basePath /= hash; - if (fs::exists(basePath.parent_path()) && !fs::exists(basePath)) - fs::create_directory(basePath); - } - if (ObjectIntrospection::ManifoldCache::download(hash, basePath)) { - downloadedRemote = true; - return true; - } - if (abortOnLoadFail) { - LOG(ERROR) << "We weren't able to pull artifacts when requested, " - "aborting run!"; - // If we aren't uploading, quit early as we requested a download and - // weren't able to get it. - return false; - } - return true; -#else - if (isEnabled() && enableDownload) { - // We tried to download when support is not enabled! - LOG(ERROR) << "Tried to download artifacts when support is not enabled!"; - return false; - } - return true; -#endif -} - -std::string OICache::generateRemoteHash(const irequest& req) { - auto buildID = symbols->locateBuildID(); - if (!buildID) { - LOG(ERROR) << "Failed to locate buildID"; - return ""; - } - - std::string remote_cache_id = *buildID + "/" + req.func + "/" + req.arg + - "/" + generatorConfig.toString(); -#if OI_PORTABILITY_META_INTERNAL() - auto version_pair = ObjectIntrospection::GobsService::getOidRpmVersions(); - remote_cache_id += "/" + version_pair.first + "/" + version_pair.second; -#endif - - LOG(INFO) << "generating remote hash from: " << remote_cache_id; - return std::to_string(std::hash{}(remote_cache_id)); -} - -} // namespace oi::detail diff --git a/oi/OICache.h b/oi/OICache.h deleted file mode 100644 index 6f8a1d4..0000000 --- a/oi/OICache.h +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include -#include -#include -#include - -#include "oi/OICodeGen.h" -#include "oi/OIParser.h" -#include "oi/SymbolService.h" - -namespace oi::detail { - -class OICache { - public: - OICache(const OICodeGen::Config& generatorConfig) - : generatorConfig(generatorConfig) { - } - - std::filesystem::path basePath{}; - std::shared_ptr symbols{}; - bool downloadedRemote = false; - bool enableUpload = false; - bool enableDownload = false; - bool abortOnLoadFail = false; - - // We need the generator config to download the cache - // with the matching configuration. - const OICodeGen::Config& generatorConfig; - - // Entity is used to index the `extensions` array - // So we must keep the Entity enum and `extensions` array in sync! - enum class Entity { - Source, - Object, - FuncDescs, - GlobalDescs, - TypeHierarchy, - PaddingInfo, - MAX - }; - static constexpr std::array(Entity::MAX)> - extensions{".cc", ".o", ".fd", ".gd", ".th", ".pd"}; - - bool isEnabled() const { - return !basePath.empty(); - } - std::optional getPath(const irequest&, Entity) const; - template - bool store(const irequest&, Entity, const T&); - template - bool load(const irequest&, Entity, T&); - - bool upload(const irequest& req); - bool download(const irequest& req); - - private: - std::string generateRemoteHash(const irequest&); -}; - -} // namespace oi::detail diff --git a/oi/OICompiler.cpp b/oi/OICompiler.cpp index cd67c68..0a4731f 100644 --- a/oi/OICompiler.cpp +++ b/oi/OICompiler.cpp @@ -41,9 +41,13 @@ #include #include #include +#include +#include +#include #include "oi/Headers.h" #include "oi/Metrics.h" +#include "oi/support/MemoryFile.h" extern "C" { #include @@ -474,7 +478,7 @@ static void debugDisAsm( bool OICompiler::compile(const std::string& code, const fs::path& sourcePath, - const fs::path& objectPath) { + std::vector& objectBuf) { metrics::Tracing _("compile"); /* @@ -507,7 +511,8 @@ bool OICompiler::compile(const std::string& code, compInv->getFrontendOpts().Inputs.push_back( FrontendInputFile(sourcePath.string(), InputKind{Language::CXX})); - compInv->getFrontendOpts().OutputFile = objectPath.string(); + MemoryFile fileBuf{"out.o"}; + compInv->getFrontendOpts().OutputFile = fileBuf.path().string(); compInv->getFrontendOpts().ProgramAction = clang::frontend::EmitObj; auto& headerSearchOptions = compInv->getHeaderSearchOpts(); @@ -561,7 +566,6 @@ bool OICompiler::compile(const std::string& code, } } - compInv->getFrontendOpts().OutputFile = objectPath; compInv->getTargetOpts().Triple = llvm::Triple::normalize(llvm::sys::getProcessTriple()); if (config.usePIC) { @@ -614,12 +618,15 @@ bool OICompiler::compile(const std::string& code, } */ + std::ifstream ifs{fileBuf.path().string(), std::ios::binary}; + objectBuf = std::vector(std::istreambuf_iterator(ifs), + std::istreambuf_iterator()); return true; } std::optional OICompiler::applyRelocs( uintptr_t baseRelocAddress, - const std::set& objectFiles, + const std::vector>& objectFiles, const std::unordered_map& syntheticSymbols) { metrics::Tracing relocationTracing("relocation"); @@ -627,16 +634,19 @@ std::optional OICompiler::applyRelocs( RuntimeDyld dyld(*memMgr, *memMgr); /* Load all the object files into the MemoryManager */ - for (const auto& objPath : objectFiles) { - VLOG(1) << "Loading object file " << objPath; - auto objFile = ObjectFile::createObjectFile(objPath.c_str()); + for (const auto& objContent : objectFiles) { + VLOG(1) << "Loading object content from memory"; + llvm::MemoryBufferRef objBuf{ + llvm::StringRef((const char*)objContent.data(), objContent.size()), + "test.o"}; + auto objFile = ObjectFile::createObjectFile(objBuf); if (!objFile) { - raw_os_ostream(LOG(ERROR)) << "Failed to load object file " << objPath - << ": " << objFile.takeError(); + raw_os_ostream(LOG(ERROR)) + << "Failed to load object file:" << objFile.takeError(); return std::nullopt; } - dyld.loadObject(*objFile->getBinary()); + dyld.loadObject(**objFile); if (dyld.hasError()) { LOG(ERROR) << "load object failed: " << dyld.getErrorString().data(); return std::nullopt; diff --git a/oi/OICompiler.h b/oi/OICompiler.h index 2ce4544..0f9fef2 100644 --- a/oi/OICompiler.h +++ b/oi/OICompiler.h @@ -132,12 +132,12 @@ class OICompiler { * Compile the given @param code and write the result in @param objectPath. * * @param code the C++ source code to compile - * @param sourcePath path/name of the code to compile (not used) - * @param objectPath path where to write the resulting Object file + * @param sourcePath path/name of the code to compile for debug info. + * @param objectBuf memory location to store the resultant code. * * @return true if the compilation succeeded, false otherwise. */ - bool compile(const std::string&, const fs::path&, const fs::path&); + bool compile(const std::string&, const fs::path&, std::vector&); /** * Load the @param objectFiles in memory and apply relocation at @@ -158,7 +158,7 @@ class OICompiler { */ std::optional applyRelocs( uintptr_t, - const std::set&, + const std::vector>&, const std::unordered_map&); /** diff --git a/oi/OID.cpp b/oi/OID.cpp index 6403b61..094845e 100644 --- a/oi/OID.cpp +++ b/oi/OID.cpp @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include #include #include @@ -39,10 +40,9 @@ extern "C" { #include "oi/Portability.h" #include "oi/TimeUtils.h" #include "oi/TreeBuilder.h" - -#if OI_PORTABILITY_META_INTERNAL() -#include -#endif +#include "oi/cache/GobsBuilder.h" +#include "oi/cache/LocalCache.h" +#include "oi/cache/ManifoldCache.h" namespace oi::detail { @@ -334,15 +334,21 @@ static ExitStatus::ExitStatus runScript( } weak_oid = oid; // set the weak_ptr for signal handlers + Cache cache; if (!oidConfig.cacheBasePath.empty()) { - oid->setCacheBasePath(oidConfig.cacheBasePath); + cache.registerSource( + std::make_unique(oidConfig.cacheBasePath)); } + if constexpr (kIsMetaInternal) { + auto manifold = std::make_unique(); + manifold->setReadable(oidConfig.remoteCacheDownload); + manifold->setWriteable(oidConfig.remoteCacheUpload); + cache.registerSource(std::move(manifold)); - oid->setCacheRemoteEnabled(oidConfig.cacheRemoteUpload, - oidConfig.cacheRemoteDownload); - if (!oid->validateCache()) { - return ExitStatus::UsageError; + cache.registerBuilder(std::make_unique()); } + oid->setCache(std::move(cache)); + oid->setCustomCodeFile(oidConfig.customCodeFile); oid->setHardDisableDrgn(oidConfig.hardDisableDrgn); oid->setStrict(oidConfig.strict); @@ -404,8 +410,7 @@ static ExitStatus::ExitStatus runScript( } VLOG(1) << "init took " << std::dec << time_ns(time_hr::now() - initStart) - << " nsecs\n" - << "Compilation Started"; + << " nsecs\n"; auto compileStart = time_hr::now(); @@ -489,11 +494,11 @@ static ExitStatus::ExitStatus runScript( } } - // Upload cache artifacts if present - if (!oid->uploadCache()) { - LOG(ERROR) << "cache upload requested and failed"; - return ExitStatus::CacheUploadError; - } + // TODO: Upload cache artifacts if present + // if (!oid->uploadCache()) { + // LOG(ERROR) << "cache upload requested and failed"; + // return ExitStatus::CacheUploadError; + // } std::cout << "SUCCESS " << fileName << std::endl; VLOG(1) << "Entire process took " << time_ns(time_hr::now() - progStart) @@ -524,14 +529,14 @@ int main(int argc, char* argv[]) { metrics::Tracing _("main"); -#if OI_PORTABILITY_META_INTERNAL() - folly::InitOptions init; - init.useGFlags(false); - init.removeFlags(false); - folly::init(&argc, &argv, init); -#else - google::InitGoogleLogging(argv[0]); -#endif + if constexpr (kIsMetaInternal) { + folly::InitOptions init; + init.useGFlags(false); + init.removeFlags(false); + folly::init(&argc, &argv, init); + } else { + google::InitGoogleLogging(argv[0]); + } google::SetStderrLogging(google::WARNING); int c = 0; @@ -555,7 +560,6 @@ int main(int argc, char* argv[]) { // change default settings for prod oidConfig.hardDisableDrgn = true; oidConfig.cacheRemoteDownload = true; - oidConfig.cacheBasePath = "/tmp/oid-cache"; features[Feature::ChaseRawPointers] = true; } else if (strcmp("strict", optarg) == 0) { oidConfig.strict = true; diff --git a/oi/OIDebugger.cpp b/oi/OIDebugger.cpp index 9c98fc6..03e00cf 100644 --- a/oi/OIDebugger.cpp +++ b/oi/OIDebugger.cpp @@ -44,6 +44,7 @@ extern "C" { #include +#include "oi/Cache.h" #include "oi/CodeGen.h" #include "oi/Config.h" #include "oi/ContainerInfo.h" @@ -1886,8 +1887,7 @@ bool OIDebugger::removeTrap(pid_t pid, const trapInfo& t) { OIDebugger::OIDebugger(const OICodeGen::Config& genConfig, OICompiler::Config ccConfig, TreeBuilder::Config tbConfig) - : cache{genConfig}, - compilerConfig{std::move(ccConfig)}, + : compilerConfig{std::move(ccConfig)}, generatorConfig{genConfig}, treeBuilderConfig{std::move(tbConfig)} { debug = true; @@ -1904,7 +1904,6 @@ OIDebugger::OIDebugger(pid_t pid, symbols = std::make_shared(traceePid); setDataSegmentSize(dataSegSize); createSegmentConfigFile(); - cache.symbols = symbols; } OIDebugger::OIDebugger(fs::path debugInfo, @@ -1913,7 +1912,6 @@ OIDebugger::OIDebugger(fs::path debugInfo, TreeBuilder::Config tbConfig) : OIDebugger(genConfig, std::move(ccConfig), std::move(tbConfig)) { symbols = std::make_shared(std::move(debugInfo)); - cache.symbols = symbols; } /* @@ -2163,8 +2161,8 @@ bool OIDebugger::compileCode() { assert(pdata.numReqs() == 1); const auto& preq = pdata.getReq(); - OICompiler compiler{symbols, compilerConfig}; - std::set objectFiles{}; + std::optional compiler; + std::vector> objectContents; /* * Global probes don't have multiple arguments, but calling `getReqForArg(X)` @@ -2176,106 +2174,89 @@ bool OIDebugger::compileCode() { for (size_t i = 0; i < argCount; i++) { const auto& req = preq.getReqForArg(i); - if (cache.isEnabled()) { - // try to download cache artifacts if present - if (!downloadCache()) { -#if OI_PORTABILITY_META_INTERNAL() - // Send a request to the GOBS service - char buf[PATH_MAX]; - const std::string procpath = - "/proc/" + std::to_string(traceePid) + "/exe"; - size_t buf_size; - if ((buf_size = readlink(procpath.c_str(), buf, PATH_MAX)) == -1) { - LOG(ERROR) - << "Failed to get binary path for tracee, could not call GOBS: " - << strerror(errno); - } else { - LOG(INFO) << "Attempting to get cache request from gobs"; - ObjectIntrospection::GobsService::requestCache( - procpath, - std::string(buf, buf_size), - req.toString(), - generatorConfig.toOptions()); - } -#endif - LOG(ERROR) << "No cache file found, exiting!"; - return false; - } + cache::Source::Request cacheReq{ + .features = generatorConfig.features, + .probe = req, + .traceePid = traceePid ? traceePid : -1, + .buildId = symbols->locateBuildID(), + }; - if (req.type == "global") { - decltype(symbols->globalDescs) gds; - if (cache.load(req, OICache::Entity::GlobalDescs, gds)) { - symbols->globalDescs.merge(std::move(gds)); - } - } else { - decltype(symbols->funcDescs) fds; - if (cache.load(req, OICache::Entity::FuncDescs, fds)) { - symbols->funcDescs.merge(std::move(fds)); - } + cache::ObjectCode object; + cache::TypeHierarchy th; + cache::PaddingInfo pad; + + bool skipCodeGen = cache_.read(cacheReq, object) && + cache_.read(cacheReq, th) && cache_.read(cacheReq, pad); + + // Attempt to execute on a remote builder before building locally + if (!skipCodeGen) { + cache::Builder::Request builderReq{ + .location = cacheReq, + }; + switch (cache_.build(builderReq)) { + case cache::Builder::Response::Success: + LOG(INFO) << "remote builder success"; + skipCodeGen = cache_.read(cacheReq, object) && + cache_.read(cacheReq, th) && cache_.read(cacheReq, pad); + break; + case cache::Builder::Response::Processing: + LOG(INFO) << "remote builder processing, please come back later"; + return false; + case cache::Builder::Response::Failure: + break; // proceed to local generation } } - auto sourcePath = cache.getPath(req, OICache::Entity::Source); - auto objectPath = cache.getPath(req, OICache::Entity::Object); - auto typeHierarchyPath = cache.getPath(req, OICache::Entity::TypeHierarchy); - auto paddingInfoPath = cache.getPath(req, OICache::Entity::PaddingInfo); + if (skipCodeGen) { + LOG(INFO) + << "Successfully loaded from the cache, skipping code generation."; + typeInfos.emplace(req, std::make_tuple(th.root, th.th, pad.info)); + objectContents.emplace_back(std::move(object.code)); + continue; + } - if (!sourcePath || !objectPath || !typeHierarchyPath || !paddingInfoPath) { - LOG(ERROR) << "Failed to get all cache paths, aborting!"; + // 1. Code generate + // 2. Compile + // 3. Save into cache + VLOG(2) << "Compiling probe for '" << req.arg; + + auto genCode = generateCode(req); + if (!genCode.has_value()) { + LOG(ERROR) << "Failed to generate code"; + return false; + } + cache::SourceCode code{std::move(*genCode)}; + + if (!compiler.has_value()) + compiler.emplace(symbols, compilerConfig); + + fs::path sourcePath{}; + if (!compiler->compile(code.code, sourcePath, object.code)) { + LOG(ERROR) << "Failed to compile code"; return false; } - bool skipCodeGen = cache.isEnabled() && fs::exists(*objectPath) && - fs::exists(*typeHierarchyPath) && - fs::exists(*paddingInfoPath); - if (skipCodeGen) { - std::pair th; - skipCodeGen = - skipCodeGen && cache.load(req, OICache::Entity::TypeHierarchy, th); + bool cacheSuccess = true; + cacheSuccess &= cache_.write(cacheReq, code); + cacheSuccess &= cache_.write(cacheReq, object); - std::map pad; - skipCodeGen = - skipCodeGen && cache.load(req, OICache::Entity::PaddingInfo, pad); - - if (skipCodeGen) { - typeInfos.emplace(req, std::make_tuple(th.first, th.second, pad)); - } + if (req.type == "global") { + cacheSuccess &= + cache_.write(cacheReq, cache::GlobalDescs(symbols->globalDescs)); + } else { + cacheSuccess &= + cache_.write(cacheReq, cache::FuncDescs(symbols->funcDescs)); } - if (!skipCodeGen) { - VLOG(2) << "Compiling probe for '" << req.arg - << "' into: " << *objectPath; + const auto& [rootType, typeHierarchy, paddingInfo] = typeInfos.at(req); + cacheSuccess &= + cache_.write(cacheReq, cache::TypeHierarchy(rootType, typeHierarchy)); + cacheSuccess &= cache_.write(cacheReq, cache::PaddingInfo(paddingInfo)); - auto code = generateCode(req); - if (!code.has_value()) { - LOG(ERROR) << "Failed to generate code"; - return false; - } + if (!cacheSuccess) + LOG(ERROR) << "Failed to write all objects to cache"; - bool doCompile = !cache.isEnabled() || !fs::exists(*objectPath); - if (doCompile) { - if (!compiler.compile(*code, *sourcePath, *objectPath)) { - LOG(ERROR) << "Failed to compile code"; - return false; - } - } - } - - if (cache.isEnabled() && !skipCodeGen) { - if (req.type == "global") { - cache.store(req, OICache::Entity::GlobalDescs, symbols->globalDescs); - } else { - cache.store(req, OICache::Entity::FuncDescs, symbols->funcDescs); - } - - const auto& [rootType, typeHierarchy, paddingInfo] = typeInfos.at(req); - cache.store(req, - OICache::Entity::TypeHierarchy, - std::make_pair(rootType, typeHierarchy)); - cache.store(req, OICache::Entity::PaddingInfo, paddingInfo); - } - - objectFiles.insert(*objectPath); + objectContents.emplace_back(std::move(object.code)); } if (traceePid) { // we attach to a process @@ -2287,11 +2268,10 @@ bool OIDebugger::compileCode() { }; VLOG(2) << "Relocating..."; - for (const auto& o : objectFiles) { - VLOG(2) << " * " << o; - } - auto relocRes = compiler.applyRelocs( - segConfig.jitCodeStart, objectFiles, syntheticSymbols); + if (!compiler.has_value()) + compiler.emplace(symbols, compilerConfig); + auto relocRes = compiler->applyRelocs(segConfig.jitCodeStart, + objectContents, syntheticSymbols); if (!relocRes.has_value()) { LOG(ERROR) << "Failed to relocate object code"; return false; @@ -2950,10 +2930,6 @@ std::optional OIDebugger::generateCode(const irequest& req) { codegen2.codegenFromDrgn(root->type.type, code); } - if (auto sourcePath = cache.getPath(req, OICache::Entity::Source)) { - std::ofstream(*sourcePath) << code; - } - if (!customCodeFile.empty()) { auto ifs = std::ifstream(customCodeFile); code.assign(std::istreambuf_iterator(ifs), diff --git a/oi/OIDebugger.h b/oi/OIDebugger.h index 804d2cd..b206781 100644 --- a/oi/OIDebugger.h +++ b/oi/OIDebugger.h @@ -20,7 +20,7 @@ #include #include -#include "oi/OICache.h" +#include "oi/Cache.h" #include "oi/OICodeGen.h" #include "oi/OICompiler.h" #include "oi/OIParser.h" @@ -77,60 +77,15 @@ class OIDebugger { return oidShouldExit; }; - void setCacheBasePath(std::filesystem::path basePath) { - if (std::filesystem::exists(basePath.parent_path()) && - !std::filesystem::exists(basePath)) { - // Create cachedir if parent directory exists - // TODO if returning false here, throw an error - std::filesystem::create_directory(basePath); - } - cache.basePath = std::move(basePath); - } - - void setCacheRemoteEnabled(bool upload, bool download) { - cache.enableUpload = upload; - cache.enableDownload = download; - cache.abortOnLoadFail = download && !upload; - } - - bool validateCache() { - if ((cache.enableUpload || cache.enableDownload) && !cache.isEnabled()) { - LOG(ERROR) << "Cache download/upload option specified when cache is " - "disabled - aborting!"; - return false; - } - return true; - } - void setHardDisableDrgn(bool val) { symbols->setHardDisableDrgn(val); } void setStrict(bool val) { treeBuilderConfig.strict = val; } - - bool uploadCache() { - return std::all_of( - std::begin(pdata), std::end(pdata), [this](const auto& req) { - return std::all_of( - std::begin(req.args), - std::end(req.args), - [this, &req](const auto& arg) { - return cache.upload(irequest{req.type, req.func, arg}); - }); - }); + void setCache(Cache cache) { + cache_ = std::move(cache); } - bool downloadCache() { - return std::all_of( - std::begin(pdata), std::end(pdata), [this](const auto& req) { - return std::all_of( - std::begin(req.args), - std::end(req.args), - [this, &req](const auto& arg) { - return cache.download(irequest{req.type, req.func, arg}); - }); - }); - }; std::pair getTreeBuilderTyping() { assert(pdata.numReqs() == 1); @@ -179,7 +134,7 @@ class OIDebugger { const int replayInstSize = 512; bool trapsRemoved{false}; std::shared_ptr symbols; - OICache cache; + Cache cache_; /* * Map address of valid INT3 instruction to metadata for that interrupt. diff --git a/oi/OIGenerator.cpp b/oi/OIGenerator.cpp index 1cb2a49..6311fd7 100644 --- a/oi/OIGenerator.cpp +++ b/oi/OIGenerator.cpp @@ -153,14 +153,18 @@ fs::path OIGenerator::generateForType(const OICodeGen::Config& generatorConfig, OICompiler compiler{{}, compilerConfig}; + std::vector objectCode; + if (!compiler.compile(code, sourcePath, objectCode)) { + return {}; + } // TODO: Revert to outputPath and remove printing when typegraph is done. fs::path tmpObject = outputPath; tmpObject.replace_extension( "." + std::to_string(std::hash{}(linkageName)) + ".o"); + std::ofstream objectFile{tmpObject, std::ios::out | std::ios::binary}; + objectFile.write(reinterpret_cast(objectCode.data()), + objectCode.size()); - if (!compiler.compile(code, sourcePath, tmpObject)) { - return {}; - } return tmpObject; } diff --git a/oi/OILibraryImpl.cpp b/oi/OILibraryImpl.cpp index 6442e9a..70a0fbf 100644 --- a/oi/OILibraryImpl.cpp +++ b/oi/OILibraryImpl.cpp @@ -61,24 +61,6 @@ OILibraryImpl::LocalTextSegment::~LocalTextSegment() { << "segment unmap failed"; } -OILibraryImpl::MemoryFile::MemoryFile(const char* name) { - fd_ = memfd_create(name, 0); - if (fd_ == -1) - throw std::runtime_error(std::string("memfd creation failed: ") + - std::strerror(errno)); -} - -OILibraryImpl::MemoryFile::~MemoryFile() { - if (fd_ == -1) - return; - - PLOG_IF(ERROR, close(fd_) == -1) << "memfd close failed"; -} - -std::filesystem::path OILibraryImpl::MemoryFile::path() { - return {(boost::format("/dev/fd/%1%") % fd_).str()}; -} - OILibraryImpl::OILibraryImpl(void* atomicHole, std::unordered_set fs, GeneratorOptions opts) @@ -132,13 +114,13 @@ std::pair OILibraryImpl::compileCode() { outputFile << code; } - auto object = MemoryFile("oil_object_code"); OICompiler compiler{symbols, compilerConfig_}; - if (!compiler.compile(code, sourcePath, object.path())) + std::vector object; + if (!compiler.compile(code, sourcePath, object)) throw std::runtime_error("oil jit compilation failed!"); auto relocRes = compiler.applyRelocs( - reinterpret_cast(textSeg.data().data()), {object.path()}, {}); + reinterpret_cast(textSeg.data().data()), {object}, {}); if (!relocRes) throw std::runtime_error("oil jit relocation failed!"); diff --git a/oi/OIParser.h b/oi/OIParser.h index b0f94fd..5bcb20e 100644 --- a/oi/OIParser.h +++ b/oi/OIParser.h @@ -14,6 +14,9 @@ * limitations under the License. */ #pragma once + +#include + #include #include #include @@ -41,8 +44,7 @@ namespace std { template <> struct hash { std::size_t operator()(const irequest& req) const noexcept { - auto h = hash(); - return h(req.type) ^ h(req.func) ^ h(req.arg); + return folly::hash::hash_combine(req.type, req.func, req.arg); } }; diff --git a/oi/Serialize.h b/oi/Serialize.h index 313a021..518b520 100644 --- a/oi/Serialize.h +++ b/oi/Serialize.h @@ -55,23 +55,26 @@ DEFINE_TYPE_VERSION(TypeHierarchy, 384, 7) namespace boost::serialization { -#define DECL_SERIALIZE(Type) \ - template \ - void serialize(Archive&, Type&, const unsigned int) +template +void serialize(Archive&, PaddingInfo&, unsigned int); -DECL_SERIALIZE(PaddingInfo); +template +void serialize(Archive&, FuncDesc::Arg&, unsigned int); +template +void serialize(Archive&, FuncDesc&, unsigned int); +template +void serialize(Archive&, GlobalDesc&, unsigned int); -DECL_SERIALIZE(FuncDesc::Arg); -DECL_SERIALIZE(FuncDesc); -DECL_SERIALIZE(GlobalDesc); +template +void serialize(Archive&, drgn_type&, unsigned int); +template +void serialize(Archive&, drgn_qualified_type&, unsigned int); +template +void serialize(Archive&, RootInfo&, unsigned int); -DECL_SERIALIZE(struct drgn_type); -DECL_SERIALIZE(struct drgn_qualified_type); -DECL_SERIALIZE(RootInfo); - -DECL_SERIALIZE(DrgnClassMemberInfo); -DECL_SERIALIZE(TypeHierarchy); - -#undef DECL_SERIALIZE +template +void serialize(Archive&, DrgnClassMemberInfo&, unsigned int); +template +void serialize(Archive&, TypeHierarchy&, unsigned int); } // namespace boost::serialization diff --git a/oi/cache/Builder.h b/oi/cache/Builder.h new file mode 100644 index 0000000..1af8b47 --- /dev/null +++ b/oi/cache/Builder.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "oi/Features.h" +#include "oi/cache/Source.h" + +namespace oi::detail::cache { + +// Builder +// +// A builder which takes a request and satisfies it into a cache, usually +// remote. It is expected that you first check for presence in a Source, then +// call a remote Builder if an entry is not found. +class Builder { + public: + struct Request { + Source::Request location; + }; + enum class Response { + Failure, + Processing, + Success, + }; + + virtual ~Builder() = default; + + virtual Response build(const Request&); +}; + +} // namespace oi::detail::cache diff --git a/oi/cache/Entity.cpp b/oi/cache/Entity.cpp new file mode 100644 index 0000000..21ccfec --- /dev/null +++ b/oi/cache/Entity.cpp @@ -0,0 +1,78 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "Entity.h" + +#include +#include + +#include "oi/Serialize.h" + +namespace oi::detail::cache { + +void SourceCode::store(std::ostream& os) const { + os << code; +} +void SourceCode::load(std::istream& is) { + is >> code; +} + +void ObjectCode::store(std::ostream& os) const { + std::copy(code.begin(), code.end(), std::ostreambuf_iterator(os)); +} +void ObjectCode::load(std::istream& is) { + std::copy(std::istreambuf_iterator(is), + std::istreambuf_iterator(), std::back_inserter(code)); +} + +void FuncDescs::store(std::ostream& os) const { + boost::archive::text_oarchive oa{os}; + oa << descs; +} +void FuncDescs::load(std::istream& is) { + boost::archive::text_iarchive ia{is}; + ia >> descs; +} + +void GlobalDescs::store(std::ostream& os) const { + boost::archive::text_oarchive oa{os}; + oa << descs; +} +void GlobalDescs::load(std::istream& is) { + boost::archive::text_iarchive ia{is}; + ia >> descs; +} + +void TypeHierarchy::store(std::ostream& os) const { + boost::archive::text_oarchive oa{os}; + oa << root; + oa << th; +} +void TypeHierarchy::load(std::istream& is) { + boost::archive::text_iarchive ia{is}; + ia >> root; + ia >> th; +} + +void PaddingInfo::store(std::ostream& os) const { + boost::archive::text_oarchive oa{os}; + oa << info; +} +void PaddingInfo::load(std::istream& is) { + boost::archive::text_iarchive ia{is}; + ia >> info; +} + +} // namespace oi::detail::cache diff --git a/oi/cache/Entity.h b/oi/cache/Entity.h new file mode 100644 index 0000000..2cc3ee4 --- /dev/null +++ b/oi/cache/Entity.h @@ -0,0 +1,145 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include + +#include "oi/Descs.h" +#include "oi/PaddingHunter.h" +#include "oi/TypeHierarchy.h" + +namespace oi::detail::cache { + +enum class EntityType { + SourceCode, + ObjectCode, + FuncDescs, + GlobalDescs, + TypeHierarchy, + PaddingInfo, +}; + +class Entity { + public: + virtual ~Entity() = default; + + virtual EntityType type() const = 0; + + virtual void store(std::ostream&) const = 0; + virtual void load(std::istream&) = 0; +}; + +struct SourceCode final : public Entity { + SourceCode() { + } + SourceCode(std::string code_) : code{std::move(code_)} { + } + + std::string code; + + EntityType type() const override { + return EntityType::SourceCode; + } + + void store(std::ostream&) const override; + void load(std::istream&) override; +}; +struct ObjectCode final : public Entity { + ObjectCode() { + } + ObjectCode(std::vector code_) : code{std::move(code_)} { + } + + std::vector code; + + EntityType type() const override { + return EntityType::ObjectCode; + } + + void store(std::ostream&) const override; + void load(std::istream&) override; +}; +struct FuncDescs final : public Entity { + FuncDescs() { + } + FuncDescs(std::unordered_map> descs_) + : descs{std::move(descs_)} { + } + + std::unordered_map> descs; + + EntityType type() const override { + return EntityType::FuncDescs; + } + + void store(std::ostream&) const override; + void load(std::istream&) override; +}; +struct GlobalDescs final : public Entity { + GlobalDescs() { + } + GlobalDescs( + std::unordered_map> descs_) + : descs{std::move(descs_)} { + } + + std::unordered_map> descs; + + EntityType type() const override { + return EntityType::GlobalDescs; + } + + void store(std::ostream&) const override; + void load(std::istream&) override; +}; +struct TypeHierarchy final : public Entity { + TypeHierarchy() { + } + TypeHierarchy(RootInfo root_, ::TypeHierarchy th_) + : root{std::move(root_)}, th{std::move(th_)} { + } + + ::RootInfo root; + ::TypeHierarchy th; + + EntityType type() const override { + return EntityType::TypeHierarchy; + } + + void store(std::ostream&) const override; + void load(std::istream&) override; +}; +struct PaddingInfo final : public Entity { + PaddingInfo() { + } + PaddingInfo(std::map info_) + : info{std::move(info_)} { + } + + std::map info; + + EntityType type() const override { + return EntityType::PaddingInfo; + } + + void store(std::ostream&) const override; + void load(std::istream&) override; +}; + +} // namespace oi::detail::cache diff --git a/oi/cache/GobsBuilder.h b/oi/cache/GobsBuilder.h new file mode 100644 index 0000000..b089aae --- /dev/null +++ b/oi/cache/GobsBuilder.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "oi/cache/Builder.h" + +namespace oi::detail::cache { + +class GobsBuilderImpl; + +class GobsBuilder : public Builder { + public: + GobsBuilder(); + + Response build(const Request&) override; + + private: + std::unique_ptr impl_; +}; + +} // namespace oi::detail::cache diff --git a/oi/cache/LocalCache.cpp b/oi/cache/LocalCache.cpp new file mode 100644 index 0000000..fc6b27d --- /dev/null +++ b/oi/cache/LocalCache.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "LocalCache.h" + +#include +#include // TODO: remove +#include + +namespace oi::detail::cache { +namespace { + +std::string genHash(const Source::Request& req); +std::string_view getExtension(const Entity& en); + +} // namespace + +bool LocalCache::read(const Source::Request& req, Entity& en) { + std::string name = genHash(req); + name.append(getExtension(en)); + std::filesystem::path path = cache_dir_ / name; + + if (!std::filesystem::exists(path)) + return false; + + std::ifstream ifs{path}; + en.load(ifs); + + return true; +} + +bool LocalCache::write(const Source::Request& req, const Entity& en) { + std::string name = genHash(req); + name.append(getExtension(en)); + std::filesystem::path path = cache_dir_ / name; + + std::filesystem::create_directories(cache_dir_); + + std::ofstream of{path}; + en.store(of); + + return true; +} + +namespace { + +std::string genHash(const Source::Request& req) { + return std::to_string(std::hash{}(req)); +} + +std::string_view getExtension(const Entity& en) { + switch (en.type()) { + case EntityType::SourceCode: + return ".cc"; + case EntityType::ObjectCode: + return ".o"; + case EntityType::FuncDescs: + return ".fd"; + case EntityType::GlobalDescs: + return ".gd"; + case EntityType::TypeHierarchy: + return ".th"; + case EntityType::PaddingInfo: + return ".pd"; + } +} + +} // namespace +} // namespace oi::detail::cache diff --git a/oi/cache/LocalCache.h b/oi/cache/LocalCache.h new file mode 100644 index 0000000..b8b1f02 --- /dev/null +++ b/oi/cache/LocalCache.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "oi/cache/Source.h" + +namespace oi::detail::cache { + +class LocalCache : public Source { + public: + LocalCache(std::filesystem::path cache_dir) + : cache_dir_{std::move(cache_dir)} { + } + ~LocalCache() override = default; + + bool read(const Request& req, Entity& en) override; + bool write(const Request& req, const Entity& en) override; + + private: + std::filesystem::path cache_dir_; +}; + +} // namespace oi::detail::cache diff --git a/oi/cache/ManifoldCache.h b/oi/cache/ManifoldCache.h new file mode 100644 index 0000000..b2be46f --- /dev/null +++ b/oi/cache/ManifoldCache.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "oi/cache/Source.h" + +namespace oi::detail::cache { + +class ManifoldCacheImpl; + +class ManifoldCache : public Source { + public: + ManifoldCache(); + + bool read(const Request& req, Entity& en) override; + bool write(const Request& req, const Entity& en) override; + + void setReadable(bool); + void setWriteable(bool); + + private: + std::unique_ptr impl_; +}; + +} // namespace oi::detail::cache diff --git a/oi/cache/Source.h b/oi/cache/Source.h new file mode 100644 index 0000000..56c87de --- /dev/null +++ b/oi/cache/Source.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +#include +#include +#include + +#include "oi/Features.h" +#include "oi/OIParser.h" +#include "oi/cache/Entity.h" + +namespace oi::detail::cache { + +// Source +// +// A source of cached Entities. May be implemented as a local or remote +// cache. Not expected to build a cache if one is not found. +class Source { + public: + struct Request { + FeatureSet features; + irequest probe; + + pid_t traceePid = -1; + std::optional buildId; + }; + + virtual ~Source() = default; + + // Read and fill an Entity from the cache + // + // Return true if the operation did not fail. This includes if this Source is + // intentionally write-only. + virtual bool read(const Request&, Entity&) { + return true; + } + + // Write a filled Entity to the cache + // + // Return true if the operation did not fail. This includes if this Source is + // intentionally read-only. + virtual bool write(const Request&, const Entity&) { + return true; + } +}; + +} // namespace oi::detail::cache + +namespace std { + +template <> +struct hash { + size_t operator()(const oi::detail::cache::Source::Request& req) const { + return folly::hash::hash_combine(req.features, req.probe, req.buildId); + } +}; + +} // namespace std diff --git a/oi/support/MemoryFile.cpp b/oi/support/MemoryFile.cpp new file mode 100644 index 0000000..7f08ee9 --- /dev/null +++ b/oi/support/MemoryFile.cpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "MemoryFile.h" + +#include +#include + +#include +#include +#include +#include + +MemoryFile::MemoryFile(const char* name) { + fd_ = memfd_create(name, 0); + if (fd_ == -1) + throw std::runtime_error(std::string("memfd creation failed: ") + + std::strerror(errno)); +} + +MemoryFile::~MemoryFile() { + if (fd_ == -1) + return; + + PLOG_IF(ERROR, close(fd_) == -1) << "memfd close failed"; +} + +std::filesystem::path MemoryFile::path() { + return {(boost::format("/dev/fd/%1%") % fd_).str()}; +} diff --git a/oi/support/MemoryFile.h b/oi/support/MemoryFile.h new file mode 100644 index 0000000..85023db --- /dev/null +++ b/oi/support/MemoryFile.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +class MemoryFile { + public: + MemoryFile(const char* name); + ~MemoryFile(); + + std::filesystem::path path(); + + private: + int fd_ = -1; +}; diff --git a/test/integration/runner_common.cpp b/test/integration/runner_common.cpp index c86bd59..e7a0827 100644 --- a/test/integration/runner_common.cpp +++ b/test/integration/runner_common.cpp @@ -207,6 +207,7 @@ OidProc OidIntegration::runOidOnProcess(OidOpts opts, "--dump-json"s, "--script-source"s, opts.scriptSource, "--mode=strict"s, + "--cache-path=."s, }; // clang-format on diff --git a/test/test_compiler.cpp b/test/test_compiler.cpp index cf9f48d..899850b 100644 --- a/test/test_compiler.cpp +++ b/test/test_compiler.cpp @@ -43,11 +43,11 @@ TEST(CompilerTest, CompileAndRelocate) { EXPECT_NE(mkdtemp(const_cast(tmpdir.c_str())), nullptr); auto sourcePath = tmpdir / "src.cpp"; - auto objectPath = tmpdir / "obj.o"; + std::vector objectContent; OICompiler compiler{symbols, {}}; - EXPECT_TRUE(compiler.compile(code, sourcePath, objectPath)); + EXPECT_TRUE(compiler.compile(code, sourcePath, objectContent)); const size_t relocSlabSize = 4096; void* relocSlab = mmap(nullptr, @@ -59,7 +59,7 @@ TEST(CompilerTest, CompileAndRelocate) { EXPECT_NE(relocSlab, nullptr); auto relocResult = - compiler.applyRelocs((uintptr_t)relocSlab, {objectPath}, {}); + compiler.applyRelocs((uintptr_t)relocSlab, {objectContent}, {}); EXPECT_TRUE(relocResult.has_value()); auto& [_, segs, jitSymbols] = relocResult.value(); @@ -121,14 +121,14 @@ TEST(CompilerTest, CompileAndRelocateMultipleObjs) { EXPECT_NE(mkdtemp(const_cast(tmpdir.c_str())), nullptr); auto sourceXPath = tmpdir / "srcX.cpp"; - auto objectXPath = tmpdir / "objX.o"; + std::vector objectXContent; auto sourceYPath = tmpdir / "srcY.cpp"; - auto objectYPath = tmpdir / "objY.o"; + std::vector objectYContent; OICompiler compiler{symbols, {}}; - EXPECT_TRUE(compiler.compile(codeX, sourceXPath, objectXPath)); - EXPECT_TRUE(compiler.compile(codeY, sourceYPath, objectYPath)); + EXPECT_TRUE(compiler.compile(codeX, sourceXPath, objectXContent)); + EXPECT_TRUE(compiler.compile(codeY, sourceYPath, objectYContent)); const size_t relocSlabSize = 8192; void* relocSlab = mmap(nullptr, @@ -139,8 +139,8 @@ TEST(CompilerTest, CompileAndRelocateMultipleObjs) { 0); EXPECT_NE(relocSlab, nullptr); - auto relocResult = compiler.applyRelocs( - (uintptr_t)relocSlab, {objectXPath, objectYPath}, {}); + auto relocResult = compiler.applyRelocs((uintptr_t)relocSlab, + {objectXContent, objectYContent}, {}); EXPECT_TRUE(relocResult.has_value()); auto& [_, segs, jitSymbols] = relocResult.value();