diff --git a/.circleci/config.yml b/.circleci/config.yml index 589245a..dba98d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,20 +83,26 @@ jobs: libgtest-dev \ libjemalloc-dev \ libmsgpack-dev \ + libomp-12-dev \ + liburing-dev \ libzstd-dev \ llvm-15-dev \ ninja-build \ pkg-config \ python3-setuptools pip3 install toml + + # Ubuntu 22.04 CMake is too old and we don't have a newer image yet + git clone --depth 1 --branch v3.30.2 https://github.com/Kitware/CMake.git /tmp/cmake + (cd /tmp/cmake && cmake -B build/ -G Ninja && cmake --build build/) environment: DEBIAN_FRONTEND: noninteractive - checkout - run: name: Build command: | - cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >> - cmake --build build/ + /tmp/cmake/build/bin/cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >> + ninja -C build/ # Testing rubbish: cp test/ci.oid.toml build/testing.oid.toml - persist_to_workspace: @@ -131,10 +137,12 @@ jobs: sudo apt-get install -y \ clang-15 \ libboost-all-dev \ - libgflags-dev \ - llvm-15-dev \ libfmt-dev \ - libjemalloc-dev + libgflags-dev \ + libgoogle-glog-dev \ + libjemalloc-dev \ + libomp-12-dev \ + llvm-15-dev environment: DEBIAN_FRONTEND: noninteractive - run: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..071a433 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,54 @@ +name: CI +on: + pull_request: +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.7 + - uses: cachix/install-nix-action@v27 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + - name: nix fmt + run: |- + nix --experimental-features 'nix-command flakes' fmt + git diff --exit-code + + build-test: + runs-on: 16-core-ubuntu + strategy: + matrix: + llvm_version: [15] + steps: + - uses: actions/checkout@v4.1.7 + - uses: cachix/install-nix-action@v27 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + - name: build (LLVM ${{ matrix.llvm_version }}) + # Run the build manually in `nix develop` to keep non-outputs around + run: | + nix develop .#oid-llvm${{ matrix.llvm_version }} --command cmake -B build -G Ninja -DWITH_FLAKY_TESTS=Off -DFORCE_BOOST_STATIC=Off + nix develop .#oid-llvm${{ matrix.llvm_version }} --command ninja -C build + - name: test (LLVM ${{ matrix.llvm_version }}) + env: + # disable drgn multithreading as tests are already run in parallel + OMP_NUM_THREADS: 1 + run: | + echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope + nix develop .#oid-llvm${{ matrix.llvm_version }} --command ./tools/config_gen.py -c clang++ build/testing.oid.toml + nix develop .#oid-llvm${{ matrix.llvm_version }} --command ctest \ + --test-dir build/test/ \ + --test-action Test \ + --parallel \ + --no-compress-output \ + --schedule-random \ + --timeout 90 \ + --repeat until-pass:3 \ + --exclude-from-file ../../.github/workflows/tests_failing_under_nix.txt \ + --output-junit results.xml + - name: upload results + uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: test-results-${{ matrix.llvm_version }} + path: build/test/results.xml diff --git a/.github/workflows/object-introspection.yml b/.github/workflows/object-introspection.yml deleted file mode 100644 index 70b41aa..0000000 --- a/.github/workflows/object-introspection.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: facebookexperimental/object-introspection -on: - pull_request: -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4.1.0 - - uses: cachix/install-nix-action@v25 - with: - github_access_token: ${{ secrets.GITHUB_TOKEN }} - - name: nix fmt - run: |- - nix --experimental-features 'nix-command flakes' fmt - git diff --exit-code diff --git a/.github/workflows/test-report.yml b/.github/workflows/test-report.yml new file mode 100644 index 0000000..91524b1 --- /dev/null +++ b/.github/workflows/test-report.yml @@ -0,0 +1,20 @@ +name: 'Test Report' +on: + workflow_run: + workflows: ['CI'] + types: + - completed +permissions: + contents: read + actions: read + checks: write +jobs: + report: + runs-on: ubuntu-latest + steps: + - uses: dorny/test-reporter@v1 + with: + artifact: test-results-15 + name: CTest Tests + path: results.xml + reporter: jest-junit diff --git a/.github/workflows/tests_failing_under_nix.txt b/.github/workflows/tests_failing_under_nix.txt new file mode 100644 index 0000000..ed514f4 --- /dev/null +++ b/.github/workflows/tests_failing_under_nix.txt @@ -0,0 +1,235 @@ +AddChildrenTest.InheritancePolymorphic +ClangTypeParserTest.MemberAlignment +ClangTypeParserTest.SimpleStruct +DrgnParserTest.ClassTemplateInt +DrgnParserTest.Container +DrgnParserTest.TemplateEnumValue +DrgnParserTest.TemplateEnumValueGaps +DrgnParserTest.TemplateEnumValueNegative +OidIntegration.alignment_wrapper_member_alignment +OidIntegration.alignment_wrapper_member_lower +OidIntegration.alignment_wrapper_member_override +OidIntegration.alignment_wrapper_struct +OidIntegration.alignment_wrapper_two_members +OidIntegration.alignment_wrapper_union_member +OidIntegration.anonymous_anon_struct +OidIntegration.anonymous_anon_typedef +OidIntegration.anonymous_anon_union +OidIntegration.anonymous_nested_anon_struct +OidIntegration.anonymous_regular_struct +OidIntegration.arrays_member_int0 +OidIntegration.arrays_member_int10 +OidIntegration.arrays_multidim +OidIntegration.arrays_multidim_legacy +OidIntegration.bitfields_enum +OidIntegration.bitfields_mixed +OidIntegration.bitfields_single +OidIntegration.bitfields_straddle_bytes +OidIntegration.bitfields_within_bytes +OidIntegration.bitfields_zero_bits +OidIntegration.cycles_raw_ptr +OidIntegration.cycles_raw_ptr_wrapped +OidIntegration.cycles_shared_ptr +OidIntegration.cycles_unique_ptr +OidIntegration.enums_params_scoped_enum_val +OidIntegration.enums_params_scoped_enum_val_cast +OidIntegration.enums_params_scoped_enum_val_gaps +OidIntegration.enums_params_scoped_enum_val_negative +OidIntegration.enums_params_unscoped_enum_val_cast +OidIntegration.fbstring_empty +OidIntegration.fbstring_heap_allocated +OidIntegration.fbstring_inline +OidIntegration.fbstring_string_pooled_unique +OidIntegration.folly_f14_fast_map_a +OidIntegration.folly_f14_fast_set_a +OidIntegration.folly_f14_node_map_a +OidIntegration.folly_f14_node_set_a +OidIntegration.folly_f14_value_map_a +OidIntegration.folly_f14_value_set_a +OidIntegration.folly_f14_vector_map_a +OidIntegration.folly_f14_vector_set_a +OidIntegration.folly_small_vector_int_always_heap +OidIntegration.folly_small_vector_int_default_empty +OidIntegration.folly_small_vector_int_default_inlined +OidIntegration.folly_small_vector_int_default_overflow +OidIntegration.folly_small_vector_vector_3_empty +OidIntegration.folly_small_vector_vector_3_inlined +OidIntegration.folly_small_vector_vector_3_overflow +OidIntegration.folly_sorted_vector_map_int_int_empty +OidIntegration.folly_sorted_vector_map_int_int_reserve +OidIntegration.folly_sorted_vector_map_int_int_some +OidIntegration.ignored_member +OidIntegration.ignored_roottype +OidIntegration.ignored_subtype +OidIntegration.inheritance_access_private +OidIntegration.inheritance_access_protected +OidIntegration.inheritance_access_public +OidIntegration.inheritance_access_public_as_base +OidIntegration.inheritance_multiple_a +OidIntegration.inheritance_polymorphic_a_as_a +OidIntegration.inheritance_polymorphic_b_as_a +OidIntegration.inheritance_polymorphic_b_as_b +OidIntegration.inheritance_polymorphic_c_as_a +OidIntegration.inheritance_polymorphic_c_as_b +OidIntegration.inheritance_polymorphic_c_as_c +OidIntegration.inheritance_polymorphic_diamond_child_as_child +OidIntegration.inheritance_polymorphic_diamond_child_as_middle1 +OidIntegration.inheritance_polymorphic_diamond_child_as_middle1_root +OidIntegration.inheritance_polymorphic_diamond_child_as_middle2 +OidIntegration.inheritance_polymorphic_diamond_child_as_middle2_root +OidIntegration.inheritance_polymorphic_diamond_middle1_as_middle1 +OidIntegration.inheritance_polymorphic_diamond_middle1_as_root +OidIntegration.inheritance_polymorphic_diamond_middle2_as_middle2 +OidIntegration.inheritance_polymorphic_diamond_middle2_as_root +OidIntegration.inheritance_polymorphic_diamond_root_as_root +OidIntegration.inheritance_polymorphic_non_dynamic_base_a_as_a +OidIntegration.inheritance_polymorphic_non_dynamic_base_a_no_polymorphic +OidIntegration.inheritance_polymorphic_non_dynamic_base_b_as_a +OidIntegration.inheritance_polymorphic_non_dynamic_base_b_as_b +OidIntegration.inheritance_polymorphic_non_dynamic_base_b_no_polymorphic +OidIntegration.inheritance_polymorphic_non_dynamic_base_c_as_a +OidIntegration.inheritance_polymorphic_non_dynamic_base_c_as_b +OidIntegration.inheritance_polymorphic_non_dynamic_base_c_as_c +OidIntegration.inheritance_polymorphic_non_dynamic_base_c_no_polymorphic +OidIntegration.multi_arg_tb_all_fail_crashes +OidIntegration.multi_arg_tb_fail_first_arg +OidIntegration.namespaces_queue +OidIntegration.namespaces_stack +OidIntegration.packed_a +OidIntegration.padding_bool_padding +OidIntegration.padding_nested_padding +OidIntegration.pointers_feature_config +OidIntegration.pointers_feature_flag_disabled +OidIntegration.pointers_incomplete_containing_struct +OidIntegration.pointers_incomplete_containing_struct_no_follow +OidIntegration.pointers_incomplete_shared_ptr +OidIntegration.pointers_incomplete_shared_ptr_null +OidIntegration.pointers_incomplete_unique_ptr +OidIntegration.pointers_incomplete_unique_ptr_null +OidIntegration.pointers_struct_primitive_ptrs +OidIntegration.pointers_struct_primitive_ptrs_no_follow +OidIntegration.pointers_struct_primitive_ptrs_null +OidIntegration.pointers_struct_vector_ptr +OidIntegration.pointers_struct_vector_ptr_no_follow +OidIntegration.pointers_struct_vector_ptr_null +OidIntegration.pointers_vector_of_pointers +OidIntegration.primitives_long_double +OidIntegration.simple_class +OidIntegration.simple_struct +OidIntegration.simple_union +OidIntegration.sorted_vector_set_no_ints +OidIntegration.sorted_vector_set_some_ints +OidIntegration.std_array_uint64_length_0 +OidIntegration.std_array_uint64_length_1 +OidIntegration.std_array_uint64_length_8 +OidIntegration.std_array_vector_length_1 +OidIntegration.std_array_vector_length_2 +OidIntegration.std_conditional_a +OidIntegration.std_deque_del_allocator_a +OidIntegration.std_deque_deque_int_empty +OidIntegration.std_deque_deque_int_some +OidIntegration.std_deque_int_empty +OidIntegration.std_deque_int_some +OidIntegration.std_list_del_allocator_a +OidIntegration.std_list_int_empty +OidIntegration.std_list_int_some +OidIntegration.std_list_list_int_empty +OidIntegration.std_list_list_int_some +OidIntegration.std_list_struct_some +OidIntegration.std_map_custom_comparator_a +OidIntegration.std_multimap_custom_comparator_a +OidIntegration.std_multiset_custom_comparator_a +OidIntegration.std_optional_uint64_empty +OidIntegration.std_optional_uint64_present +OidIntegration.std_optional_vector_empty +OidIntegration.std_optional_vector_present +OidIntegration.std_pair_uint64_uint32 +OidIntegration.std_pair_uint64_uint64 +OidIntegration.std_pair_vector_vector +OidIntegration.std_priority_queue_adapter_deque_empty +OidIntegration.std_priority_queue_adapter_deque_some +OidIntegration.std_priority_queue_int_empty +OidIntegration.std_priority_queue_int_some +OidIntegration.std_queue_adapter_vector_empty +OidIntegration.std_queue_adapter_vector_some +OidIntegration.std_queue_int_empty +OidIntegration.std_queue_int_some +OidIntegration.std_queue_queue_int_empty +OidIntegration.std_queue_queue_int_some +OidIntegration.std_reference_wrapper_int +OidIntegration.std_reference_wrapper_vector +OidIntegration.std_set_custom_comparator_a +OidIntegration.std_smart_ptr_shared_ptr_const_uint64_empty +OidIntegration.std_smart_ptr_shared_ptr_const_vector_empty +OidIntegration.std_smart_ptr_shared_ptr_uint64_empty +OidIntegration.std_smart_ptr_shared_ptr_uint64_present +OidIntegration.std_smart_ptr_shared_ptr_vector_empty +OidIntegration.std_smart_ptr_shared_ptr_vector_present +OidIntegration.std_smart_ptr_shared_ptr_void_empty +OidIntegration.std_smart_ptr_shared_ptr_void_present +OidIntegration.std_smart_ptr_unique_ptr_const_uint64_empty +OidIntegration.std_smart_ptr_unique_ptr_const_vector_empty +OidIntegration.std_smart_ptr_unique_ptr_uint64_empty +OidIntegration.std_smart_ptr_unique_ptr_uint64_present +OidIntegration.std_smart_ptr_unique_ptr_vector_empty +OidIntegration.std_smart_ptr_unique_ptr_vector_present +OidIntegration.std_smart_ptr_unique_ptr_void_empty +OidIntegration.std_smart_ptr_unique_ptr_void_present +OidIntegration.std_smart_ptr_weak_ptr_int64_empty +OidIntegration.std_smart_ptr_weak_ptr_int64_expired +OidIntegration.std_smart_ptr_weak_ptr_int64_expired_chase +OidIntegration.std_smart_ptr_weak_ptr_int64_present +OidIntegration.std_smart_ptr_weak_ptr_int64_present_chase +OidIntegration.std_smart_ptr_weak_ptr_int64_void_empty +OidIntegration.std_stack_adapter_vector_empty +OidIntegration.std_stack_adapter_vector_some +OidIntegration.std_stack_int_empty +OidIntegration.std_stack_int_some +OidIntegration.std_stack_stack_int_empty +OidIntegration.std_stack_stack_int_some +OidIntegration.std_string_empty +OidIntegration.std_string_heap_allocated +OidIntegration.std_string_sso +OidIntegration.std_tuple_uint64_uint64 +OidIntegration.std_unordered_map_custom_operator_a +OidIntegration.std_unordered_multimap_custom_operator_a +OidIntegration.std_unordered_multiset_custom_operator_a +OidIntegration.std_unordered_set_custom_operator_a +OidIntegration.std_variant_256_params_256 +OidIntegration.std_variant_256_params_empty +OidIntegration.std_variant_char_int64_1 +OidIntegration.std_variant_char_int64_2 +OidIntegration.std_variant_empty +OidIntegration.std_variant_optional +OidIntegration.std_variant_vector_int_1 +OidIntegration.std_variant_vector_int_2 +OidIntegration.std_vector_del_allocator_a +OidIntegration.std_vector_int_empty +OidIntegration.std_vector_int_some +OidIntegration.std_vector_reserve +OidIntegration.std_vector_struct_some +OidIntegration.std_vector_vector_int_empty +OidIntegration.std_vector_vector_int_some +OidIntegration.templates_int +OidIntegration.templates_two +OidIntegration.templates_value +OidIntegration.templates_vector +OidIntegration.typedefed_parent_multilevel_typedef_parent +OidIntegration.typedefed_parent_simple_typedef_parent +OidIntegration.typedefs_anonymous +OidIntegration.typedefs_container +OidIntegration.unions_alignment +OidIntegration.unions_int +OidIntegration.unions_tagged_int +OidIntegration.unions_tagged_unordered_map +OidIntegration.unions_tagged_vector +OidIntegration.unions_unordered_map +OidIntegration.unions_vector +OilIntegration.folly_f14_fast_map_a +OilIntegration.folly_f14_fast_set_a +OilIntegration.folly_f14_node_map_a +OilIntegration.folly_f14_node_set_a +OilIntegration.folly_f14_value_map_a +OilIntegration.folly_f14_value_set_a +OilIntegration.folly_f14_vector_map_a +OilIntegration.folly_f14_vector_set_a diff --git a/CMakeLists.txt b/CMakeLists.txt index 8674d46..de8f86f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ # object-introspection -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.24) project(object-introspection) # Lets find_program() locate SETUID binaries @@ -33,75 +33,107 @@ find_package(gflags REQUIRED) ### tomlplusplus (for configuration files) FetchContent_Declare( tomlplusplus - GIT_REPOSITORY https://github.com/marzer/tomlplusplus.git - GIT_TAG 4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0 - GIT_PROGRESS TRUE + GIT_REPOSITORY https://github.com/marzer/tomlplusplus.git + GIT_TAG 4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0 + GIT_PROGRESS TRUE + FIND_PACKAGE_ARGS ) FetchContent_MakeAvailable(tomlplusplus) ### glog -FetchContent_Declare( - glog - GIT_REPOSITORY https://github.com/google/glog.git - GIT_TAG 96a2f23dca4cc7180821ca5f32e526314395d26a - GIT_PROGRESS TRUE -) -FetchContent_MakeAvailable(glog) +find_package(glog) +if (NOT glog_FOUND) + FetchContent_Declare( + glog + GIT_REPOSITORY https://github.com/google/glog.git + GIT_TAG 96a2f23dca4cc7180821ca5f32e526314395d26a + GIT_PROGRESS TRUE + ) + FetchContent_MakeAvailable(glog) -# These glog executable targets still generate warnings - disable warnings for -# them explicitly -target_compile_options(demangle_unittest PRIVATE "-w") -target_compile_options(logging_unittest PRIVATE "-w") -target_compile_options(stl_logging_unittest PRIVATE "-w") -target_compile_options(symbolize_unittest PRIVATE "-w") -target_compile_options(utilities_unittest PRIVATE "-w") + # These glog executable targets still generate warnings - disable warnings for + # them explicitly + target_compile_options(demangle_unittest PRIVATE "-w") + target_compile_options(logging_unittest PRIVATE "-w") + target_compile_options(stl_logging_unittest PRIVATE "-w") + target_compile_options(symbolize_unittest PRIVATE "-w") + target_compile_options(utilities_unittest PRIVATE "-w") +endif() -### googletest +### GTest # Do this in the main file so it can be fetched before setting project warnings. # After this is fixed with FetchContent, move to test/CMakeLists.txt. FetchContent_Declare( - googletest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG 1ed6a8c67a0bd675149ece27bbec0ef1759854cf - GIT_PROGRESS TRUE + GTest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG 1ed6a8c67a0bd675149ece27bbec0ef1759854cf + GIT_PROGRESS TRUE + FIND_PACKAGE_ARGS ) -FetchContent_MakeAvailable(googletest) +FetchContent_MakeAvailable(GTest) + +### liburing (for RocksDB) +find_package(uring REQUIRED) ### rocksdb -FetchContent_Declare( - rocksdb - GIT_REPOSITORY https://github.com/facebook/rocksdb.git - GIT_TAG f32521662acf3352397d438b732144c7813bbbec # v8.5.3 - GIT_PROGRESS TRUE -) -FetchContent_Populate(rocksdb) -add_custom_target(librocksdb ALL - WORKING_DIRECTORY ${rocksdb_SOURCE_DIR} - COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off - COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb - BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a - COMMENT "Building RocksDB" - USES_TERMINAL -) -include_directories(SYSTEM "${rocksdb_SOURCE_DIR}/include") +find_package(RocksDB 8.11 CONFIG) +if (NOT RocksDB_FOUND) + FetchContent_Declare( + rocksdb + GIT_REPOSITORY https://github.com/facebook/rocksdb.git + GIT_TAG f32521662acf3352397d438b732144c7813bbbec # v8.5.3 + GIT_PROGRESS TRUE + ) + FetchContent_Populate(rocksdb) + + add_custom_target(librocksdb_build ALL + WORKING_DIRECTORY ${rocksdb_SOURCE_DIR} + COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off + COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb + BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a + COMMENT "Building RocksDB" + USES_TERMINAL + ) + + ### zstd (for rocksdb) + find_package(zstd REQUIRED) + + add_library(librocksdb INTERFACE) + add_dependencies(librocksdb librocksdb_build) + target_include_directories(librocksdb INTERFACE SYSTEM "${rocksdb_SOURCE_DIR}/include") + target_link_libraries(librocksdb INTERFACE ${rocksdb_BINARY_DIR}/librocksdb.a zstd::zstd) + + add_library(RocksDB::rocksdb ALIAS librocksdb) +endif() ### folly ### use folly as a header only library. some features won't be supported. -FetchContent_Declare( - folly - GIT_REPOSITORY https://github.com/facebook/folly.git - GIT_TAG c5aa5c46291a27f69acc920894d43605ceb43eba - GIT_PROGRESS TRUE - PATCH_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h -) -FetchContent_Populate(folly) +find_package(folly CONFIG) +if (NOT folly_FOUND) + FetchContent_Declare( + folly + GIT_REPOSITORY https://github.com/facebook/folly.git + GIT_TAG c5aa5c46291a27f69acc920894d43605ceb43eba + GIT_PROGRESS TRUE + PATCH_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h + ) + FetchContent_Populate(folly) + + add_library(folly_headers INTERFACE) + target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR}) + target_link_libraries(folly_headers INTERFACE Boost::headers) +else() + add_library(folly_headers INTERFACE) + target_include_directories(folly_headers SYSTEM INTERFACE ${FOLLY_INCLUDE_DIR}) +endif() ### range-v3 FetchContent_Declare( range-v3 - GIT_REPOSITORY https://github.com/ericniebler/range-v3.git - GIT_TAG a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0 - GIT_PROGRESS TRUE + GIT_REPOSITORY https://github.com/ericniebler/range-v3.git + GIT_TAG a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0 + GIT_PROGRESS TRUE + FIND_PACKAGE_ARGS ) FetchContent_MakeAvailable(range-v3) @@ -124,6 +156,18 @@ endif() ### (Re)download submodules find_package(Git QUIET) +if (DEFINED drgn_SOURCE_DIR) + # drgn's autotools build requires source modification. in case we have a + # readonly source (read: nix) we must copy this. do this always to avoid + # polluting the source. + file(COPY "${drgn_SOURCE_DIR}/" DESTINATION "${FETCHCONTENT_BASE_DIR}/drgn-src" NO_SOURCE_PERMISSIONS) + SET(drgn_SOURCE_DIR "${FETCHCONTENT_BASE_DIR}/drgn-src") + SET(drgn_BINARY_DIR "${CMAKE_BINARY_DIR}/_deps/drgn-build") +else() + SET(drgn_SOURCE_DIR "${PROJECT_SOURCE_DIR}/extern/drgn") + SET(drgn_BINARY_DIR "${drgn_SOURCE_DIR}/build") +endif() + # TODO: No idea if this huge block is required, just picked from an example. There may be a short-hand. if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git") # Update submodules as needed @@ -145,20 +189,17 @@ if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git") if(NOT GIT_SUBMOD_RESULT EQUAL "0") message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules") endif() + + if(NOT EXISTS "${drgn_SOURCE_DIR}") + message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.") + endif() endif() endif() -if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn") - message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.") -endif() ### Select Python version find_program(PYTHON NAMES python3.9 python3) -add_library(folly_headers INTERFACE) -target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR}) -target_link_libraries(folly_headers INTERFACE Boost::headers) - ### bison & flex (for oid_parser) find_package(BISON 3.5 REQUIRED) find_package(FLEX) @@ -194,8 +235,11 @@ find_package(msgpack REQUIRED CONFIG) get_target_property(MSGPACK_INCLUDE_DIRS msgpackc INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${MSGPACK_INCLUDE_DIRS}) -### zstd (for rocksdb) -find_package(zstd REQUIRED) +### drgn/elfutils dependencies +find_package(BZip2 REQUIRED) +find_package(OpenMP REQUIRED) +find_package(LibLZMA REQUIRED) +find_package(ZLIB REQUIRED) ### drgn # The setup.py script in drgn is really meant to build drgn (python @@ -214,46 +258,38 @@ find_package(zstd REQUIRED) # make # # Since setup.py has a single cmd to do this, just use it for now. -# -# Another extemely annoying point. drgn pretty much has to be compiled with gcc only -# clang-12 does NOT work. clang fails with the following error :- -# configure: error: gcc with GNU99 support required set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no") if (ASAN) list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes") endif() -add_custom_target(libdrgn ALL - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn - COMMAND unset BISON_PKGDATADIR && CC=gcc CFLAGS="${DRGN_CFLAGS}" CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp build - BYPRODUCTS ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/.libs/libdrgnimpl.a - ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdw/libdw.a - ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libelf/libelf.a - ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdwelf/libdwelf.a + +add_custom_target(drgn_build ALL + WORKING_DIRECTORY ${drgn_SOURCE_DIR} + COMMAND unset BISON_PKGDATADIR && CFLAGS="${DRGN_CFLAGS}" + CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp "${drgn_BINARY_DIR}" + BYPRODUCTS ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a + ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a + ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a + ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a COMMENT "Building drgn" USES_TERMINAL ) -set(DRGN_PATH "${PROJECT_SOURCE_DIR}/extern/drgn/build") -# Ideally drgn stuff should be together at the end. But looks like rpath needs -# to be set before add_executable() unfortunately. Maybe split libdrgn stuff -# into a separate file later. -set(CMAKE_SKIP_BUILD_RPATH FALSE) -set(CMAKE_INSTALL_RPATH - "${DRGN_PATH}/.libs" - "${DRGN_PATH}/velfutils/libdw" - "${DRGN_PATH}/velfutils/libelf" - "${DRGN_PATH}/velfutils/libdwelf" -) -set(CMAKE_BUILD_RPATH - "${DRGN_PATH}/.libs" - "${DRGN_PATH}/velfutils/libdw" - "${DRGN_PATH}/velfutils/libelf" - "${DRGN_PATH}/velfutils/libdwelf" -) -set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) +add_library(drgn INTERFACE) +add_dependencies(drgn drgn_build) +target_link_libraries(drgn INTERFACE + ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a + ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a + ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a + ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a -include_directories(SYSTEM "${DRGN_PATH}") + BZip2::BZip2 + LibLZMA::LibLZMA + OpenMP::OpenMP_CXX + ZLIB::ZLIB +) +target_include_directories(drgn SYSTEM INTERFACE "${drgn_SOURCE_DIR}" "${drgn_BINARY_DIR}") if (STATIC_LINK) # glog links against the `gflags` target, which is an alias for `gflags_shared` @@ -289,7 +325,6 @@ add_library(oicore oi/PaddingHunter.cpp oi/Serialize.cpp ) -add_dependencies(oicore libdrgn) target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS}) target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS}) target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) @@ -323,7 +358,6 @@ else() endif() target_link_libraries(oicore - "-L${DRGN_PATH}/.libs" drgn dw pthread @@ -334,11 +368,9 @@ add_library(treebuilder oi/TreeBuilder.cpp oi/exporters/TypeCheckingWalker.cpp ) -add_dependencies(treebuilder librocksdb) target_link_libraries(treebuilder - ${rocksdb_BINARY_DIR}/librocksdb.a + RocksDB::rocksdb oicore # overkill but it does need a lot of stuff - zstd::zstd ) @@ -375,10 +407,8 @@ target_link_libraries(oip oicore) ### Object Introspection RocksDB Printer (OIRP) add_executable(oirp tools/OIRP.cpp) -add_dependencies(oirp librocksdb) target_link_libraries(oirp - ${rocksdb_BINARY_DIR}/librocksdb.a - zstd::zstd + RocksDB::rocksdb msgpackc ) @@ -417,3 +447,6 @@ endif() if (DEFINED ENV{CMAKE_HOOK}) include($ENV{CMAKE_HOOK}) endif() + +install(TARGETS oid DESTINATION ${CMAKE_INSTALL_BINDIR}) + diff --git a/README.md b/README.md index 9f95bf8..abce6f8 100644 --- a/README.md +++ b/README.md @@ -14,3 +14,61 @@ See the [CONTRIBUTING](CONTRIBUTING.md) file for how to help out. ## License Object Introspection is licensed under the [Apache 2.0 License](LICENSE). + +## Getting started with Nix + +Nix is the easiest way to get started with `oid` as it is non-trivial to build otherwise. Explicit Nix support for Object Introspection as a Library will come down the line, but Nix can currently provide you a reproducible development environment in which to build it. + +These examples expect you to have `nix` installed and available with no other dependencies required. Find the installation guide at https://nixos.org/download.html. + +We also required flake support. To enable flakes globally run: + + $ mkdir -p ~/.config/nix + $ echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf + +Or suffix every `nix` command with `nix --extra-experimental-features 'nix-command flakes'`. + +### Run upstream OID without modifying the source + + $ nix run github:facebookexperimental/object-introspection -- --help + +This will download the latest source into your Nix store along with all of its dependencies, running help afterwards. + +### Build OID locally + + $ git clone https://github.com/facebookexperimental/object-introspection + $ nix build + $ ./result/bin/oid --help + +This will build OID from your local sources. Please note that this will NOT pick up changes to `extern/drgn` or `extern/drgn/libdrgn/velfutils`. + +### Get a development environment + + $ nix develop + $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off + $ ninja -C build + $ build/oid --help + +This command provides a development shell with all the required dependencies. This is the most flexible option and will pick up source changes as CMake normally would. + +Sometimes this developer environment can be polluted by things installed on your normal system. If this is an issue, use: + + $ nix develop -i + +This removes the environment from your host system and makes the build pure. + +### Run the tests + + $ nix develop + $ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off + $ ninja -C build + $ ./tools/config_gen.py -c clang++ build/testing.oid.toml + $ ctest -j --test-dir build/test + +Running tests under `nix` is new to the project and may take some time to mature. The CI is the source of truth for now. + +### Format source + + $ nix fmt + +This formats the Nix, C++, and Python code in the repository. diff --git a/cmake/Finduring.cmake b/cmake/Finduring.cmake new file mode 100644 index 0000000..3b007fe --- /dev/null +++ b/cmake/Finduring.cmake @@ -0,0 +1,27 @@ +# - Find liburing +# +# uring_INCLUDE_DIR - Where to find liburing.h +# uring_LIBRARIES - List of libraries when using uring. +# uring_FOUND - True if uring found. + +find_path(uring_INCLUDE_DIR + NAMES liburing.h) +find_library(uring_LIBRARIES + NAMES liburing.a liburing.so) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(uring + DEFAULT_MSG uring_LIBRARIES uring_INCLUDE_DIR) + +mark_as_advanced( + uring_INCLUDE_DIR + uring_LIBRARIES) + +if(uring_FOUND AND NOT TARGET uring::uring) + add_library(uring::uring UNKNOWN IMPORTED) + set_target_properties(uring::uring PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${uring_INCLUDE_DIR}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${uring_LIBRARIES}") +endif() + diff --git a/flake.nix b/flake.nix index 7a3382e..f925621 100644 --- a/flake.nix +++ b/flake.nix @@ -1,5 +1,5 @@ { - description = "A flake for building Object Introspection."; + description = "Object level memory profiler for C++"; inputs = { nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; @@ -18,7 +18,99 @@ treefmt-nix, ... }@inputs: - flake-utils.lib.eachDefaultSystem ( + flake-utils.lib.eachSystem [ flake-utils.lib.system.x86_64-linux ] ( + system: + let + pkgs = import nixpkgs { inherit system; }; + + drgnSrc = pkgs.fetchFromGitHub { + owner = "JakeHillion"; + repo = "drgn"; + rev = "b1f8c3e8526611b6720800250ba858a713dd9e4f"; + hash = "sha256-5WhMHgx/RKtqjxGx4AyiqVKMot5xulr+6c8i2E9IxiA="; + fetchSubmodules = true; + }; + + mkOidPackage = + llvmPackages: + with pkgs; + pkgs.llvmPackages.stdenv.mkDerivation rec { + name = "oid"; + + src = self; + + nativeBuildInputs = [ + autoconf + automake + bison + cmake + flex + gettext + git + hexdump + libtool + ninja + pkgconf + python312 + python312Packages.setuptools + python312Packages.toml + glibcLocales + ]; + + buildInputs = [ + llvmPackages.libclang + llvmPackages.llvm + + boost + bzip2 + curl + double-conversion + elfutils + flex + folly + folly.fmt + gflags + glog + gtest + icu + jemalloc + libarchive + libmicrohttpd + liburing + libxml2 + lzma + msgpack + range-v3 + rocksdb_8_11 + sqlite + tomlplusplus + zstd + + llvmPackages.openmp # should match the stdenv clang version, see: https://github.com/NixOS/nixpkgs/issues/79818 + ]; + + cmakeFlags = [ + "-Ddrgn_SOURCE_DIR=${drgnSrc}" + "-DFORCE_BOOST_STATIC=Off" + ]; + + outputs = [ "out" ]; + }; + in + { + packages = rec { + default = oid-llvm15; + + oid-llvm15 = mkOidPackage pkgs.llvmPackages_15; + }; + + apps.default = { + type = "app"; + program = "${self.packages.${system}.default}/bin/oid"; + }; + } + ) + // flake-utils.lib.eachDefaultSystem ( system: let pkgs = nixpkgs.legacyPackages.${system}; diff --git a/oi/CMakeLists.txt b/oi/CMakeLists.txt index bb73871..7d805e5 100644 --- a/oi/CMakeLists.txt +++ b/oi/CMakeLists.txt @@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus) add_library(drgn_utils DrgnUtils.cpp) target_link_libraries(drgn_utils glog::glog - - "-L${DRGN_PATH}/.libs" drgn ) -add_dependencies(drgn_utils libdrgn) add_library(symbol_service Descs.cpp diff --git a/oi/type_graph/CMakeLists.txt b/oi/type_graph/CMakeLists.txt index 6c4da13..c878e08 100644 --- a/oi/type_graph/CMakeLists.txt +++ b/oi/type_graph/CMakeLists.txt @@ -20,12 +20,9 @@ add_library(type_graph TypeIdentifier.cpp Types.cpp ) -add_dependencies(type_graph libdrgn) target_link_libraries(type_graph container_info symbol_service - - "-L${DRGN_PATH}/.libs" drgn ) target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS}) diff --git a/tools/config_gen.py b/tools/config_gen.py index c20ca40..5ac56ff 100755 --- a/tools/config_gen.py +++ b/tools/config_gen.py @@ -16,6 +16,7 @@ import argparse import getpass +import os import pathlib import subprocess import typing @@ -170,7 +171,10 @@ def pull_base_toml() -> typing.Dict: # Now, we need to replace any placeholders that might be present in the base toml file with the real verisons. user = getpass.getuser() - pwd = str(repo_path.resolve()) + if "IN_NIX_SHELL" in os.environ and "src" in os.environ: + pwd = os.environ['src'] + else: + pwd = str(repo_path.resolve()) container_list = base.get("types", {}).get("containers") if container_list: