JakeHillion/object-introspection
1
0
Fork 0
mirror of https://github.com/JakeHillion/object-introspection.git synced 2024-11-09 21:24:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

nix: add building oid to the flake

Browse Source 
OI's build is challenging and has often been a problem for the Open Source
community. It requires an extremely specific set of dependencies that are very
hard to achieve on most systems. There are frequent breakages, like when
updating to CentOS Stream 9, or when trying to update the CI's clang from
clang-12 to clang-15 - OI requires the clang libraries to be version 15 but
can't be compiled with it on the CI!

This changes provides a mostly working build environment with `nix`. This
environment is pinned to a specific nixpkgs revision using `flake.lock`, and
only updates when we explicitly tell it to.

Summary of changes:
- Update CMakeLists.txt required version to 3.24. This allows specifying
  `FIND_PACKAGE_ARGS` in `FetchContent`, meaning we can use system packages.
  This is available on most up to date distros (3.30.2 is current).
- Extends `flake.nix` to be able to build OI. Adds instructions for building
  and developing OI using `nix`.
- Partially runs the tests in GitHub Actions. A huge amount must be excluded
  because of incompatibilites between the clangStdenv compiler and drgn. We
  have similar, though fewer, issues when building with the clang-12/libstdcxx
  mix on the Ubuntu 22.04 CircleCI, though this is at least reproducible.
- Updates CircleCI to build CMake from source as we don't have a newer image
  available. Also add some newly found dependencies (not sure how it was
  working without them before).

Test plan:

This change requires less testing than previous build related changes because
it deprecates most of the build types.

- The internal BUCK build is unaffected. No special testing.
- The semi-internal CMake build is gone. Use Nix.
- The Nix build for clang-15 and some tests are continuously tested in GitHub
  actions.
- Tested the set of Nix commands in the README. All work except the one that
  points to GitHub as this must be merged first.
- The existing CircleCI runs on Ubuntu 20.04 are maintained.
- Unable to test the new `test-report.yml` as it must be merged due to the
  permissions it needs. Will follow up with testing after this is merged. See:
  https://github.com/dorny/test-reporter?tab=readme-ov-file#recommended-setup-for-public-repositories

The list of exclusions for GitHub Actions/nix testing is currently very long, I
think 29% of the tests. This should be stable and reproducible though, and
likely needs deep changes to OI to fix. That's why fixes are excluded from this
PR. It's all to do with the forked drgn not being able to parse clang's newer
DWARF output, and can't be fixed by rolling back as we required a relatively
new libcxx.
This commit is contained in:
Jake Hillion 2024-08-15 15:48:08 +01:00 committed by Jake Hillion
parent e161d31e7c
commit fe9b4b232a
12 changed files with 637 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
.circleci/config.yml
View File

@ -83,20 +83,26 @@ jobs:
libgtest-dev \ libgtest-dev \
libjemalloc-dev \ libjemalloc-dev \
libmsgpack-dev \ libmsgpack-dev \
libomp-12-dev \
liburing-dev \
libzstd-dev \ libzstd-dev \
llvm-15-dev \ llvm-15-dev \
ninja-build \ ninja-build \
pkg-config \ pkg-config \
python3-setuptools python3-setuptools
pip3 install toml pip3 install toml
# Ubuntu 22.04 CMake is too old and we don't have a newer image yet
git clone --depth 1 --branch v3.30.2 https://github.com/Kitware/CMake.git /tmp/cmake
(cd /tmp/cmake && cmake -B build/ -G Ninja && cmake --build build/)
environment: environment:
DEBIAN_FRONTEND: noninteractive DEBIAN_FRONTEND: noninteractive
- checkout - checkout
- run: - run:
name: Build name: Build
command: | command: |
cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >> /tmp/cmake/build/bin/cmake -G Ninja -B build/ -DWITH_FLAKY_TESTS=Off -DCODE_COVERAGE=On -DWARNINGS_AS_ERRORS=<< parameters.warnings_as_errors >>
cmake --build build/ ninja -C build/
# Testing rubbish: # Testing rubbish:
cp test/ci.oid.toml build/testing.oid.toml cp test/ci.oid.toml build/testing.oid.toml
- persist_to_workspace: - persist_to_workspace:
@ -131,10 +137,12 @@ jobs:
sudo apt-get install -y \ sudo apt-get install -y \
clang-15 \ clang-15 \
libboost-all-dev \ libboost-all-dev \
libgflags-dev \
llvm-15-dev \
libfmt-dev \ libfmt-dev \
libjemalloc-dev libgflags-dev \
libgoogle-glog-dev \
libjemalloc-dev \
libomp-12-dev \
llvm-15-dev
environment: environment:
DEBIAN_FRONTEND: noninteractive DEBIAN_FRONTEND: noninteractive
- run: - run:

54
.github/workflows/ci.yml vendored Normal file
View File

@ -0,0 +1,54 @@
name: CI
on:
pull_request:
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.1.7
- uses: cachix/install-nix-action@v27
with:
github_access_token: ${{ secrets.GITHUB_TOKEN }}
- name: nix fmt
run: |-
nix --experimental-features 'nix-command flakes' fmt
git diff --exit-code
build-test:
runs-on: 16-core-ubuntu
strategy:
matrix:
llvm_version: [15]
steps:
- uses: actions/checkout@v4.1.7
- uses: cachix/install-nix-action@v27
with:
github_access_token: ${{ secrets.GITHUB_TOKEN }}
- name: build (LLVM ${{ matrix.llvm_version }})
# Run the build manually in `nix develop` to keep non-outputs around
run: |
nix develop .#oid-llvm${{ matrix.llvm_version }} --command cmake -B build -G Ninja -DWITH_FLAKY_TESTS=Off -DFORCE_BOOST_STATIC=Off
nix develop .#oid-llvm${{ matrix.llvm_version }} --command ninja -C build
- name: test (LLVM ${{ matrix.llvm_version }})
env:
# disable drgn multithreading as tests are already run in parallel
OMP_NUM_THREADS: 1
run: |
echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
nix develop .#oid-llvm${{ matrix.llvm_version }} --command ./tools/config_gen.py -c clang++ build/testing.oid.toml
nix develop .#oid-llvm${{ matrix.llvm_version }} --command ctest \
--test-dir build/test/ \
--test-action Test \
--parallel \
--no-compress-output \
--schedule-random \
--timeout 90 \
--repeat until-pass:3 \
--exclude-from-file ../../.github/workflows/tests_failing_under_nix.txt \
--output-junit results.xml
- name: upload results
uses: actions/upload-artifact@v4
if: success() || failure()
with:
name: test-results-${{ matrix.llvm_version }}
path: build/test/results.xml

15
.github/workflows/object-introspection.yml vendored
View File

@ -1,15 +0,0 @@
name: facebookexperimental/object-introspection
on:
pull_request:
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.1.0
- uses: cachix/install-nix-action@v25
with:
github_access_token: ${{ secrets.GITHUB_TOKEN }}
- name: nix fmt
run: |-
nix --experimental-features 'nix-command flakes' fmt
git diff --exit-code

20
.github/workflows/test-report.yml vendored Normal file
View File

@ -0,0 +1,20 @@
name: 'Test Report'
on:
workflow_run:
workflows: ['CI']
types:
- completed
permissions:
contents: read
actions: read
checks: write
jobs:
report:
runs-on: ubuntu-latest
steps:
- uses: dorny/test-reporter@v1
with:
artifact: test-results-15
name: CTest Tests
path: results.xml
reporter: jest-junit

235
.github/workflows/tests_failing_under_nix.txt vendored Normal file
View File

@ -0,0 +1,235 @@
AddChildrenTest.InheritancePolymorphic
ClangTypeParserTest.MemberAlignment
ClangTypeParserTest.SimpleStruct
DrgnParserTest.ClassTemplateInt
DrgnParserTest.Container
DrgnParserTest.TemplateEnumValue
DrgnParserTest.TemplateEnumValueGaps
DrgnParserTest.TemplateEnumValueNegative
OidIntegration.alignment_wrapper_member_alignment
OidIntegration.alignment_wrapper_member_lower
OidIntegration.alignment_wrapper_member_override
OidIntegration.alignment_wrapper_struct
OidIntegration.alignment_wrapper_two_members
OidIntegration.alignment_wrapper_union_member
OidIntegration.anonymous_anon_struct
OidIntegration.anonymous_anon_typedef
OidIntegration.anonymous_anon_union
OidIntegration.anonymous_nested_anon_struct
OidIntegration.anonymous_regular_struct
OidIntegration.arrays_member_int0
OidIntegration.arrays_member_int10
OidIntegration.arrays_multidim
OidIntegration.arrays_multidim_legacy
OidIntegration.bitfields_enum
OidIntegration.bitfields_mixed
OidIntegration.bitfields_single
OidIntegration.bitfields_straddle_bytes
OidIntegration.bitfields_within_bytes
OidIntegration.bitfields_zero_bits
OidIntegration.cycles_raw_ptr
OidIntegration.cycles_raw_ptr_wrapped
OidIntegration.cycles_shared_ptr
OidIntegration.cycles_unique_ptr
OidIntegration.enums_params_scoped_enum_val
OidIntegration.enums_params_scoped_enum_val_cast
OidIntegration.enums_params_scoped_enum_val_gaps
OidIntegration.enums_params_scoped_enum_val_negative
OidIntegration.enums_params_unscoped_enum_val_cast
OidIntegration.fbstring_empty
OidIntegration.fbstring_heap_allocated
OidIntegration.fbstring_inline
OidIntegration.fbstring_string_pooled_unique
OidIntegration.folly_f14_fast_map_a
OidIntegration.folly_f14_fast_set_a
OidIntegration.folly_f14_node_map_a
OidIntegration.folly_f14_node_set_a
OidIntegration.folly_f14_value_map_a
OidIntegration.folly_f14_value_set_a
OidIntegration.folly_f14_vector_map_a
OidIntegration.folly_f14_vector_set_a
OidIntegration.folly_small_vector_int_always_heap
OidIntegration.folly_small_vector_int_default_empty
OidIntegration.folly_small_vector_int_default_inlined
OidIntegration.folly_small_vector_int_default_overflow
OidIntegration.folly_small_vector_vector_3_empty
OidIntegration.folly_small_vector_vector_3_inlined
OidIntegration.folly_small_vector_vector_3_overflow
OidIntegration.folly_sorted_vector_map_int_int_empty
OidIntegration.folly_sorted_vector_map_int_int_reserve
OidIntegration.folly_sorted_vector_map_int_int_some
OidIntegration.ignored_member
OidIntegration.ignored_roottype
OidIntegration.ignored_subtype
OidIntegration.inheritance_access_private
OidIntegration.inheritance_access_protected
OidIntegration.inheritance_access_public
OidIntegration.inheritance_access_public_as_base
OidIntegration.inheritance_multiple_a
OidIntegration.inheritance_polymorphic_a_as_a
OidIntegration.inheritance_polymorphic_b_as_a
OidIntegration.inheritance_polymorphic_b_as_b
OidIntegration.inheritance_polymorphic_c_as_a
OidIntegration.inheritance_polymorphic_c_as_b
OidIntegration.inheritance_polymorphic_c_as_c
OidIntegration.inheritance_polymorphic_diamond_child_as_child
OidIntegration.inheritance_polymorphic_diamond_child_as_middle1
OidIntegration.inheritance_polymorphic_diamond_child_as_middle1_root
OidIntegration.inheritance_polymorphic_diamond_child_as_middle2
OidIntegration.inheritance_polymorphic_diamond_child_as_middle2_root
OidIntegration.inheritance_polymorphic_diamond_middle1_as_middle1
OidIntegration.inheritance_polymorphic_diamond_middle1_as_root
OidIntegration.inheritance_polymorphic_diamond_middle2_as_middle2
OidIntegration.inheritance_polymorphic_diamond_middle2_as_root
OidIntegration.inheritance_polymorphic_diamond_root_as_root
OidIntegration.inheritance_polymorphic_non_dynamic_base_a_as_a
OidIntegration.inheritance_polymorphic_non_dynamic_base_a_no_polymorphic
OidIntegration.inheritance_polymorphic_non_dynamic_base_b_as_a
OidIntegration.inheritance_polymorphic_non_dynamic_base_b_as_b
OidIntegration.inheritance_polymorphic_non_dynamic_base_b_no_polymorphic
OidIntegration.inheritance_polymorphic_non_dynamic_base_c_as_a
OidIntegration.inheritance_polymorphic_non_dynamic_base_c_as_b
OidIntegration.inheritance_polymorphic_non_dynamic_base_c_as_c
OidIntegration.inheritance_polymorphic_non_dynamic_base_c_no_polymorphic
OidIntegration.multi_arg_tb_all_fail_crashes
OidIntegration.multi_arg_tb_fail_first_arg
OidIntegration.namespaces_queue
OidIntegration.namespaces_stack
OidIntegration.packed_a
OidIntegration.padding_bool_padding
OidIntegration.padding_nested_padding
OidIntegration.pointers_feature_config
OidIntegration.pointers_feature_flag_disabled
OidIntegration.pointers_incomplete_containing_struct
OidIntegration.pointers_incomplete_containing_struct_no_follow
OidIntegration.pointers_incomplete_shared_ptr
OidIntegration.pointers_incomplete_shared_ptr_null
OidIntegration.pointers_incomplete_unique_ptr
OidIntegration.pointers_incomplete_unique_ptr_null
OidIntegration.pointers_struct_primitive_ptrs
OidIntegration.pointers_struct_primitive_ptrs_no_follow
OidIntegration.pointers_struct_primitive_ptrs_null
OidIntegration.pointers_struct_vector_ptr
OidIntegration.pointers_struct_vector_ptr_no_follow
OidIntegration.pointers_struct_vector_ptr_null
OidIntegration.pointers_vector_of_pointers
OidIntegration.primitives_long_double
OidIntegration.simple_class
OidIntegration.simple_struct
OidIntegration.simple_union
OidIntegration.sorted_vector_set_no_ints
OidIntegration.sorted_vector_set_some_ints
OidIntegration.std_array_uint64_length_0
OidIntegration.std_array_uint64_length_1
OidIntegration.std_array_uint64_length_8
OidIntegration.std_array_vector_length_1
OidIntegration.std_array_vector_length_2
OidIntegration.std_conditional_a
OidIntegration.std_deque_del_allocator_a
OidIntegration.std_deque_deque_int_empty
OidIntegration.std_deque_deque_int_some
OidIntegration.std_deque_int_empty
OidIntegration.std_deque_int_some
OidIntegration.std_list_del_allocator_a
OidIntegration.std_list_int_empty
OidIntegration.std_list_int_some
OidIntegration.std_list_list_int_empty
OidIntegration.std_list_list_int_some
OidIntegration.std_list_struct_some
OidIntegration.std_map_custom_comparator_a
OidIntegration.std_multimap_custom_comparator_a
OidIntegration.std_multiset_custom_comparator_a
OidIntegration.std_optional_uint64_empty
OidIntegration.std_optional_uint64_present
OidIntegration.std_optional_vector_empty
OidIntegration.std_optional_vector_present
OidIntegration.std_pair_uint64_uint32
OidIntegration.std_pair_uint64_uint64
OidIntegration.std_pair_vector_vector
OidIntegration.std_priority_queue_adapter_deque_empty
OidIntegration.std_priority_queue_adapter_deque_some
OidIntegration.std_priority_queue_int_empty
OidIntegration.std_priority_queue_int_some
OidIntegration.std_queue_adapter_vector_empty
OidIntegration.std_queue_adapter_vector_some
OidIntegration.std_queue_int_empty
OidIntegration.std_queue_int_some
OidIntegration.std_queue_queue_int_empty
OidIntegration.std_queue_queue_int_some
OidIntegration.std_reference_wrapper_int
OidIntegration.std_reference_wrapper_vector
OidIntegration.std_set_custom_comparator_a
OidIntegration.std_smart_ptr_shared_ptr_const_uint64_empty
OidIntegration.std_smart_ptr_shared_ptr_const_vector_empty
OidIntegration.std_smart_ptr_shared_ptr_uint64_empty
OidIntegration.std_smart_ptr_shared_ptr_uint64_present
OidIntegration.std_smart_ptr_shared_ptr_vector_empty
OidIntegration.std_smart_ptr_shared_ptr_vector_present
OidIntegration.std_smart_ptr_shared_ptr_void_empty
OidIntegration.std_smart_ptr_shared_ptr_void_present
OidIntegration.std_smart_ptr_unique_ptr_const_uint64_empty
OidIntegration.std_smart_ptr_unique_ptr_const_vector_empty
OidIntegration.std_smart_ptr_unique_ptr_uint64_empty
OidIntegration.std_smart_ptr_unique_ptr_uint64_present
OidIntegration.std_smart_ptr_unique_ptr_vector_empty
OidIntegration.std_smart_ptr_unique_ptr_vector_present
OidIntegration.std_smart_ptr_unique_ptr_void_empty
OidIntegration.std_smart_ptr_unique_ptr_void_present
OidIntegration.std_smart_ptr_weak_ptr_int64_empty
OidIntegration.std_smart_ptr_weak_ptr_int64_expired
OidIntegration.std_smart_ptr_weak_ptr_int64_expired_chase
OidIntegration.std_smart_ptr_weak_ptr_int64_present
OidIntegration.std_smart_ptr_weak_ptr_int64_present_chase
OidIntegration.std_smart_ptr_weak_ptr_int64_void_empty
OidIntegration.std_stack_adapter_vector_empty
OidIntegration.std_stack_adapter_vector_some
OidIntegration.std_stack_int_empty
OidIntegration.std_stack_int_some
OidIntegration.std_stack_stack_int_empty
OidIntegration.std_stack_stack_int_some
OidIntegration.std_string_empty
OidIntegration.std_string_heap_allocated
OidIntegration.std_string_sso
OidIntegration.std_tuple_uint64_uint64
OidIntegration.std_unordered_map_custom_operator_a
OidIntegration.std_unordered_multimap_custom_operator_a
OidIntegration.std_unordered_multiset_custom_operator_a
OidIntegration.std_unordered_set_custom_operator_a
OidIntegration.std_variant_256_params_256
OidIntegration.std_variant_256_params_empty
OidIntegration.std_variant_char_int64_1
OidIntegration.std_variant_char_int64_2
OidIntegration.std_variant_empty
OidIntegration.std_variant_optional
OidIntegration.std_variant_vector_int_1
OidIntegration.std_variant_vector_int_2
OidIntegration.std_vector_del_allocator_a
OidIntegration.std_vector_int_empty
OidIntegration.std_vector_int_some
OidIntegration.std_vector_reserve
OidIntegration.std_vector_struct_some
OidIntegration.std_vector_vector_int_empty
OidIntegration.std_vector_vector_int_some
OidIntegration.templates_int
OidIntegration.templates_two
OidIntegration.templates_value
OidIntegration.templates_vector
OidIntegration.typedefed_parent_multilevel_typedef_parent
OidIntegration.typedefed_parent_simple_typedef_parent
OidIntegration.typedefs_anonymous
OidIntegration.typedefs_container
OidIntegration.unions_alignment
OidIntegration.unions_int
OidIntegration.unions_tagged_int
OidIntegration.unions_tagged_unordered_map
OidIntegration.unions_tagged_vector
OidIntegration.unions_unordered_map
OidIntegration.unions_vector
OilIntegration.folly_f14_fast_map_a
OilIntegration.folly_f14_fast_set_a
OilIntegration.folly_f14_node_map_a
OilIntegration.folly_f14_node_set_a
OilIntegration.folly_f14_value_map_a
OilIntegration.folly_f14_value_set_a
OilIntegration.folly_f14_vector_map_a
OilIntegration.folly_f14_vector_set_a

229
CMakeLists.txt
View File

@ -1,5 +1,5 @@
# object-introspection # object-introspection
cmake_minimum_required(VERSION 3.20) cmake_minimum_required(VERSION 3.24)
project(object-introspection) project(object-introspection)
# Lets find_program() locate SETUID binaries # Lets find_program() locate SETUID binaries
@ -33,75 +33,107 @@ find_package(gflags REQUIRED)
### tomlplusplus (for configuration files) ### tomlplusplus (for configuration files)
FetchContent_Declare( FetchContent_Declare(
tomlplusplus tomlplusplus
GIT_REPOSITORY https://github.com/marzer/tomlplusplus.git GIT_REPOSITORY https://github.com/marzer/tomlplusplus.git
GIT_TAG 4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0 GIT_TAG 4b166b69f28e70a416a1a04a98f365d2aeb90de8 # v3.2.0
GIT_PROGRESS TRUE GIT_PROGRESS TRUE
FIND_PACKAGE_ARGS
) )
FetchContent_MakeAvailable(tomlplusplus) FetchContent_MakeAvailable(tomlplusplus)
### glog ### glog
FetchContent_Declare( find_package(glog)
glog if (NOT glog_FOUND)
GIT_REPOSITORY https://github.com/google/glog.git FetchContent_Declare(
GIT_TAG 96a2f23dca4cc7180821ca5f32e526314395d26a glog
GIT_PROGRESS TRUE GIT_REPOSITORY https://github.com/google/glog.git
) GIT_TAG 96a2f23dca4cc7180821ca5f32e526314395d26a
FetchContent_MakeAvailable(glog) GIT_PROGRESS TRUE
)
FetchContent_MakeAvailable(glog)
# These glog executable targets still generate warnings - disable warnings for # These glog executable targets still generate warnings - disable warnings for
# them explicitly # them explicitly
target_compile_options(demangle_unittest PRIVATE "-w") target_compile_options(demangle_unittest PRIVATE "-w")
target_compile_options(logging_unittest PRIVATE "-w") target_compile_options(logging_unittest PRIVATE "-w")
target_compile_options(stl_logging_unittest PRIVATE "-w") target_compile_options(stl_logging_unittest PRIVATE "-w")
target_compile_options(symbolize_unittest PRIVATE "-w") target_compile_options(symbolize_unittest PRIVATE "-w")
target_compile_options(utilities_unittest PRIVATE "-w") target_compile_options(utilities_unittest PRIVATE "-w")
endif()
### googletest ### GTest
# Do this in the main file so it can be fetched before setting project warnings. # Do this in the main file so it can be fetched before setting project warnings.
# After this is fixed with FetchContent, move to test/CMakeLists.txt. # After this is fixed with FetchContent, move to test/CMakeLists.txt.
FetchContent_Declare( FetchContent_Declare(
googletest GTest
GIT_REPOSITORY https://github.com/google/googletest.git GIT_REPOSITORY https://github.com/google/googletest.git
GIT_TAG 1ed6a8c67a0bd675149ece27bbec0ef1759854cf GIT_TAG 1ed6a8c67a0bd675149ece27bbec0ef1759854cf
GIT_PROGRESS TRUE GIT_PROGRESS TRUE
FIND_PACKAGE_ARGS
) )
FetchContent_MakeAvailable(googletest) FetchContent_MakeAvailable(GTest)
### liburing (for RocksDB)
find_package(uring REQUIRED)
### rocksdb ### rocksdb
FetchContent_Declare( find_package(RocksDB 8.11 CONFIG)
rocksdb if (NOT RocksDB_FOUND)
GIT_REPOSITORY https://github.com/facebook/rocksdb.git FetchContent_Declare(
GIT_TAG f32521662acf3352397d438b732144c7813bbbec # v8.5.3 rocksdb
GIT_PROGRESS TRUE GIT_REPOSITORY https://github.com/facebook/rocksdb.git
) GIT_TAG f32521662acf3352397d438b732144c7813bbbec # v8.5.3
FetchContent_Populate(rocksdb) GIT_PROGRESS TRUE
add_custom_target(librocksdb ALL )
WORKING_DIRECTORY ${rocksdb_SOURCE_DIR} FetchContent_Populate(rocksdb)
COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb add_custom_target(librocksdb_build ALL
BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a WORKING_DIRECTORY ${rocksdb_SOURCE_DIR}
COMMENT "Building RocksDB" COMMAND cmake -G Ninja -B ${rocksdb_BINARY_DIR} -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=Off -DWITH_LIBURING=Off -DWITH_ZSTD=On -DFAIL_ON_WARNINGS=Off
USES_TERMINAL COMMAND cmake --build ${rocksdb_BINARY_DIR} --target rocksdb
) BYPRODUCTS ${rocksdb_BINARY_DIR}/librocksdb.a
include_directories(SYSTEM "${rocksdb_SOURCE_DIR}/include") COMMENT "Building RocksDB"
USES_TERMINAL
)
### zstd (for rocksdb)
find_package(zstd REQUIRED)
add_library(librocksdb INTERFACE)
add_dependencies(librocksdb librocksdb_build)
target_include_directories(librocksdb INTERFACE SYSTEM "${rocksdb_SOURCE_DIR}/include")
target_link_libraries(librocksdb INTERFACE ${rocksdb_BINARY_DIR}/librocksdb.a zstd::zstd)
add_library(RocksDB::rocksdb ALIAS librocksdb)
endif()
### folly ### folly
### use folly as a header only library. some features won't be supported. ### use folly as a header only library. some features won't be supported.
FetchContent_Declare( find_package(folly CONFIG)
folly if (NOT folly_FOUND)
GIT_REPOSITORY https://github.com/facebook/folly.git FetchContent_Declare(
GIT_TAG c5aa5c46291a27f69acc920894d43605ceb43eba folly
GIT_PROGRESS TRUE GIT_REPOSITORY https://github.com/facebook/folly.git
PATCH_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h GIT_TAG c5aa5c46291a27f69acc920894d43605ceb43eba
) GIT_PROGRESS TRUE
FetchContent_Populate(folly) PATCH_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/extern/shim-folly-config.h folly/folly-config.h
)
FetchContent_Populate(folly)
add_library(folly_headers INTERFACE)
target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
target_link_libraries(folly_headers INTERFACE Boost::headers)
else()
add_library(folly_headers INTERFACE)
target_include_directories(folly_headers SYSTEM INTERFACE ${FOLLY_INCLUDE_DIR})
endif()
### range-v3 ### range-v3
FetchContent_Declare( FetchContent_Declare(
range-v3 range-v3
GIT_REPOSITORY https://github.com/ericniebler/range-v3.git GIT_REPOSITORY https://github.com/ericniebler/range-v3.git
GIT_TAG a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0 GIT_TAG a81477931a8aa2ad025c6bda0609f38e09e4d7ec # 0.12.0
GIT_PROGRESS TRUE GIT_PROGRESS TRUE
FIND_PACKAGE_ARGS
) )
FetchContent_MakeAvailable(range-v3) FetchContent_MakeAvailable(range-v3)
@ -124,6 +156,18 @@ endif()
### (Re)download submodules ### (Re)download submodules
find_package(Git QUIET) find_package(Git QUIET)
if (DEFINED drgn_SOURCE_DIR)
# drgn's autotools build requires source modification. in case we have a
# readonly source (read: nix) we must copy this. do this always to avoid
# polluting the source.
file(COPY "${drgn_SOURCE_DIR}/" DESTINATION "${FETCHCONTENT_BASE_DIR}/drgn-src" NO_SOURCE_PERMISSIONS)
SET(drgn_SOURCE_DIR "${FETCHCONTENT_BASE_DIR}/drgn-src")
SET(drgn_BINARY_DIR "${CMAKE_BINARY_DIR}/_deps/drgn-build")
else()
SET(drgn_SOURCE_DIR "${PROJECT_SOURCE_DIR}/extern/drgn")
SET(drgn_BINARY_DIR "${drgn_SOURCE_DIR}/build")
endif()
# TODO: No idea if this huge block is required, just picked from an example. There may be a short-hand. # TODO: No idea if this huge block is required, just picked from an example. There may be a short-hand.
if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git") if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
# Update submodules as needed # Update submodules as needed
@ -145,20 +189,17 @@ if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
if(NOT GIT_SUBMOD_RESULT EQUAL "0") if(NOT GIT_SUBMOD_RESULT EQUAL "0")
message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules") message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules")
endif() endif()
if(NOT EXISTS "${drgn_SOURCE_DIR}")
message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
endif()
endif() endif()
endif() endif()
if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/drgn")
message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
endif()
### Select Python version ### Select Python version
find_program(PYTHON NAMES python3.9 python3) find_program(PYTHON NAMES python3.9 python3)
add_library(folly_headers INTERFACE)
target_include_directories(folly_headers SYSTEM INTERFACE ${folly_SOURCE_DIR})
target_link_libraries(folly_headers INTERFACE Boost::headers)
### bison & flex (for oid_parser) ### bison & flex (for oid_parser)
find_package(BISON 3.5 REQUIRED) find_package(BISON 3.5 REQUIRED)
find_package(FLEX) find_package(FLEX)
@ -194,8 +235,11 @@ find_package(msgpack REQUIRED CONFIG)
get_target_property(MSGPACK_INCLUDE_DIRS msgpackc INTERFACE_INCLUDE_DIRECTORIES) get_target_property(MSGPACK_INCLUDE_DIRS msgpackc INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM ${MSGPACK_INCLUDE_DIRS}) include_directories(SYSTEM ${MSGPACK_INCLUDE_DIRS})
### zstd (for rocksdb) ### drgn/elfutils dependencies
find_package(zstd REQUIRED) find_package(BZip2 REQUIRED)
find_package(OpenMP REQUIRED)
find_package(LibLZMA REQUIRED)
find_package(ZLIB REQUIRED)
### drgn ### drgn
# The setup.py script in drgn is really meant to build drgn (python # The setup.py script in drgn is really meant to build drgn (python
@ -214,46 +258,38 @@ find_package(zstd REQUIRED)
# make # make
# #
# Since setup.py has a single cmd to do this, just use it for now. # Since setup.py has a single cmd to do this, just use it for now.
#
# Another extemely annoying point. drgn pretty much has to be compiled with gcc only
# clang-12 does NOT work. clang fails with the following error :-
# configure: error: gcc with GNU99 support required
set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no") set(DRGN_CONFIGURE_FLAGS "--with-libkdumpfile=no")
if (ASAN) if (ASAN)
list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes") list(APPEND DRGN_CONFIGURE_FLAGS "--enable-asan=yes")
endif() endif()
add_custom_target(libdrgn ALL
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn add_custom_target(drgn_build ALL
COMMAND unset BISON_PKGDATADIR && CC=gcc CFLAGS="${DRGN_CFLAGS}" CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp build WORKING_DIRECTORY ${drgn_SOURCE_DIR}
BYPRODUCTS ${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/.libs/libdrgnimpl.a COMMAND unset BISON_PKGDATADIR && CFLAGS="${DRGN_CFLAGS}"
${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdw/libdw.a CONFIGURE_FLAGS="${DRGN_CONFIGURE_FLAGS}" ${PYTHON} ./setup.py build --build-temp "${drgn_BINARY_DIR}"
${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libelf/libelf.a BYPRODUCTS ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
${CMAKE_CURRENT_SOURCE_DIR}/extern/drgn/build/velfutils/libdwelf/libdwelf.a ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
COMMENT "Building drgn" COMMENT "Building drgn"
USES_TERMINAL USES_TERMINAL
) )
set(DRGN_PATH "${PROJECT_SOURCE_DIR}/extern/drgn/build")
# Ideally drgn stuff should be together at the end. But looks like rpath needs add_library(drgn INTERFACE)
# to be set before add_executable() unfortunately. Maybe split libdrgn stuff add_dependencies(drgn drgn_build)
# into a separate file later. target_link_libraries(drgn INTERFACE
set(CMAKE_SKIP_BUILD_RPATH FALSE) ${drgn_BINARY_DIR}/.libs/libdrgnimpl.a
set(CMAKE_INSTALL_RPATH ${drgn_BINARY_DIR}/velfutils/libdw/libdw.a
"${DRGN_PATH}/.libs" ${drgn_BINARY_DIR}/velfutils/libelf/libelf.a
"${DRGN_PATH}/velfutils/libdw" ${drgn_BINARY_DIR}/velfutils/libdwelf/libdwelf.a
"${DRGN_PATH}/velfutils/libelf"
"${DRGN_PATH}/velfutils/libdwelf"
)
set(CMAKE_BUILD_RPATH
"${DRGN_PATH}/.libs"
"${DRGN_PATH}/velfutils/libdw"
"${DRGN_PATH}/velfutils/libelf"
"${DRGN_PATH}/velfutils/libdwelf"
)
set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
include_directories(SYSTEM "${DRGN_PATH}") BZip2::BZip2
LibLZMA::LibLZMA
OpenMP::OpenMP_CXX
ZLIB::ZLIB
)
target_include_directories(drgn SYSTEM INTERFACE "${drgn_SOURCE_DIR}" "${drgn_BINARY_DIR}")
if (STATIC_LINK) if (STATIC_LINK)
# glog links against the `gflags` target, which is an alias for `gflags_shared` # glog links against the `gflags` target, which is an alias for `gflags_shared`
@ -289,7 +325,6 @@ add_library(oicore
oi/PaddingHunter.cpp oi/PaddingHunter.cpp
oi/Serialize.cpp oi/Serialize.cpp
) )
add_dependencies(oicore libdrgn)
target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS}) target_include_directories(oicore SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})
target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS}) target_compile_definitions(oicore PRIVATE ${LLVM_DEFINITIONS})
target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) target_include_directories(oicore PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@ -323,7 +358,6 @@ else()
endif() endif()
target_link_libraries(oicore target_link_libraries(oicore
"-L${DRGN_PATH}/.libs"
drgn drgn
dw dw
pthread pthread
@ -334,11 +368,9 @@ add_library(treebuilder
oi/TreeBuilder.cpp oi/TreeBuilder.cpp
oi/exporters/TypeCheckingWalker.cpp oi/exporters/TypeCheckingWalker.cpp
) )
add_dependencies(treebuilder librocksdb)
target_link_libraries(treebuilder target_link_libraries(treebuilder
${rocksdb_BINARY_DIR}/librocksdb.a RocksDB::rocksdb
oicore # overkill but it does need a lot of stuff oicore # overkill but it does need a lot of stuff
zstd::zstd
) )
@ -375,10 +407,8 @@ target_link_libraries(oip oicore)
### Object Introspection RocksDB Printer (OIRP) ### Object Introspection RocksDB Printer (OIRP)
add_executable(oirp tools/OIRP.cpp) add_executable(oirp tools/OIRP.cpp)
add_dependencies(oirp librocksdb)
target_link_libraries(oirp target_link_libraries(oirp
${rocksdb_BINARY_DIR}/librocksdb.a RocksDB::rocksdb
zstd::zstd
msgpackc msgpackc
) )
@ -417,3 +447,6 @@ endif()
if (DEFINED ENV{CMAKE_HOOK}) if (DEFINED ENV{CMAKE_HOOK})
include($ENV{CMAKE_HOOK}) include($ENV{CMAKE_HOOK})
endif() endif()
install(TARGETS oid DESTINATION ${CMAKE_INSTALL_BINDIR})

58
README.md
View File

@ -14,3 +14,61 @@ See the [CONTRIBUTING](CONTRIBUTING.md) file for how to help out.
## License ## License
Object Introspection is licensed under the [Apache 2.0 License](LICENSE). Object Introspection is licensed under the [Apache 2.0 License](LICENSE).
## Getting started with Nix
Nix is the easiest way to get started with `oid` as it is non-trivial to build otherwise. Explicit Nix support for Object Introspection as a Library will come down the line, but Nix can currently provide you a reproducible development environment in which to build it.
These examples expect you to have `nix` installed and available with no other dependencies required. Find the installation guide at https://nixos.org/download.html.
We also required flake support. To enable flakes globally run:
$ mkdir -p ~/.config/nix
$ echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf
Or suffix every `nix` command with `nix --extra-experimental-features 'nix-command flakes'`.
### Run upstream OID without modifying the source
$ nix run github:facebookexperimental/object-introspection -- --help
This will download the latest source into your Nix store along with all of its dependencies, running help afterwards.
### Build OID locally
$ git clone https://github.com/facebookexperimental/object-introspection
$ nix build
$ ./result/bin/oid --help
This will build OID from your local sources. Please note that this will NOT pick up changes to `extern/drgn` or `extern/drgn/libdrgn/velfutils`.
### Get a development environment
$ nix develop
$ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
$ ninja -C build
$ build/oid --help
This command provides a development shell with all the required dependencies. This is the most flexible option and will pick up source changes as CMake normally would.
Sometimes this developer environment can be polluted by things installed on your normal system. If this is an issue, use:
$ nix develop -i
This removes the environment from your host system and makes the build pure.
### Run the tests
$ nix develop
$ cmake -B build -G Ninja -DFORCE_BOOST_STATIC=Off
$ ninja -C build
$ ./tools/config_gen.py -c clang++ build/testing.oid.toml
$ ctest -j --test-dir build/test
Running tests under `nix` is new to the project and may take some time to mature. The CI is the source of truth for now.
### Format source
$ nix fmt
This formats the Nix, C++, and Python code in the repository.

27
cmake/Finduring.cmake Normal file
View File

@ -0,0 +1,27 @@
# - Find liburing
#
# uring_INCLUDE_DIR - Where to find liburing.h
# uring_LIBRARIES - List of libraries when using uring.
# uring_FOUND - True if uring found.
find_path(uring_INCLUDE_DIR
NAMES liburing.h)
find_library(uring_LIBRARIES
NAMES liburing.a liburing.so)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(uring
DEFAULT_MSG uring_LIBRARIES uring_INCLUDE_DIR)
mark_as_advanced(
uring_INCLUDE_DIR
uring_LIBRARIES)
if(uring_FOUND AND NOT TARGET uring::uring)
add_library(uring::uring UNKNOWN IMPORTED)
set_target_properties(uring::uring PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${uring_INCLUDE_DIR}"
IMPORTED_LINK_INTERFACE_LANGUAGES "C"
IMPORTED_LOCATION "${uring_LIBRARIES}")
endif()

96
flake.nix
View File

@ -1,5 +1,5 @@
{ {
description = "A flake for building Object Introspection."; description = "Object level memory profiler for C++";
inputs = { inputs = {
nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
@ -18,7 +18,99 @@
treefmt-nix, treefmt-nix,
... ...
}@inputs: }@inputs:
flake-utils.lib.eachDefaultSystem ( flake-utils.lib.eachSystem [ flake-utils.lib.system.x86_64-linux ] (
system:
let
pkgs = import nixpkgs { inherit system; };
drgnSrc = pkgs.fetchFromGitHub {
owner = "JakeHillion";
repo = "drgn";
rev = "b1f8c3e8526611b6720800250ba858a713dd9e4f";
hash = "sha256-5WhMHgx/RKtqjxGx4AyiqVKMot5xulr+6c8i2E9IxiA=";
fetchSubmodules = true;
};
mkOidPackage =
llvmPackages:
with pkgs;
pkgs.llvmPackages.stdenv.mkDerivation rec {
name = "oid";
src = self;
nativeBuildInputs = [
autoconf
automake
bison
cmake
flex
gettext
git
hexdump
libtool
ninja
pkgconf
python312
python312Packages.setuptools
python312Packages.toml
glibcLocales
];
buildInputs = [
llvmPackages.libclang
llvmPackages.llvm
boost
bzip2
curl
double-conversion
elfutils
flex
folly
folly.fmt
gflags
glog
gtest
icu
jemalloc
libarchive
libmicrohttpd
liburing
libxml2
lzma
msgpack
range-v3
rocksdb_8_11
sqlite
tomlplusplus
zstd
llvmPackages.openmp # should match the stdenv clang version, see: https://github.com/NixOS/nixpkgs/issues/79818
];
cmakeFlags = [
"-Ddrgn_SOURCE_DIR=${drgnSrc}"
"-DFORCE_BOOST_STATIC=Off"
];
outputs = [ "out" ];
};
in
{
packages = rec {
default = oid-llvm15;
oid-llvm15 = mkOidPackage pkgs.llvmPackages_15;
};
apps.default = {
type = "app";
program = "${self.packages.${system}.default}/bin/oid";
};
}
)
// flake-utils.lib.eachDefaultSystem (
system: system:
let let
pkgs = nixpkgs.legacyPackages.${system}; pkgs = nixpkgs.legacyPackages.${system};

3
oi/CMakeLists.txt
View File

@ -6,11 +6,8 @@ target_link_libraries(toml PUBLIC tomlplusplus::tomlplusplus)
add_library(drgn_utils DrgnUtils.cpp) add_library(drgn_utils DrgnUtils.cpp)
target_link_libraries(drgn_utils target_link_libraries(drgn_utils
glog::glog glog::glog
"-L${DRGN_PATH}/.libs"
drgn drgn
) )
add_dependencies(drgn_utils libdrgn)
add_library(symbol_service add_library(symbol_service
Descs.cpp Descs.cpp

3
oi/type_graph/CMakeLists.txt
View File

@ -20,12 +20,9 @@ add_library(type_graph
TypeIdentifier.cpp TypeIdentifier.cpp
Types.cpp Types.cpp
) )
add_dependencies(type_graph libdrgn)
target_link_libraries(type_graph target_link_libraries(type_graph
container_info container_info
symbol_service symbol_service
"-L${DRGN_PATH}/.libs"
drgn drgn
) )
target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS}) target_include_directories(type_graph SYSTEM PUBLIC ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS})

6
tools/config_gen.py
View File

@ -16,6 +16,7 @@
import argparse import argparse
import getpass import getpass
import os
import pathlib import pathlib
import subprocess import subprocess
import typing import typing
@ -170,7 +171,10 @@ def pull_base_toml() -> typing.Dict:
# Now, we need to replace any placeholders that might be present in the base toml file with the real verisons. # Now, we need to replace any placeholders that might be present in the base toml file with the real verisons.
user = getpass.getuser() user = getpass.getuser()
pwd = str(repo_path.resolve()) if "IN_NIX_SHELL" in os.environ and "src" in os.environ:
pwd = os.environ['src']
else:
pwd = str(repo_path.resolve())
container_list = base.get("types", {}).get("containers") container_list = base.get("types", {}).get("containers")
if container_list: if container_list: