diff --git a/CMakeLists.txt b/CMakeLists.txt index ee4848c03..6ef4cf5d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,6 +25,8 @@ include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/utils.cmake") include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/options.cmake") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules") +set(REMILL_ENABLE_TESTING OFF) + if(REMILL_ENABLE_TESTING) include(CTest) endif() @@ -40,22 +42,44 @@ find_package(Z3 CONFIG REQUIRED) # LLVM find_package(LLVM CONFIG REQUIRED) -llvm_map_components_to_libnames(llvm_libs - support core irreader - bitreader bitwriter - passes asmprinter - aarch64info aarch64desc aarch64codegen aarch64asmparser - armcodegen armasmparser - interpreter mcjit - nvptxdesc - x86info x86codegen x86asmparser - sparccodegen sparcasmparser - webassemblydesc) +# https://github.com/JonathanSalwan/Triton/issues/1082#issuecomment-1030826696 +if(LLVM_LINK_LLVM_DYLIB) + set(llvm_libs LLVM) +else() + llvm_map_components_to_libnames(llvm_libs + support + core + irreader + bitreader + bitwriter + passes + asmprinter + aarch64info + aarch64desc + aarch64codegen + aarch64asmparser + armcodegen + armasmparser + HipStdPar + CodeGenTypes + interpreter + mcjit + FrontendOffloading + nvptxdesc + x86info + x86codegen + x86asmparser + sparccodegen + sparcasmparser + webassemblydesc) +endif() message(STATUS "LLVM Libraries: ${llvm_libs}") message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") +include_directories(SYSTEM ${LLVM_INCLUDE_DIRS}) + string(REPLACE "." ";" LLVM_VERSION_LIST ${LLVM_PACKAGE_VERSION}) list(GET LLVM_VERSION_LIST 0 LLVM_MAJOR_VERSION) list(GET LLVM_VERSION_LIST 1 LLVM_MINOR_VERSION) @@ -85,6 +109,7 @@ find_package(XED CONFIG REQUIRED) find_package(glog CONFIG REQUIRED) # Google gflags +set(GFLAGS_USE_TARGET_NAMESPACE ON) find_package(gflags CONFIG REQUIRED) set(sleigh_ENABLE_TESTS OFF) @@ -94,6 +119,17 @@ file(GLOB sleigh_patches "${CMAKE_CURRENT_SOURCE_DIR}/patches/sleigh/*.patch") set(sleigh_ADDITIONAL_PATCHES "${sleigh_patches}" CACHE STRING "" FORCE) +if(WIN32) + # In one of the sleigh headers they use std::byte without the std namespace prefix. + # This then causes a windows compilation failure, because of a conflicting definition + # of 'byte' in one of the windows SDK headers. So as a fix we must append + # _HAS_STD_BYTE to the compile definitions. + add_compile_definitions(_HAS_STD_BYTE=0) + # In another sleigh source file, there is an implicit narrowing conversion. + # This flag allows implicit narrowing. + add_compile_options ( -Xclang -Wno-narrowing) +endif() + # GHIDRA SLEIGH FetchContent_Declare(sleigh GIT_REPOSITORY https://github.com/lifting-bits/sleigh.git @@ -131,7 +167,9 @@ add_library(remill_settings INTERFACE) target_include_directories(remill_settings INTERFACE $ - $) + $ + $ +) if(WIN32) # warnings and compiler settings @@ -203,7 +241,7 @@ else() endif() target_compile_definitions(remill_settings INTERFACE - "REMILL_INSTALL_SEMANTICS_DIR=\"${REMILL_INSTALL_SEMANTICS_DIR}\"" + "REMILL_INSTALL_SEMANTICS_DIR=\"semantics\"" "REMILL_BUILD_SEMANTICS_DIR_X86=\"${REMILL_BUILD_SEMANTICS_DIR_X86}\"" "REMILL_BUILD_SEMANTICS_DIR_AARCH32=\"${REMILL_BUILD_SEMANTICS_DIR_AARCH32}\"" "REMILL_BUILD_SEMANTICS_DIR_AARCH64=\"${REMILL_BUILD_SEMANTICS_DIR_AARCH64}\"" @@ -279,6 +317,10 @@ target_link_libraries(remill INTERFACE ${LINKER_END_GROUP} ) +if(WIN32) + target_link_libraries(remill INTERFACE Ws2_32.lib) +endif() + # # Also install clang, libllvm and llvm-link # @@ -325,33 +367,3 @@ if(REMILL_ENABLE_INSTALL_TARGET) install(EXPORT remillTargets DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/remill") endif() - -# tests -if(REMILL_ENABLE_TESTING) - # Tests require enabling exports on binaries - # https://cmake.org/cmake/help/latest/variable/CMAKE_ENABLE_EXPORTS.html#variable:CMAKE_ENABLE_EXPORTS - set(CMAKE_ENABLE_EXPORTS ON) - - find_package(Threads REQUIRED) - add_custom_target(test_dependencies) - - if(REMILL_ENABLE_TESTING_SLEIGH_THUMB) - message(STATUS "thumb tests enabled") - add_subdirectory(tests/Thumb) - endif() - - if(REMILL_ENABLE_TESTING_SLEIGH_PPC) - message(STATUS "ppc tests enabled") - add_subdirectory(tests/PPC) - endif() - - if(REMILL_ENABLE_TESTING_X86) - message(STATUS "X86 tests enabled") - add_subdirectory(tests/X86) - endif() - - if(REMILL_ENABLE_TESTING_AARCH64) - message(STATUS "aarch64 tests enabled") - add_subdirectory(tests/AArch64) - endif() -endif() diff --git a/CMakePresets.json b/CMakePresets.json index 1cfd763b0..df9735eb6 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -37,22 +37,24 @@ "name": "x86_64", "hidden": true, "environment": { - "VCPKG_ARCH": "x64" + "VCPKG_ARCH": "x86_64" }, "architecture": { - "value": "x64", + "value": "x86_64", "strategy": "external" } }, { "name": "vcpkg-common", "hidden": true, - "binaryDir": "$env{INSTALL_DIR}/build/remill", - "generator": "Ninja", + "binaryDir": "${sourceDir}/build", + "generator": "Visual Studio 17 2022", + "toolset":"LLVM_v143", "cacheVariables": { - "VCPKG_TARGET_TRIPLET": "$env{VCPKG_TARGET_TRIPLET}", - "CMAKE_TOOLCHAIN_FILE": "$env{CMAKE_TOOLCHAIN_FILE}", - "CMAKE_INSTALL_PREFIX": "$env{INSTALL_DIR}/install" + "CMAKE_TOOLCHAIN_FILE": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake", + "CMAKE_PREFIX_PATH": "$env{CXX_COMMON_ROOT}/Deps/installed/x64-windows-static-md-rel/share", + "VCPKG_TARGET_TRIPLET": "x64-windows-static-md-rel", + "CMAKE_INSTALL_PREFIX": "${sourceDir}/install" } }, { diff --git a/Dockerfile b/Dockerfile index 835752781..aa436e530 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Choose your LLVM version -ARG LLVM_VERSION=16 +ARG LLVM_VERSION=17 ARG ARCH=amd64 ARG UBUNTU_VERSION=22.04 ARG DISTRO_BASE=ubuntu${UBUNTU_VERSION} diff --git a/README.md b/README.md index 45c89cdb3..c3c4b86bb 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,16 @@ Remill focuses on accurately lifting instructions. It is meant to be used as a l [![Build Status](https://img.shields.io/github/workflow/status/lifting-bits/remill/CI/master)](https://github.com/lifting-bits/remill/actions?query=workflow%3ACI) -## Additional Documentation +## Documentation - - [How to contribute](docs/CONTRIBUTING.md) +To understand how Remill works you can take a look at the following resources: + + - [Step-by-step guide on how Remill lifts an instruction](docs/LIFE_OF_AN_INSTRUCTION.md) - [How to implement the semantics of an instruction](docs/ADD_AN_INSTRUCTION.md) - - [How instructions are lifted](docs/LIFE_OF_AN_INSTRUCTION.md) - [The design and architecture of Remill](docs/DESIGN.md) +If you would like to contribute you can check out: [How to contribute](docs/CONTRIBUTING.md) + ## Getting Help If you are experiencing undocumented problems with Remill then ask for help in the `#binary-lifting` channel of the [Empire Hacking Slack](https://slack.empirehacking.nyc/). diff --git a/bin/differential_tester_x86/LiftAndCompare.cpp b/bin/differential_tester_x86/LiftAndCompare.cpp index 6f6ffa707..549db64b9 100644 --- a/bin/differential_tester_x86/LiftAndCompare.cpp +++ b/bin/differential_tester_x86/LiftAndCompare.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include #include diff --git a/bin/lift/Lift.cpp b/bin/lift/Lift.cpp index 967f51a1b..3cc0afbca 100644 --- a/bin/lift/Lift.cpp +++ b/bin/lift/Lift.cpp @@ -352,7 +352,7 @@ int main(int argc, char *argv[]) { arg_types.push_back(llvm::PointerType::get(context, 0)); } - const auto state_type = llvm::PointerType::get(context, 0); + const auto state_type = arch->StateStructType(); const auto func_type = llvm::FunctionType::get(mem_ptr_type, arg_types, false); const auto func = llvm::Function::Create( diff --git a/cmake/BCCompiler.cmake b/cmake/BCCompiler.cmake index 2efb669c9..6106555ee 100644 --- a/cmake/BCCompiler.cmake +++ b/cmake/BCCompiler.cmake @@ -17,7 +17,7 @@ set(DEFAULT_BC_COMPILER_FLAGS -ffreestanding -fno-common -fno-builtin -fno-exceptions -fno-rtti -fno-asynchronous-unwind-tables -Wno-unneeded-internal-declaration -Wno-unused-function -Wgnu-inline-cpp-without-extern - -Wno-pass-failed=transform-warning + -Wno-pass-failed=transform-warning -fshort-wchar -Xclang -mlong-double-80 ${EXTRA_BC_SYSROOT} ) diff --git a/cmake/git_watcher.cmake b/cmake/git_watcher.cmake index f121d9bf5..7cc27f163 100644 --- a/cmake/git_watcher.cmake +++ b/cmake/git_watcher.cmake @@ -134,10 +134,10 @@ macro(RunGitCommand) # Most methods have a fall-back default value that's used in case of non-zero # exit codes. If you're feeling risky, disable this safety check and use # those default values. - if(GIT_FAIL_IF_NONZERO_EXIT ) - string(REPLACE ";" " " args_with_spaces "${ARGV}") - message(FATAL_ERROR "${stderr} (${GIT_EXECUTABLE} ${args_with_spaces})") - endif() + #if(GIT_FAIL_IF_NONZERO_EXIT ) + # string(REPLACE ";" " " args_with_spaces "${ARGV}") + # message(FATAL_ERROR "${stderr} (${GIT_EXECUTABLE} ${args_with_spaces})") + #endif() endif() endmacro() diff --git a/cmake/options.cmake b/cmake/options.cmake index 775286255..741dfe379 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -1,7 +1,7 @@ include(CMakeDependentOption) -set(can_enable_testing TRUE) +set(can_enable_testing FALSE) set(can_enable_testing_x86 FALSE) set(can_enable_testing_aarch64 FALSE) diff --git a/cmake/utils.cmake b/cmake/utils.cmake index 44f911914..588766c90 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -161,7 +161,7 @@ function(InstallExternalTarget target_name target_path install_type installed_fi install(FILES "${output_file_path}" TYPE ${install_type} - PERMISSIONS OWNER_READ OWNER_EXECUTE + PERMISSIONS OWNER_READ OWNER_EXECUTE OWNER_WRITE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE ) diff --git a/docs/ADD_AN_INSTRUCTION.md b/docs/ADD_AN_INSTRUCTION.md index caccecfe8..bdf5128b6 100644 --- a/docs/ADD_AN_INSTRUCTION.md +++ b/docs/ADD_AN_INSTRUCTION.md @@ -32,14 +32,14 @@ To start off, here are two entries from the tables document: These entries contain a lot of information and are quite dense. Below are descriptions of the salient parts. - `AND`: The name of the instruction. This is typically the opcode, though sometimes it will be more specific. In general, there is a one-to-one correspondence between an instruction name and its *SEM*: a generic function that you will define to implement the multiple variants of this instruction. -- `LOGICAL`: This is the category of the instruction. This will generally tell you where to put your instruction code. In this case, we would implement the instruction in the [LOGICAL.cpp](/remill/Arch/X86/Semantics/LOGICAL.cpp) file. +- `LOGICAL`: This is the category of the instruction. This will generally tell you where to put your instruction code. In this case, we would implement the instruction in the [LOGICAL.cpp](/lib/Arch/X86/Semantics/LOGICAL.cpp) file. - `AND_GPRv_MEMv`: This is the *ISEL*: an instantiation of your instruction's semantic function. - `SCALABLE`: This tells you that a particular *ISEL* can actually relate to a number of different operand sizes. We have short forms and naming conventions for writing one *ISEL* for all the operand sizes; however, this can be done manually as well. One XED convention is that if you see a `z` or `v` within the *ISEL*, then the instruction is probably scalable. - `EXPLICIT`: This is an explicit operand. If you were to try to type out this instruction as assembly code and assemble it, then an explicit operand is one that you must specify after the opcode mnemonic. In Remill, your semantic functions will have _at least one argument for each explicit operand_. - `IMPLICIT`: This is an implicit operand. You can think of this as being an operand that you _might_ write out in assembly. Alternatively, you can see it as an operand that is explicit in at least one *ISEL*. Not to be confused with SUPPRESSED. - `SUPPRESSED`: This is an operand that is never written out in assembly, but is operated on internally by the semantics of an instruction. In Remill, you *do not* associate any arguments in your semantic functions with suppressed operands. - `R`, `RW`, `W`, `CR`, `CW`, `RCW`: These per-operand markers indicate how the semantics of the instruction operate on a particular operand, or in other words, they describe the mutability of the operand. `R` stands for read, `W` stands for write, and `C` stands for condition. Therefore, `RCW` states that the semantics will read and conditionally write to the associated operand. -- `REG`, `MEM`, `IMM`: These identify the type of the operand within a particular instruction selection (ISEL). Remill has C++ type names associated with each operand type and size, defined in [Types.h](https://github.com/lifting-bits/remill/blob/master/remill/Arch/Runtime/Types.h) +- `REG`, `MEM`, `IMM`: These identify the type of the operand within a particular instruction selection (ISEL). Remill has C++ type names associated with each operand type and size, defined in [Types.h](/include/remill/Arch/Runtime/Types.h) In the following code examples we will ignore condition code computation. Most instructions that manipulate condition codes have already been implemented. @@ -123,7 +123,7 @@ We see the use of the following Remill operators: - `UAndV32`: Performs a logical AND operation on a vector if unsigned, 32-bit integers. - `UWriteV32` writes to `dst` a vector of unsigned, 32-bit integers. -The "types" of the operators must always match. If you intend to implement support for new vector instructions, then start by looking at some existing, more complicated [examples](/remill/Arch/X86/Semantics/CONVERT.cpp). +The "types" of the operators must always match. If you intend to implement support for new vector instructions, then start by looking at some existing, more complicated [examples](/lib/Arch/X86/Semantics/CONVERT.cpp). ## Testing instructions @@ -131,7 +131,7 @@ After implementing an instruction semantic, you must implement the associated in ### Writing your first tests -We will revisit the above example of the `AND` instruction. Start by creating [`AND.S`](/tests/X86/LOGICAL/AND.S) within the category- and architecture-specific sub-directory of the [tests](/tests) directory. Then, add that file as an `#include` statement to the appropriate place in `remill/tests/X86/Tests.S`. +We will revisit the above example of the `AND` instruction. Start by creating [`AND.S`](/tests/X86/LOGICAL/AND.S) within the category- and architecture-specific sub-directory of the [`tests/X86`](/tests/X86/) directory. Then, add that file as an `#include` statement to the appropriate place in [`tests/X86/Tests.S`](/tests/X86/Tests.S). Each *ISEL* should be tested separately. For example, a test for the `AND_GPRv_GPRv_32` *ISEL* would be: @@ -181,18 +181,18 @@ Much of the process is as described for x86 above, but here we will document wha Currently, instructions that alias to another instruction (`MOV`, `CMP`, etc) are extracted as the instruction that they alias, and are not their own separate class. -All of the `ISEL` function-extracting-and-decoding shells are auto-generated by the tool at `remill/Arch/AArch64/Etc/GenOpMap.py`, which literally parses the [architecture documentation reference](https://developer.arm.com/-/media/developer/products/architecture/armv8-a-architecture/A64_v82A_ISA_xml_00bet3.2.tar.gz), to produce the following files: +All of the `ISEL` function-extracting-and-decoding shells are auto-generated by [`GenOpMap.py`](/lib/Arch/AArch64/Etc/GenOpMap.py), which literally parses the [architecture documentation reference](https://developer.arm.com/-/media/developer/products/architecture/armv8-a-architecture/A64_v82A_ISA_xml_00bet3.2.tar.gz), to produce the following files: -- `remill/Arch/AArch64/Extract.cpp` is responsible for parsing the bit sequence of an instruction and assigning it to a corresponding selector. It also populates the `InstData` structure with the correct fields, which can be located at `remill/Arch/AArch64/Decode.h`. +- `lib/Arch/AArch64/Extract.cpp` is responsible for parsing the bit sequence of an instruction and assigning it to a corresponding selector. It also populates the `InstData` structure with the correct fields, which can be located at `lib/Arch/AArch64/Decode.h`. - Any logical processing other than passing down the raw bit values should be done in the semantic definition, not here or in the decoding pipeline. -- `remill/Arch/AArch64/Decode.cpp` is filled with function skeletons that will receive the populated `InstData` struct and use it to push `Operand` objects into our `Instruction` class that will be later used in the semantic definition. Cut and paste the skeleton into `./Arch.cpp` and fill it out accordingly, using other functions in the file as a reference. +- `lib/Arch/AArch64/Decode.cpp` is filled with function skeletons that will receive the populated `InstData` struct and use it to push `Operand` objects into our `Instruction` class that will be later used in the semantic definition. Cut and paste the skeleton into `./Arch.cpp` and fill it out accordingly, using other functions in the file as a reference. - The order in which operands (immediates, registers, memory displacements, etc.) is important and will be reflected in the semantic definition parameters. - `inst.function` of type `Instruction` contains the semantic name of the target. You can manipulate this to give you a greater granularity at the semantic level later on by, for example, appending a conditional qualifier like `_EQ` or `_GE` to the semantic. ### Defining the semantic -Under `remill/Arch/AArch64/Semantics/`, figure out which class of functions your instruction fits under. If creating a new semantic class file, make sure it is included in `../Instructions.cpp`. +Under [`lib/Arch/AArch64/Semantics/`](/lib/Arch/AArch64/Semantics/), figure out which class of functions your instruction fits under. If creating a new semantic class file, make sure it is included in `../Instructions.cpp`. Similar to `X86` semantics, you will use the `DEF_ISEL` and `DEF_COND_ISEL` to implement the semantic. Below are some interesting gotchas: @@ -203,14 +203,14 @@ Similar to `X86` semantics, you will use the `DEF_ISEL` and `DEF_COND_ISEL` to i ### Writing and running the test -Mirroring `X86`, the tests can be found under `remill/tests/AArch64/*`. When adding a test, make sure it is included in the `./Tests.S` file. Below is a useful set of steps to partially recompile/build for quickly fixing tests (from the `remill-build` directory): +Mirroring `X86`, the tests can be found under [`tests/AArch64/*`](/tests/AArch64/). When adding a test, make sure it is included in the `./Tests.S` file. Below is a useful set of steps to partially recompile/build for quickly fixing tests (from the `remill-build` directory): ```bash #!/bin/sh set -e -touch ../remill/Arch/AArch64/Runtime/BasicBlock.cpp -touch ../remill/Arch/AArch64/Runtime/Instructions.cpp +touch ../lib/Arch/AArch64/Runtime/BasicBlock.cpp +touch ../lib/Arch/AArch64/Runtime/Instructions.cpp rm -f tests/AArch64/lift-* rm -f tests/AArch64/run-* make @@ -221,7 +221,7 @@ If you come across errors in the build process that "an instruction matching the ## Tips for Debugging a Failing Test Case -Debugging with `gdb` can be helpful in narrowing down where exactly a mismatch between "native" and "lifted" code occurs. For the structs we will discuss here, refer to the `State` struct located at `remill/Arch/AArch64/Runtime/State.h` (for AArch64) or `remill/Arch/X86/Runtime/State.h` (for X86). +Debugging with `gdb` can be helpful in narrowing down where exactly a mismatch between "native" and "lifted" code occurs. For the structs we will discuss here, refer to the `State` struct located at `lib/Arch/AArch64/Runtime/State.h` (for AArch64) or `lib/Arch/X86/Runtime/State.h` (for X86). In the following example, the AArch64 test case we are debugging is: @@ -260,8 +260,8 @@ Afterwards, we can set breakpoints at `udiv_w3_w0_w1_2` for the native and (nami *Pro-tip*: eliminate the tedium of typing and re-typing `gdb` commands by passing them all in from the shell. For example on X86, to break at the point where you can examine the differences between the resulting lifted and native states in a failing test case: ```shell -set $bp_line = `grep -n "Lifted and native states did not match." /remill/tests/X86/Run.cpp | cut -f1 -d:` -gdb ./tests/X86/run-x86-tests -ex "b /remill/tests/X86/Run.cpp:$bp_line" -ex "set \$native = (State *)&gNativeState" -ex "set \$lifted = (State *)&gLiftedState" -ex run +set $bp_line = `grep -n "Lifted and native states did not match." tests/X86/Run.cpp | cut -f1 -d:` +gdb ./tests/X86/run-x86-tests -ex "b tests/X86/Run.cpp:$bp_line" -ex "set \$native = (State *)&gNativeState" -ex "set \$lifted = (State *)&gLiftedState" -ex run ``` Usually it is sufficient to narrow down the differences between the particular registers/status fields involved, but a complete diff of the entire `State` struct is also helpful. In the lifter, some state fields are intentionally zeroed out to avoid being compared, so make sure to account for those). A contrived debugging example: diff --git a/docs/DESIGN.md b/docs/DESIGN.md index d261ca169..2c648c5a4 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -4,7 +4,7 @@ Remill translates machine code, and *only* machine code, into LLVM bitcode. The ## Intrinsics -Remill defers the "implementation" of memory accesses and certain types of control flows to the consumers of the produced bitcode. Deferral in this takes the form of Remill [intrinsics](https://github.com/lifting-bits/remill/blob/master/remill/Arch/Runtime/Intrinsics.h). +Remill defers the "implementation" of memory accesses and certain types of control flows to the consumers of the produced bitcode. Deferral in this takes the form of Remill [intrinsics](/include/remill/Arch/Runtime/Intrinsics.h). For example, the `__remill_read_memory_8` intrinsic function represents the action of reading 8 bits of memory. Via this and similar intrinsics, downstream tools can distinguish LLVM `load` and `store` instructions from accesses to the modeled program's memory. Downstream tools can, of course, implement memory intrinsics using LLVM's own memory access instructions. @@ -14,9 +14,9 @@ Instruction semantics are implemented using C++, and tested against their native ## Machine state -The register state of a machine is represented by a single `State` structure. For example, the x86/amd64 state structure is defined in [State.h](https://github.com/lifting-bits/remill/blob/master/remill/Arch/X86/Runtime/State.h). State structures are carefully designed to maintain the following properties. +The register state of a machine is represented by a single `State` structure. For example, the x86/amd64 state structure is defined in [State.h](/include/remill/Arch/X86/Runtime/State.h). State structures are carefully designed to maintain the following properties: -- They should actively prevent certain compiler optimizations that obscure the semantics of the translated machine code. For example, special [tear fields](https://github.com/lifting-bits/remill/blob/master/remill/Arch/X86/Runtime/State.h#L327) are introduced so as to prevent load and store coalescing, and to preserve the semantics that write to logical units of data to remain as such. +- They should actively prevent certain compiler optimizations that obscure the semantics of the translated machine code. For example, special [tear fields](https://github.com/lifting-bits/remill/blob/a6abbb818c3c523dfb806cf4e8a0211f3a8d56e4/include/remill/Arch/X86/Runtime/State.h#L698) are introduced so as to prevent load and store coalescing, and to preserve the semantics that write to logical units of data to remain as such. - They should have a uniform size across all architecture revisions and generations. This permits things such as: - Mixing separately translated bitcode from two x86 binaries, one with and one without AVX support. @@ -32,4 +32,3 @@ Remill-produced bitcode has a memory model that includes memory barriers and ato ## Runtime Remill-produced bitcode can be thought of as an emulator for a program. Through this lens, the memory used to store a `State` structure or any local variables (`alloca`s in LLVM) needed to support the emulation must be treated as distinct from the modeled program's memory itself. This separation enables Remill to maintain [transparency](http://www.burningcutlery.com/derek/docs/transparency-VEE12.pdf) with respect to memory accesses. - diff --git a/docs/INTRINSICS.md b/docs/INTRINSICS.md index 750ed85c0..7a9ef9b86 100644 --- a/docs/INTRINSICS.md +++ b/docs/INTRINSICS.md @@ -1,9 +1,9 @@ # Remill Intrinsics -Remill models the semantics of instruction logic and its effects on processor and memory state, but it does not model memory _access_ behaviors or certain types of control flow. Remill defers the "implementation" of those to the consumers of the produced bitcode. Deferral is performed using Remill _intrinsics_, [declared in Intrinsics.h](https://github.com/lifting-bits/remill/blob/master/remill/Arch/Runtime/Intrinsics.h) and [defined in Intrinsics.cpp](https://github.com/lifting-bits/remill/blob/master/remill/Arch/Runtime/Intrinsics.cpp). +Remill models the semantics of instruction logic and its effects on processor and memory state, but it does not model memory _access_ behaviors or certain types of control flow. Remill defers the "implementation" of those to the consumers of the produced bitcode. Deferral is performed using Remill _intrinsics_, declared in [`Intrinsics.h`](/include/remill/Arch/Runtime/Intrinsics.h) and defined in [`Intrinsics.cpp`](/lib/Arch/Runtime/Intrinsics.cpp). In Remill's implementation of an instruction, memory operands are represented by their addresses, but accessed only via intrinsics. For example, the `__remill_read_memory_8` intrinsic function represents the action of reading 8 bits of memory. Via this and similar intrinsics, downstream tools can distinguish LLVM `load` and `store` instructions from accesses to the modeled program's memory. Downstream tools can, of course, implement memory intrinsics using LLVM's own memory access instructions. The typical developer working on extending Remill does not need to work with Remill's memory access intrinsics directly, because they are actually wrapped by Remill's _operators_. Refer to the [Operators documentation](OPERATORS.md) for more information on those. -For an example of how Remill's control flow intrinsics are used, see how the [Remill instruction test-runner](https://github.com/lifting-bits/remill/blob/master/tests/X86/Run.cpp) uses `__remill_sync_hyper_call` to virtualize the behavior of instructions like `cpuid` (get CPU capabilities) or `readtsc` (read time stamp counter). +For an example of how Remill's control flow intrinsics are used, see how the [Remill instruction test-runner](/tests/X86/Run.cpp) uses `__remill_sync_hyper_call` to virtualize the behavior of instructions like `cpuid` (get CPU capabilities) or `readtsc` (read time stamp counter). diff --git a/docs/LIFE_OF_AN_INSTRUCTION.md b/docs/LIFE_OF_AN_INSTRUCTION.md index f4d3b3fe6..3b623db37 100644 --- a/docs/LIFE_OF_AN_INSTRUCTION.md +++ b/docs/LIFE_OF_AN_INSTRUCTION.md @@ -12,7 +12,7 @@ This document will use the instructions in the following basic block as a runnin The first step to lifting is to decode the bytes of an instruction. This decoding step takes raw instruction bytes, and turns them into a higher-level [`Instruction`](/include/remill/Arch/Instruction.h) data structure. This data structure represents the logical operands to the machine code instructions. These operands have a one-to-one correspondence with arguments that will be passed to semantics functions. -Below is a string representation of the data structures representing our example assembly. +For illustration purposes, here is a string representation of the data structures representing the running example: ```lisp ;; mov eax, 1 @@ -42,10 +42,10 @@ Below is a string representation of the data structures representing our example ## From architecture-specific to architecture-neutral -Decoded instructions must be lifted into a compatible function. Compatible functions are clones of the [`__remill_basic_block`](/lib/Arch/X86/Runtime/BasicBlock.cpp) function. The `__remill_basic_block` function is special because it defines local -variables that "point into" the [`State`](/include/remill/Arch/X86/Runtime/State.h)) structure, which represents the machine's register state. +Decoded instructions must be lifted into a compatible function. This function is generated by [`Arch::DefineLiftedFunction`](https://github.com/lifting-bits/remill/blob/a6abbb818c3c523dfb806cf4e8a0211f3a8d56e4/lib/Arch/Arch.cpp#L731-L741) (based on `__remill_jump` in [`Intrinsics.h`](/include/remill/Arch/Runtime/Intrinsics.h)) and from now on we will refer to this function as `__remill_basic_block`. -The following is an example of the `__remill_basic_block` function for X86. +The `__remill_basic_block` function is special because it defines local +variables that "point into" the [`State`](/include/remill/Arch/X86/Runtime/State.h) structure, which represents the machine's register state. Below is a simplified version of the generated (empty) `__remill_basic_block` function for X86: ```C++ // Instructions will be lifted into clones of this function. @@ -63,14 +63,15 @@ Memory *__remill_basic_block(State &state, addr_t curr_pc, Memory *memory) { ... - // Lifted code will be placed here in clones versions of this function. + // Lifted code will be placed here in cloned versions of this function. + return memory; } ``` In the case of the `push ebx` instruction from our example block, our decoder understands that `ebx` is a register. Surprisingly, the lifting side of Remill has no concept of what `ebx` is! Remill is designed to be able to translate arbitrary machine code to LLVM bitcode. To that end, there needs to be a kind of "common language" that the architecture-neutral LLVM side of things and the architecture-specific semantics functions and machine instruction decoders can use to negotiate the translation process. This common language is variable names within the `__remill_basic_block` function. The instruction decoder ensures that decoded register names correspond to variables defined in `__remill_basic_block`. The programmer implementing `__remill_basic_block` ensures the same things. The conversion from `Instruction` data structures to LLVM bitcode _assumes_ that this correspondence exists. -Let's hammer this home. If we scroll up, we see that the `Instruction` data structure corresponding to `push ebx` has `EBX` in one of its `(READ_OP (REG_32 EBX))` register operands. This operand corresponds to the following `Register` data structure. +Let's hammer this home. If we scroll up, we see that the `Instruction` data structure corresponding to `push ebx` has `EBX` in one of its `(READ_OP (REG_32 EBX))` register operands. This operand corresponds to the following `Register` data structure: ```C++ class Register { @@ -88,7 +89,7 @@ The decoder initialized the `name` field with `"EBX"`, and the lifter can look u In spirit, the lifted code for the instructions in our running example looks like the following C++ code: ```C++ -void __remill_sub_804b7a3(State *state, addr_t pc, Memory *memory) { +Memory *__remill_basic_block_804b7a3(State *state, addr_t pc, Memory *memory) { auto &EIP = state.gpr.rip.dword; auto &EAX = state.gpr.rax.dword; auto &EBX = state.gpr.rbx.dword; @@ -150,10 +151,10 @@ You can head on over to the [how to add an instruction](ADD_AN_INSTRUCTION.md) d ## We must go deeper -The spiritual lifted code makes one function call per lifted instruction, where the actual implementation of each function can be arbitrarily complex. If we optimize the bitcode that Remill produces for our few example instructions, then what we get, if translated back to C++, looks like the following: +The spiritual lifted code makes one function call per lifted instruction, where the actual implementation of each function can be arbitrarily complex. If we optimize the bitcode that Remill produces for our running example and translate it back to C++, it looks something like this: ```C++ -void __remill_sub_804b7a3(State &state, addr_t pc, Memory *memory) { +Memory *__remill_basic_block_804b7a3(State *state, addr_t pc, Memory *memory) { auto &EIP = state.gpr.rip.dword; auto &EAX = state.gpr.rax.dword; auto &EBX = state.gpr.rbx.dword; @@ -183,7 +184,7 @@ As LLVM bitcode, this looks like: ```llvm ; Function Attrs: noinline nounwind -define %struct.Memory* @__remill_sub_804b7a3(%struct.State* dereferenceable(2688) %state2, i32 %pc, %struct.Memory* %memory1) #0 { +define %struct.Memory* @__remill_basic_block_804b7a3(%struct.State* dereferenceable(2688) %state2, i32 %pc, %struct.Memory* %memory1) #0 { %1 = getelementptr inbounds %struct.State, %struct.State* %state2, i64 0, i32 6, i32 33, i32 0, i32 0 %2 = getelementptr inbounds %struct.State, %struct.State* %state2, i64 0, i32 6, i32 1, i32 0, i32 0 %3 = getelementptr inbounds %struct.State, %struct.State* %state2, i64 0, i32 6, i32 3, i32 0, i32 0 @@ -218,7 +219,7 @@ In the case of Remill, we want to be explicit about accesses to the "modeled pro The naming of "runtime" does not mean that the bitcode itself needs to be executed, nor does it mean that the intrinsics must be implemented in any specific way. Remill's intrinsics provide semantic value (in terms of their naming). A user of Remill bitcode can do anything they want with these intrinsics. For example, they could convert all of the memory intrinsics into LLVM `load` and `store` instructions. -Anyway, back to the memory model and memory ordering. Notice that the `memory` pointer is passed into every memory access intrinsic. The `memory` pointer is also replaced in the case of memory write intrinsics. This is to enforce a total order across all memory writes, and a partial order between memory reads and writes. +Anyway, back to the memory model and memory ordering. Notice that the `memory` pointer is passed into every memory access intrinsic. The `memory` pointer is also reassigned in the case of memory write intrinsics. This is to enforce a total order across all memory writes, and a partial order between memory reads and writes. The `__remill_async_hyper_call` instruction instructs the "runtime" that an explicit [interrupt](https://en.wikipedia.org/wiki/Interrupt) (`int 0x80`) happens. Again, Remill has no way of knowing what this actually means or how it works – that falls under the purview of the operating system kernel. What Remill *does* know is that an interrupt is a kind of ["indirect" control flow](https://en.wikipedia.org/wiki/Indirect_branch), and so it models it like all other indirect control flows. diff --git a/docs/OPERATORS.md b/docs/OPERATORS.md index 40dbebb01..2828eb3b6 100644 --- a/docs/OPERATORS.md +++ b/docs/OPERATORS.md @@ -1,8 +1,8 @@ # Remill Operators -In implementing a semantic function for a particular instruction in Remill, one must use Remill `Read` and `Write` _operators_ to access memory. Operators are convenience functions that wrap Remill's abstracted memory types (as defined in [Types.h](https://github.com/lifting-bits/remill/blob/master/remill/Arch/X86/Runtime/Types.h)) and memory-access types _intrinsics_, which are integral to how Remill models memory state without defining implementations of memory access behaviors (those are deferred to the consumer of Remill to implement). +In implementing a semantic function for a particular instruction in Remill, one must use Remill `Read` and `Write` _operators_ to access memory. Operators are convenience functions that wrap Remill's abstracted memory types (as defined in [Types.h](/include/remill/Arch/X86/Runtime/Types.h)) and memory-access types _intrinsics_, which are integral to how Remill models memory state without defining implementations of memory access behaviors (those are deferred to the consumer of Remill to implement). -Operators, defined in [Operators.h](https://github.com/lifting-bits/remill/blob/master/remill/Arch/Runtime/Operators.h), are architecture-independent methods for accessing and manipulating specific data-widths of memory (the Remill memory types in `Types.h`). Operators are defined as preprocessor macros and generically using C++ templates, which is an efficient way to create signed, unsigned, and floating point variants of each operator. The naming of these operators generally follows the naming of [LLVM instructions](http://llvm.org/docs/LangRef.html#binary-operations). For example, there are `UAdd`, `SAdd`, and `FAdd` for implementing unsigned, signed, and floating point addition, respectively. +Operators, defined in [Operators.h](/include/remill/Arch/Runtime/Operators.h), are architecture-independent methods for accessing and manipulating specific data-widths of memory (the Remill memory types in `Types.h`). Operators are defined as preprocessor macros and generically using C++ templates, which is an efficient way to create signed, unsigned, and floating point variants of each operator. The naming of these operators generally follows the naming of [LLVM instructions](http://llvm.org/docs/LangRef.html#binary-operations). For example, there are `UAdd`, `SAdd`, and `FAdd` for implementing unsigned, signed, and floating point addition, respectively. A rundown of commonly used Remill operators and their usages follows. This list and the examples are not exhaustive, and will not be as up-to-date as the source itself. But this document ought to serve as a guide to what kind of operators are available and which ones you should use in implementing your instruction semantic functions. diff --git a/include/remill/Arch/Arch.h b/include/remill/Arch/Arch.h index 3853d3b19..90f5ce95f 100644 --- a/include/remill/Arch/Arch.h +++ b/include/remill/Arch/Arch.h @@ -25,7 +25,7 @@ #pragma clang diagnostic ignored "-Wswitch-enum" #include -#include +#include #include #include #include diff --git a/include/remill/Arch/Runtime/Math.h b/include/remill/Arch/Runtime/Math.h index 4fa83aa9e..3d728f8e6 100644 --- a/include/remill/Arch/Runtime/Math.h +++ b/include/remill/Arch/Runtime/Math.h @@ -36,7 +36,7 @@ static_assert(8 == sizeof(float128_t), "Invalid `float128_t` size."); // A "native_float80_t" is a native type that is closes to approximating // an x86 80-bit float. // when building against CUDA, default to 64-bit float80s -#if !defined(__CUDACC__) && (defined(__x86_64__) || defined(__i386__) || defined(_M_X86)) +#if !defined(__CUDACC__) && !defined(WIN32) && (defined(__x86_64__) || defined(__i386__) || defined(_M_X86)) #if defined(__float80) typedef __float80 native_float80_t; #else @@ -53,7 +53,7 @@ union union_ld { struct { uint8_t data[kEightyBitsInBytes]; // when building against CUDA, default to 64-bit float80s -#if !defined(__CUDACC__) && (defined(__x86_64__) || defined(__i386__) || defined(_M_X86)) +#if !defined(__CUDACC__) && !defined(WIN32) && (defined(__x86_64__) || defined(__i386__) || defined(_M_X86)) // We are doing x86 on x86, so we have native x86 FP80s, but they // are not available in raw 80-bit native form. // @@ -154,8 +154,9 @@ union nan80_t { } __attribute__((packed)); } __attribute__((packed)); -static_assert(sizeof(float80_t) == sizeof(nan80_t), - "Invalid packing of `nan80_t`."); +#if !defined(_WIN32) && !defined(_WIN64) +static_assert(sizeof(float80_t) == sizeof(nan80_t), "Invalid packing"); +#endif #if __has_include() # include diff --git a/lib/Arch/AArch32/Runtime/CMakeLists.txt b/lib/Arch/AArch32/Runtime/CMakeLists.txt index 884cf2f06..0cd90d67a 100644 --- a/lib/Arch/AArch32/Runtime/CMakeLists.txt +++ b/lib/Arch/AArch32/Runtime/CMakeLists.txt @@ -30,7 +30,7 @@ function(add_runtime_helper target_name little_endian) # Visual C++ requires C++14 if(WIN32) - set(required_cpp_standard "c++14") + set(required_cpp_standard "c++17") else() set(required_cpp_standard "c++17") endif() diff --git a/lib/Arch/AArch32/Runtime/Instructions.cpp b/lib/Arch/AArch32/Runtime/Instructions.cpp index 17d181df5..22a341041 100644 --- a/lib/Arch/AArch32/Runtime/Instructions.cpp +++ b/lib/Arch/AArch32/Runtime/Instructions.cpp @@ -31,7 +31,9 @@ // A definition is required to ensure that LLVM doesn't optimize the `State` type out of the bytecode // See https://github.com/lifting-bits/remill/pull/631#issuecomment-1279989004 -State __remill_state; +extern "C" { +extern State __remill_state = {}; +} // extern C #define REG_PC state.gpr.r15.dword #define REG_LR state.gpr.r14.dword diff --git a/lib/Arch/AArch64/Arch.cpp b/lib/Arch/AArch64/Arch.cpp index 94ebb30eb..6b3b4864d 100644 --- a/lib/Arch/AArch64/Arch.cpp +++ b/lib/Arch/AArch64/Arch.cpp @@ -16,12 +16,12 @@ #include #include -#include #include #include #include #include #include +#include #include #include diff --git a/lib/Arch/AArch64/Runtime/CMakeLists.txt b/lib/Arch/AArch64/Runtime/CMakeLists.txt index 75460716b..0bd8f6b36 100644 --- a/lib/Arch/AArch64/Runtime/CMakeLists.txt +++ b/lib/Arch/AArch64/Runtime/CMakeLists.txt @@ -30,7 +30,7 @@ function(add_runtime_helper target_name address_bit_size little_endian) # Visual C++ requires C++14 if(WIN32) - set(required_cpp_standard "c++14") + set(required_cpp_standard "c++17") else() set(required_cpp_standard "c++17") endif() diff --git a/lib/Arch/AArch64/Runtime/Instructions.cpp b/lib/Arch/AArch64/Runtime/Instructions.cpp index 82db2118e..fe7204303 100644 --- a/lib/Arch/AArch64/Runtime/Instructions.cpp +++ b/lib/Arch/AArch64/Runtime/Instructions.cpp @@ -31,7 +31,9 @@ // A definition is required to ensure that LLVM doesn't optimize the `State` type out of the bytecode // See https://github.com/lifting-bits/remill/pull/631#issuecomment-1279989004 -State __remill_state; +extern "C" { +extern State __remill_state = {}; +} // extern C #define REG_PC state.gpr.pc.qword #define REG_SP state.gpr.sp.qword diff --git a/lib/Arch/Arch.cpp b/lib/Arch/Arch.cpp index fde5377e2..91c0f3e8a 100644 --- a/lib/Arch/Arch.cpp +++ b/lib/Arch/Arch.cpp @@ -16,6 +16,7 @@ #include "remill/Arch/Arch.h" +#include "llvm/IR/AttributeMask.h" #include #include #include @@ -696,8 +697,7 @@ void Arch::PrepareModuleDataLayout(llvm::Module *mod) const { for (llvm::Function &func : *mod) { auto attribs = func.getAttributes(); - attribs = attribs.removeFnAttributes(context, - llvm::AttributeMask(target_attribs)); + attribs = attribs.removeFnAttributes(context, llvm::AttributeMask(target_attribs)); func.setAttributes(attribs); } } diff --git a/lib/Arch/Name.cpp b/lib/Arch/Name.cpp index b357f3d27..59892fc83 100644 --- a/lib/Arch/Name.cpp +++ b/lib/Arch/Name.cpp @@ -16,7 +16,7 @@ #include "remill/Arch/Name.h" -#include +#include namespace remill { diff --git a/lib/Arch/PPC/Runtime/CMakeLists.txt b/lib/Arch/PPC/Runtime/CMakeLists.txt index b9bb18424..cfff5fd11 100644 --- a/lib/Arch/PPC/Runtime/CMakeLists.txt +++ b/lib/Arch/PPC/Runtime/CMakeLists.txt @@ -25,7 +25,7 @@ set_source_files_properties(BasicBlock.cpp PROPERTIES COMPILE_FLAGS "-O3 -g0") # Visual C++ requires C++14 if(WIN32) - set(required_cpp_standard "c++14") + set(required_cpp_standard "c++17") else() set(required_cpp_standard "c++17") endif() diff --git a/lib/Arch/PPC/Runtime/Instructions.cpp b/lib/Arch/PPC/Runtime/Instructions.cpp index 9495fc651..8abbacef7 100644 --- a/lib/Arch/PPC/Runtime/Instructions.cpp +++ b/lib/Arch/PPC/Runtime/Instructions.cpp @@ -21,7 +21,9 @@ // A definition is required to ensure that LLVM doesn't optimize the `State` type out of the bytecode // See https://github.com/lifting-bits/remill/pull/631#issuecomment-1279989004f -State __remill_state; +extern "C" { +extern State __remill_state = {}; +} // extern C #define HYPER_CALL state.hyper_call diff --git a/lib/Arch/Runtime/Intrinsics.cpp b/lib/Arch/Runtime/Intrinsics.cpp index b83e973ca..3b0eae310 100644 --- a/lib/Arch/Runtime/Intrinsics.cpp +++ b/lib/Arch/Runtime/Intrinsics.cpp @@ -29,7 +29,9 @@ extern "C" void __remill_mark_as_used(const void *); // Each architecture's semantics module defines this variable // See https://github.com/lifting-bits/remill/pull/631#issuecomment-1279989004 +extern "C" { extern State __remill_state; +} // extern C #if defined(REMILL_ON_SPARC32) || defined(REMILL_ON_SPARC64) extern RegisterWindow __remill_register_window; diff --git a/lib/Arch/SPARC32/Runtime/CMakeLists.txt b/lib/Arch/SPARC32/Runtime/CMakeLists.txt index 2a16d5c61..7cc5312d6 100644 --- a/lib/Arch/SPARC32/Runtime/CMakeLists.txt +++ b/lib/Arch/SPARC32/Runtime/CMakeLists.txt @@ -33,7 +33,7 @@ function(add_runtime_helper target_name little_endian) message(" > Generating runtime target: ${target_name}") # Visual C++ requires C++14 if(WIN32) - set(required_cpp_standard "c++14") + set(required_cpp_standard "c++17") else() set(required_cpp_standard "c++17") endif() diff --git a/lib/Arch/SPARC32/Runtime/Instructions.cpp b/lib/Arch/SPARC32/Runtime/Instructions.cpp index d348821b5..b6672eaa9 100644 --- a/lib/Arch/SPARC32/Runtime/Instructions.cpp +++ b/lib/Arch/SPARC32/Runtime/Instructions.cpp @@ -26,7 +26,9 @@ // A definition is required to ensure that LLVM doesn't optimize the `State` type out of the bytecode // See https://github.com/lifting-bits/remill/pull/631#issuecomment-1279989004 -State __remill_state; +extern "C" { +extern State __remill_state = {}; +} // extern C #define REG_PC state.pc.aword #define REG_NPC state.next_pc.aword diff --git a/lib/Arch/SPARC64/Runtime/CMakeLists.txt b/lib/Arch/SPARC64/Runtime/CMakeLists.txt index a7a2253d2..f79115912 100644 --- a/lib/Arch/SPARC64/Runtime/CMakeLists.txt +++ b/lib/Arch/SPARC64/Runtime/CMakeLists.txt @@ -34,7 +34,7 @@ function(add_runtime_helper target_name little_endian) # Visual C++ requires C++14 if(WIN32) - set(required_cpp_standard "c++14") + set(required_cpp_standard "c++17") else() set(required_cpp_standard "c++17") endif() diff --git a/lib/Arch/SPARC64/Runtime/Instructions.cpp b/lib/Arch/SPARC64/Runtime/Instructions.cpp index 019952d2d..aca8e3c33 100644 --- a/lib/Arch/SPARC64/Runtime/Instructions.cpp +++ b/lib/Arch/SPARC64/Runtime/Instructions.cpp @@ -26,7 +26,9 @@ // A definition is required to ensure that LLVM doesn't optimize the `State` type out of the bytecode // See https://github.com/lifting-bits/remill/pull/631#issuecomment-1279989004 -State __remill_state; +extern "C" { +extern State __remill_state = {}; +} // extern C #define REG_PC state.pc.aword #define REG_NPC state.next_pc.aword diff --git a/lib/Arch/Sleigh/AArch32Arch.cpp b/lib/Arch/Sleigh/AArch32Arch.cpp index b279ca1c1..5e767ff3e 100644 --- a/lib/Arch/Sleigh/AArch32Arch.cpp +++ b/lib/Arch/Sleigh/AArch32Arch.cpp @@ -17,12 +17,12 @@ #include "AArch32Arch.h" #include -#include #include #include #include #include #include +#include #include #include diff --git a/lib/Arch/Sleigh/ARMBase.cpp b/lib/Arch/Sleigh/ARMBase.cpp index ae201a09a..f502b4b7d 100644 --- a/lib/Arch/Sleigh/ARMBase.cpp +++ b/lib/Arch/Sleigh/ARMBase.cpp @@ -1,10 +1,10 @@ #include -#include #include #include #include #include #include +#include #include #include #include diff --git a/lib/Arch/Sleigh/X86Arch.cpp b/lib/Arch/Sleigh/X86Arch.cpp index 3e00766a0..753f07ea7 100644 --- a/lib/Arch/Sleigh/X86Arch.cpp +++ b/lib/Arch/Sleigh/X86Arch.cpp @@ -15,12 +15,12 @@ */ #include -#include #include #include #include #include #include +#include #include #include #include diff --git a/lib/Arch/X86/Arch.cpp b/lib/Arch/X86/Arch.cpp index 9b677fea3..9bfa74ef5 100644 --- a/lib/Arch/X86/Arch.cpp +++ b/lib/Arch/X86/Arch.cpp @@ -15,12 +15,12 @@ */ #include -#include #include #include #include #include #include +#include #include // For `Arch` and `ArchImpl`. #include diff --git a/lib/Arch/X86/Runtime/CMakeLists.txt b/lib/Arch/X86/Runtime/CMakeLists.txt index 170e2e90d..85f334937 100644 --- a/lib/Arch/X86/Runtime/CMakeLists.txt +++ b/lib/Arch/X86/Runtime/CMakeLists.txt @@ -30,7 +30,7 @@ function(add_runtime_helper target_name address_bit_size enable_avx enable_avx51 # Visual C++ requires C++14 if(WIN32) - set(required_cpp_standard "c++14") + set(required_cpp_standard "c++17") else() set(required_cpp_standard "c++11") endif() diff --git a/lib/Arch/X86/Runtime/Instructions.cpp b/lib/Arch/X86/Runtime/Instructions.cpp index 1a18e4c81..1401e2350 100644 --- a/lib/Arch/X86/Runtime/Instructions.cpp +++ b/lib/Arch/X86/Runtime/Instructions.cpp @@ -30,7 +30,9 @@ // A definition is required to ensure that LLVM doesn't optimize the `State` type out of the bytecode // See https://github.com/lifting-bits/remill/pull/631#issuecomment-1279989004 -State __remill_state; +extern "C" { +extern State __remill_state = {}; +} // extern C #define REG_IP state.gpr.rip.word #define REG_EIP state.gpr.rip.dword diff --git a/lib/BC/InstructionLifter.cpp b/lib/BC/InstructionLifter.cpp index 0acf2c7ae..be075d79b 100644 --- a/lib/BC/InstructionLifter.cpp +++ b/lib/BC/InstructionLifter.cpp @@ -353,16 +353,24 @@ llvm::Value *InstructionLifter::LoadWordRegValOrZero(llvm::BasicBlock *block, } auto val = LoadRegValue(block, state_ptr, reg_name); - auto val_type = llvm::dyn_cast_or_null(val->getType()); + auto val_type = val->getType(); auto word_type = zero->getType(); CHECK(val_type) << "Register " << reg_name << " expected to be an integer."; + + auto *val_int_type = llvm::dyn_cast(val_type); + auto *word_int_type = llvm::dyn_cast(word_type); + + if (!val_int_type || !word_int_type) { + llvm::report_fatal_error("val_type o word_type non sono tipi interi validi"); + } + + auto val_size = val_int_type->getBitWidth(); + auto word_size = word_int_type->getBitWidth(); - auto val_size = val_type->getBitWidth(); - auto word_size = word_type->getBitWidth(); CHECK_LE(val_size, word_size) << "Register " << reg_name << " expected to be no larger than the " - << "machine word size (" << word_type->getBitWidth() << " bits)."; + << "machine word size (" << (llvm::dyn_cast_or_null(word_type))->getBitWidth() << " bits)."; if (val_size < word_size) { val = new llvm::ZExtInst(val, word_type, llvm::Twine::createNull(), block); diff --git a/lib/BC/Optimizer.cpp b/lib/BC/Optimizer.cpp index 1b20f2955..39b5ce98f 100644 --- a/lib/BC/Optimizer.cpp +++ b/lib/BC/Optimizer.cpp @@ -17,7 +17,8 @@ #include "remill/BC/Optimizer.h" #include -#include +#include +#include #include #include #include @@ -29,9 +30,15 @@ #include #include #include +#include #include +#include +#include +#include +#include #include -#include +#include +#include #include #include #include @@ -46,81 +53,39 @@ namespace remill { void OptimizeModule(const remill::Arch *arch, llvm::Module *module, std::function generator, OptimizationGuide guide) { - - llvm::legacy::FunctionPassManager func_manager(module); - llvm::legacy::PassManager module_manager; - - auto TLI = - new llvm::TargetLibraryInfoImpl(llvm::Triple(module->getTargetTriple())); - - TLI->disableAllFunctions(); // `-fno-builtin`. - - llvm::PassManagerBuilder builder; - // TODO(alex): Some of the optimization passes that the builder adds still rely on typed pointers - // so we cannot use them. We should switch to using the new pass manager and choose which passes - // we want. - builder.OptLevel = 0; - builder.SizeLevel = 0; - builder.Inliner = llvm::createFunctionInliningPass(250); - builder.LibraryInfo = TLI; // Deleted by `llvm::~PassManagerBuilder`. - builder.DisableUnrollLoops = false; // Unroll loops! -#if LLVM_VERSION_NUMBER < LLVM_VERSION(16, 0) - builder.RerollLoops = false; -#endif - builder.SLPVectorize = guide.slp_vectorize; - builder.LoopVectorize = guide.loop_vectorize; - builder.VerifyInput = guide.verify_input; - builder.VerifyOutput = guide.verify_output; - builder.MergeFunctions = false; - - builder.populateFunctionPassManager(func_manager); - builder.populateModulePassManager(module_manager); - func_manager.doInitialization(); - llvm::Function *func = nullptr; - while (nullptr != (func = generator())) { - func_manager.run(*func); - } - func_manager.doFinalization(); - module_manager.run(*module); + OptimizeBareModule(module, guide); } // Optimize a normal module. This might not contain special Remill-specific // intrinsics functions like `__remill_jump`, etc. void OptimizeBareModule(llvm::Module *module, OptimizationGuide guide) { - llvm::legacy::FunctionPassManager func_manager(module); - llvm::legacy::PassManager module_manager; - auto TLI = - new llvm::TargetLibraryInfoImpl(llvm::Triple(module->getTargetTriple())); + llvm::ModuleAnalysisManager mam; + llvm::FunctionAnalysisManager fam; + llvm::LoopAnalysisManager lam; + llvm::CGSCCAnalysisManager cam; + + + llvm::PipelineTuningOptions opts; + opts.InlinerThreshold = 250; + llvm::PassBuilder pb(nullptr, opts); + + pb.registerModuleAnalyses(mam); + pb.registerFunctionAnalyses(fam); + pb.registerLoopAnalyses(lam); + pb.registerCGSCCAnalyses(cam); + pb.crossRegisterProxies(lam, fam, cam, mam); + + llvm::ModulePassManager mpm; + mpm.addPass(llvm::ModuleInlinerPass(llvm::getInlineParams(250))); - TLI->disableAllFunctions(); // `-fno-builtin`. + mpm.run(*module, mam); - llvm::PassManagerBuilder builder; - // TODO(alex): Some of the optimization passes that the builder adds still rely on typed pointers - // so we cannot use them. We should switch to using the new pass manager and choose which passes - // we want. - builder.OptLevel = 0; - builder.SizeLevel = 0; - builder.Inliner = llvm::createFunctionInliningPass(250); - builder.LibraryInfo = TLI; // Deleted by `llvm::~PassManagerBuilder`. - builder.DisableUnrollLoops = false; // Unroll loops! -#if LLVM_VERSION_NUMBER < LLVM_VERSION(16, 0) - builder.RerollLoops = false; -#endif - builder.SLPVectorize = guide.slp_vectorize; - builder.LoopVectorize = guide.loop_vectorize; - builder.VerifyInput = guide.verify_input; - builder.VerifyOutput = guide.verify_output; - builder.MergeFunctions = false; - builder.populateFunctionPassManager(func_manager); - builder.populateModulePassManager(module_manager); - func_manager.doInitialization(); - for (auto &func : *module) { - func_manager.run(func); - } - func_manager.doFinalization(); - module_manager.run(*module); + mam.clear(); + fam.clear(); + lam.clear(); + cam.clear(); } } // namespace remill diff --git a/lib/BC/Util.cpp b/lib/BC/Util.cpp index da06309e0..42cc1a619 100644 --- a/lib/BC/Util.cpp +++ b/lib/BC/Util.cpp @@ -47,6 +47,8 @@ #include #include #include +#include +#include #include "remill/Arch/Arch.h" #include "remill/Arch/Name.h" @@ -1121,6 +1123,7 @@ MoveConstantIntoModule(llvm::Constant *c, llvm::Module *dest_module, return nullptr; } } else if (auto ce = llvm::dyn_cast(c)) { + const llvm::DataLayout &DL = dest_module->getDataLayout(); switch (ce->getOpcode()) { case llvm::Instruction::Add: { const auto b = llvm::dyn_cast(ce); @@ -1145,20 +1148,16 @@ MoveConstantIntoModule(llvm::Constant *c, llvm::Module *dest_module, return ret; } case llvm::Instruction::And: { - auto ret = llvm::ConstantExpr::getAnd( - MoveConstantIntoModule(ce->getOperand(0), dest_module, value_map, - type_map), - MoveConstantIntoModule(ce->getOperand(1), dest_module, value_map, - type_map)); + auto ret = ConstantFoldBinaryOpOperands(ce->getOpcode(), MoveConstantIntoModule(ce->getOperand(0), dest_module, value_map, type_map), + MoveConstantIntoModule(ce->getOperand(1), dest_module, value_map, type_map), + DL); moved_c = ret; return ret; } case llvm::Instruction::Or: { - auto ret = llvm::ConstantExpr::getOr( - MoveConstantIntoModule(ce->getOperand(0), dest_module, value_map, - type_map), - MoveConstantIntoModule(ce->getOperand(1), dest_module, value_map, - type_map)); + auto ret = ConstantFoldBinaryOpOperands(ce->getOpcode(), MoveConstantIntoModule(ce->getOperand(0), dest_module, value_map, type_map), + MoveConstantIntoModule(ce->getOperand(1), dest_module, value_map, type_map), + DL); moved_c = ret; return ret; } @@ -1182,20 +1181,24 @@ MoveConstantIntoModule(llvm::Constant *c, llvm::Module *dest_module, return ret; } case llvm::Instruction::ZExt: { - auto ret = llvm::ConstantExpr::getZExt( - MoveConstantIntoModule(ce->getOperand(0), dest_module, value_map, - type_map), - type); - moved_c = ret; - return ret; + auto newConst = MoveConstantIntoModule(ce->getOperand(0), dest_module, value_map, type_map); + unsigned destSize = DL.getTypeSizeInBits(type); + + if (auto *constInt = llvm::dyn_cast(newConst)) { + auto ret = llvm::ConstantInt::get(type, constInt->getValue().zext(destSize)); + moved_c = ret; + return ret; + } } case llvm::Instruction::SExt: { - auto ret = llvm::ConstantExpr::getSExt( - MoveConstantIntoModule(ce->getOperand(0), dest_module, value_map, - type_map), - type); - moved_c = ret; - return ret; + auto newConst = MoveConstantIntoModule(ce->getOperand(0), dest_module, value_map, type_map); + unsigned destSize = DL.getTypeSizeInBits(type); + + if (auto *constInt = llvm::dyn_cast(newConst)) { + auto ret = llvm::ConstantInt::get(type, constInt->getValue().sext(destSize)); + moved_c = ret; + return ret; + } } case llvm::Instruction::Trunc: { auto ret = llvm::ConstantExpr::getTrunc( @@ -1205,17 +1208,6 @@ MoveConstantIntoModule(llvm::Constant *c, llvm::Module *dest_module, moved_c = ret; return ret; } - case llvm::Instruction::Select: { - auto ret = llvm::ConstantExpr::getSelect( - MoveConstantIntoModule(ce->getOperand(0), dest_module, value_map, - type_map), - MoveConstantIntoModule(ce->getOperand(1), dest_module, value_map, - type_map), - MoveConstantIntoModule(ce->getOperand(2), dest_module, value_map, - type_map)); - moved_c = ret; - return ret; - } case llvm::Instruction::Shl: { const auto b = llvm::dyn_cast(ce); auto ret = llvm::ConstantExpr::getShl( @@ -1229,23 +1221,17 @@ MoveConstantIntoModule(llvm::Constant *c, llvm::Module *dest_module, } case llvm::Instruction::LShr: { const auto b = llvm::dyn_cast(ce); - auto ret = llvm::ConstantExpr::getLShr( - MoveConstantIntoModule(ce->getOperand(0), dest_module, value_map, - type_map), - MoveConstantIntoModule(ce->getOperand(1), dest_module, value_map, - type_map), - b->isExact()); + auto ret = ConstantFoldBinaryOpOperands(ce->getOpcode(), MoveConstantIntoModule(ce->getOperand(0), dest_module, value_map, type_map), + MoveConstantIntoModule(ce->getOperand(1), dest_module, value_map, type_map), + DL/* b->isExact()*/); moved_c = ret; return ret; } case llvm::Instruction::AShr: { const auto b = llvm::dyn_cast(ce); - auto ret = llvm::ConstantExpr::getAShr( - MoveConstantIntoModule(ce->getOperand(0), dest_module, value_map, - type_map), - MoveConstantIntoModule(ce->getOperand(1), dest_module, value_map, - type_map), - b->isExact()); + auto ret = ConstantFoldBinaryOpOperands(ce->getOpcode(), MoveConstantIntoModule(ce->getOperand(0), dest_module, value_map, type_map), + MoveConstantIntoModule(ce->getOperand(1), dest_module, value_map,type_map), + DL /* b->isExact()*/); moved_c = ret; return ret; } diff --git a/lib/OS/OS.cpp b/lib/OS/OS.cpp index 3ee6b0c97..8b4074185 100644 --- a/lib/OS/OS.cpp +++ b/lib/OS/OS.cpp @@ -18,7 +18,7 @@ #include #include -#include +#include namespace remill { diff --git a/scripts/build.sh b/scripts/build.sh index 2da55623e..3d541ebc7 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -23,11 +23,11 @@ DOWNLOAD_DIR="$( cd "$( dirname "${SRC_DIR}" )" && pwd )/lifting-bits-downloads" CURR_DIR=$( pwd ) BUILD_DIR="${CURR_DIR}/remill-build" INSTALL_DIR=/usr/local -LLVM_VERSION=llvm-16 +LLVM_VERSION=llvm-17 OS_VERSION= ARCH_VERSION= BUILD_FLAGS= -CXX_COMMON_VERSION="0.3.1" +CXX_COMMON_VERSION="0.6.0" CREATE_PACKAGES=true # There are pre-build versions of various libraries for specific @@ -164,9 +164,9 @@ function DownloadLibraries #BUILD_FLAGS="${BUILD_FLAGS} -DCMAKE_OSX_SYSROOT=${sdk_root}" # Min version supported - OS_VERSION="macos-12" + OS_VERSION="macos-13" # Hard-coded to match pre-built binaries in CI - XCODE_VERSION="14.2" + XCODE_VERSION="15.0" SYSTEM_VERSION=$(sw_vers -productVersion) if [[ "${SYSTEM_VERSION}" == "13.*" ]]; then echo "Found MacOS Ventura" @@ -300,6 +300,10 @@ function GetLLVMVersion LLVM_VERSION=llvm-16 return 0 ;; + 17) + LLVM_VERSION=llvm-17 + return 0 + ;; *) # unknown option echo "[x] Unknown or unsupported LLVM version ${1}. You may be able to manually build it with cxx-common." diff --git a/scripts/docker-lifter-entrypoint.sh b/scripts/docker-lifter-entrypoint.sh index 335cb3357..ed24109d7 100755 --- a/scripts/docker-lifter-entrypoint.sh +++ b/scripts/docker-lifter-entrypoint.sh @@ -4,11 +4,8 @@ V="" case ${LLVM_VERSION} in - llvm15*) - V=15 - ;; - llvm16*) - V=16 + llvm17*) + V=17 ;; *) echo "Unknown LLVM version: ${LLVM_VERSION}" diff --git a/test_runner_lib/CMakeLists.txt b/test_runner_lib/CMakeLists.txt index d6085b968..4b72f3b6f 100644 --- a/test_runner_lib/CMakeLists.txt +++ b/test_runner_lib/CMakeLists.txt @@ -14,21 +14,21 @@ set(TEST_RUNNER_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include") -add_library( - test-runner - STATIC - TestRunner.cpp - "${TEST_RUNNER_INCLUDE_DIR}/test_runner/TestRunner.h" - "${TEST_RUNNER_INCLUDE_DIR}/test_runner/TestOutputSpec.h" -) +#add_library( +# test-runner +# STATIC +# TestRunner.cpp +# "${TEST_RUNNER_INCLUDE_DIR}/test_runner/TestRunner.h" +# "${TEST_RUNNER_INCLUDE_DIR}/test_runner/TestOutputSpec.h" +#) -target_link_libraries( - test-runner - PRIVATE - remill - glog::glog -) +#target_link_libraries( +# test-runner +# PRIVATE +# remill +# glog::glog +#) -target_include_directories(test-runner PUBLIC "${TEST_RUNNER_INCLUDE_DIR}") +#target_include_directories(test-runner PUBLIC "${TEST_RUNNER_INCLUDE_DIR}") -set_property(TARGET test-runner PROPERTY POSITION_INDEPENDENT_CODE ON) +#set_property(TARGET test-runner PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/test_runner_lib/include/test_runner/TestRunner.h b/test_runner_lib/include/test_runner/TestRunner.h index 85d23755a..612efbdd8 100644 --- a/test_runner_lib/include/test_runner/TestRunner.h +++ b/test_runner_lib/include/test_runner/TestRunner.h @@ -117,7 +117,7 @@ void ExecuteLiftedFunction( llvm::InitializeNativeTarget(); llvm::InitializeNativeTargetAsmParser(); llvm::InitializeNativeTargetAsmPrinter(); - llvm::InitializeAllTargetMCs(); + auto res = remill::VerifyModuleMsg(tgt_mod.get()); if (res.has_value()) { diff --git a/tests/AArch64/CMakeLists.txt b/tests/AArch64/CMakeLists.txt index 1dee6d921..54280a5a6 100644 --- a/tests/AArch64/CMakeLists.txt +++ b/tests/AArch64/CMakeLists.txt @@ -4,7 +4,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -34,14 +34,13 @@ target_compile_options(lift-aarch64-tests -DIN_TEST_GENERATOR ) - file(GLOB AARCH64_TEST_FILES "${CMAKE_CURRENT_LIST_DIR}/*/*.S" ) set_target_properties(lift-aarch64-tests PROPERTIES OBJECT_DEPENDS "${AARCH64_TEST_FILES}") -target_link_libraries(lift-aarch64-tests PUBLIC remill ${PROJECT_LIBRARIES} ) +target_link_libraries(lift-aarch64-tests PUBLIC remill ${PROJECT_LIBRARIES}) target_include_directories(lift-aarch64-tests PUBLIC ${PROJECT_INCLUDEDIRECTORIES}) target_include_directories(lift-aarch64-tests PRIVATE ${CMAKE_SOURCE_DIR}) @@ -65,13 +64,12 @@ add_custom_command( ) add_custom_command( - OUTPUT tests_aarch64.S + OUTPUT tests_aarch64.S COMMAND ${CMAKE_BC_COMPILER} - -Wno-override-module - -S -O1 -g0 - -c tests_aarch64.bc - -o tests_aarch64.S - -mllvm -opaque-pointers + -Wno-override-module + -S -O1 -g0 + -c tests_aarch64.bc + -o tests_aarch64.S DEPENDS tests_aarch64.bc ) @@ -80,10 +78,10 @@ target_include_directories(run-aarch64-tests PUBLIC ${PROJECT_INCLUDEDIRECTORIES target_include_directories(run-aarch64-tests PRIVATE ${CMAKE_SOURCE_DIR}) target_compile_options(run-aarch64-tests - PRIVATE #-I${CMAKE_SOURCE_DIR} - -DADDRESS_SIZE_BITS=64 - -DGTEST_HAS_RTTI=0 - -DGTEST_HAS_TR1_TUPLE=0 + PRIVATE # -I${CMAKE_SOURCE_DIR} + -DADDRESS_SIZE_BITS=64 + -DGTEST_HAS_RTTI=0 + -DGTEST_HAS_TR1_TUPLE=0 ) message(STATUS "Adding test: aarch64 as run-aarch64-tests") diff --git a/tests/X86/CMakeLists.txt b/tests/X86/CMakeLists.txt index 471e2ea46..dd298ebec 100644 --- a/tests/X86/CMakeLists.txt +++ b/tests/X86/CMakeLists.txt @@ -4,7 +4,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -52,7 +52,7 @@ function(COMPILE_X86_TESTS name address_size has_avx has_avx512) add_custom_command( OUTPUT tests_${name}.S - COMMAND ${CMAKE_BC_COMPILER} -Wno-override-module -S -O0 -g0 -c tests_${name}.bc -o tests_${name}.S -mllvm -opaque-pointers + COMMAND ${CMAKE_BC_COMPILER} -Wno-override-module -S -O0 -g0 -c tests_${name}.bc -o tests_${name}.S DEPENDS tests_${name}.bc ) @@ -75,7 +75,7 @@ find_package(GTest CONFIG REQUIRED) enable_testing() -if (NOT APPLE) +if(NOT APPLE) COMPILE_X86_TESTS(x86 32 0 0) COMPILE_X86_TESTS(x86_avx 32 1 0) endif()