Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/master' into alex/llvm-15
Browse files Browse the repository at this point in the history
  • Loading branch information
@tetsuo-cpp
tetsuo-cpp committed Nov 16, 2022
2 parents d2d0557 + d3ccd47 commit 3f00eb5
Show file tree
Hide file tree
Showing 45 changed files with 1,984 additions and 3,899 deletions.
158 changes: 93 additions & 65 deletions .github/workflows/build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,14 @@ on:

push:
branches:
- '*'
- "*"

tags:
- '*'
- "*"

pull_request:
branches:
- '*'



- "*"

jobs:
cleanup_stale_workflows:
Expand All @@ -47,21 +44,18 @@ jobs:
run: |
python3 .github/scripts/abort_previous_workflows.py
build_linux:
needs: cleanup_stale_workflows

strategy:
fail-fast: false
matrix:
image:
- { name: 'ubuntu', tag: '20.04' }
- { name: "ubuntu", tag: "20.04" }
binja:
- {channel: 'headless', version: '3.1.3479'}
llvm: [ '14', '15' ]
cxxcommon_version: [ 'v0.2.12' ]
- { channel: "headless", version: "3.1.3479" }
llvm: ["14", "15"]
cxxcommon_version: ["v0.2.12"]

runs-on: ubuntu-20.04
container:
Expand Down Expand Up @@ -130,7 +124,8 @@ jobs:
make \
ninja-build \
ccache \
rpm
rpm \
openssh-client
- name: Clone the anvill repository
uses: actions/checkout@v2
Expand All @@ -142,7 +137,7 @@ jobs:
- name: Clone the remill repository
uses: actions/checkout@v2
with:
repository: 'lifting-bits/remill'
repository: "lifting-bits/remill"
path: ${{ steps.build_paths.outputs.REL_SOURCE }}/remill
fetch-depth: 0

Expand All @@ -152,6 +147,49 @@ jobs:
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config --global user.name "github-actions[bot]"
<<<<<<< HEAD
=======
- name: Install stable rust
shell: bash
run: |
apt-get install -y cargo
- name: "Install Just"
shell: bash
run: |
cargo install just
- uses: actions/setup-java@v2
with:
distribution: "temurin"
java-version: "11"
- name: Clone Ghidra Spec Generation
uses: actions/checkout@v2
with:
path: ${{ steps.build_paths.outputs.REL_SOURCE }}/irene3
repository: "trailofbits/irene3"
fetch-depth: 0
submodules: false
ssh-key: "${{ secrets.IRENE3_DEPLOY }}"

- name: Add cargo bin to path
shell: bash
run: |
echo ~/.cargo/bin >>$GITHUB_PATH
- name: "Setup Ghidra"
run: |
wget https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_10.1.5_build/ghidra_10.1.5_PUBLIC_20220726.zip --output-document=ghidra.zip
unzip ghidra.zip
echo "GHIDRA_INSTALL_DIR=$(pwd)/ghidra_10.1.5_PUBLIC" >> $GITHUB_ENV
- name: Install Ghidra Spec Generation
shell: bash
working-directory: ${{ steps.build_paths.outputs.REL_SOURCE }}/irene3
run: |
just install-irene3-ghidra
>>>>>>> origin/master
- name: Select the remill version
shell: bash
run: |
Expand Down Expand Up @@ -311,24 +349,7 @@ jobs:

run: |
scripts/build-preset.sh debug
- name: Install Binary Ninja

env:
BINJA_DECODE_KEY: ${{ secrets.BINJA_DECODE_KEY }}

shell: bash
working-directory: ${{ steps.build_paths.outputs.SOURCE }}/anvill/ci
run: |
./install_binja.sh
- name: Update Binary Ninja
shell: bash
working-directory: ${{ steps.build_paths.outputs.SOURCE }}/anvill/ci
run: |
python3 ./switcher.py --version_string ${{ matrix.binja.version }} ${{ matrix.binja.channel }}
- name: Run the tests

env:
BINJA_DECODE_KEY: ${{ secrets.BINJA_DECODE_KEY }}
CTEST_OUTPUT_ON_FAILURE: 1
Expand Down Expand Up @@ -366,29 +387,25 @@ jobs:
run: |
dpkg -i ${{ steps.package_names.outputs.DEB_PACKAGE_PATH }}
- name: Test the DEB package
run: |
python3 -m anvill --help
- name: Run Integration Tests (AMP Challenge Binaries)
shell: bash
working-directory: ${{ steps.build_paths.outputs.REL_SOURCE }}/anvill
run: |
python3 -m pip install -r libraries/lifting-tools-ci/requirements.txt
scripts/test-amp-challenge-bins.sh \
--python-cmd "python3 -m anvill" \
--decompile-cmd "anvill-decompile-json"
--ghidra-install-dir $GHIDRA_INSTALL_DIR \
--decompile-cmd "anvill-decompile-spec"
env:
TOB_AMP_PASSPHRASE: ${{secrets.TOB_AMP_PASSPHRASE}}

- name: Run Integration Tests (AnghaBench 1K)
shell: bash
working-directory: ${{ steps.build_paths.outputs.REL_SOURCE }}/anvill
run: |
python3 -m pip install -r libraries/lifting-tools-ci/requirements.txt
scripts/test-angha-1k.sh \
--python-cmd "python3 -m anvill" \
--decompile-cmd "anvill-decompile-json"
# - name: Run Integration Tests (AnghaBench 1K)
# shell: bash
# working-directory: ${{ steps.build_paths.outputs.REL_SOURCE }}/anvill
# run: |
# python3 -m pip install -r libraries/lifting-tools-ci/requirements.txt
# scripts/test-angha-1k.sh \
# --python-cmd "python3 -m anvill" \
# --decompile-cmd "anvill-decompile-json"

- name: Store the DEB package
uses: actions/upload-artifact@v1
Expand All @@ -408,9 +425,6 @@ jobs:
name: ${{ matrix.image.name }}-${{ matrix.image.tag }}_llvm${{ matrix.llvm }}_tgz_package
path: ${{ steps.package_names.outputs.TGZ_PACKAGE_PATH }}




# Note that we have a limit of 5 concurrent workers, org-wide.
# It is best to try and keep the amount of macOS jobs as low
# as possible.
Expand All @@ -420,9 +434,15 @@ jobs:
strategy:
fail-fast: false
matrix:
<<<<<<< HEAD
os: [ 'macos-11' ]
llvm: [ '14', '15' ]
cxxcommon_version: [ 'v0.2.12' ]
=======
os: ["macos-11"]
llvm: ["14"]
cxxcommon_version: ["v0.2.10"]
>>>>>>> origin/master

runs-on: ${{ matrix.os }}

Expand Down Expand Up @@ -491,7 +511,7 @@ jobs:
- name: Clone the remill repository
uses: actions/checkout@v2
with:
repository: 'lifting-bits/remill'
repository: "lifting-bits/remill"
path: ${{ steps.build_paths.outputs.REL_SOURCE }}/remill
fetch-depth: 0

Expand Down Expand Up @@ -695,12 +715,9 @@ jobs:
name: ${{ matrix.os }}_llvm${{ matrix.llvm }}_tgz_package
path: ${{ steps.package_names.outputs.TGZ_PACKAGE_PATH }}




release_packages:
# Do not run the release procedure if any of the builds has failed
needs: [ build_linux, build_macos ]
needs: [build_linux, build_macos]
runs-on: ubuntu-20.04
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')

Expand Down Expand Up @@ -776,8 +793,9 @@ jobs:
llvm: ["14", "15"]
ubuntu: ["20.04"]
binja:
- {channel: 'headless', version: '3.1.3479'}
- { channel: "headless", version: "3.1.3479" }
steps:
<<<<<<< HEAD
- uses: actions/checkout@v2
with:
submodules: true
Expand All @@ -802,14 +820,24 @@ jobs:
docker run --rm -v $(pwd):/workspace -w /workspace docker.pkg.github.com/lifting-bits/anvill/anvill-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest -spec /workspace/bin/Decompile/tests/specs/ret0.json -bc_out /tmp/ret0.bc -ir_out /tmp/ret0.ir
docker run --rm -v $(pwd):/workspace -w /workspace docker.pkg.github.com/lifting-bits/anvill/anvill-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest -spec /workspace/bin/Decompile/tests/specs/jmp_ret0.json -bc_out /tmp/jmp_ret0.bc -ir_out /tmp/jmp_ret0.ir
docker run --rm -v $(pwd):/workspace -w /workspace docker.pkg.github.com/lifting-bits/anvill/anvill-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest -spec /workspace/bin/Decompile/tests/specs/jmp_ret1.json -bc_out /tmp/jmp_ret1.bc -ir_out /tmp/jmp_ret1.ir
=======
- uses: actions/checkout@v2
with:
submodules: true
- name: Set remill-commit-id
shell: bash
run: |
remill_commit_id_path=".remill_commit_id"
>>>>>>> origin/master

- name: Integration Decompile and Specify Decompiled Bitcode
run: |
docker run --rm -v $(pwd):/workspace -w /workspace --entrypoint "/bin/bash" docker.pkg.github.com/lifting-bits/anvill/anvill-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest -c "anvill-decompile-json -spec /workspace/bin/Decompile/tests/specs/ret0.json -bc_out /tmp/ret0.bc && anvill-specify-bitcode --bc_in /tmp/ret0.bc"
- name: Build with Binja
run: |
docker build . -t anvill-binja-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest -f Dockerfile --build-arg REMILL_COMMIT_ID=${REMILL_COMMIT_ID} --build-arg UBUNTU_VERSION=${{ matrix.ubuntu }} --build-arg BINJA_VERSION=${{ matrix.binja.version }} --build-arg BINJA_CHANNEL=${{ matrix.binja.channel }} --build-arg ARCH=amd64 --build-arg LLVM_VERSION=${{ matrix.llvm }} --build-arg BINJA_DECODE_KEY=${{ secrets.BINJA_DECODE_KEY }} --target binja
- name: Test with Binja
run: |
docker run --rm -v $(pwd):/workspace -w /workspace anvill-binja-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --bin_in /usr/bin/yes --spec_out /tmp/yes.json
if [[ -f "${remill_commit_id_path}" ]] ; then
remill_commit_id=$(cat ${remill_commit_id_path})
echo "Using the following remill commit: ${remill_commit_id} in docker"
else
remill_commit_id="master"
echo "WARNING: Builds/tests will work against remill main commit"
fi
echo "REMILL_COMMIT_ID=$remill_commit_id" >> $GITHUB_ENV
- name: Build LLVM ${{ matrix.llvm }} on ${{ matrix.ubuntu }}
run: |
docker build . -t docker.pkg.github.com/lifting-bits/anvill/anvill-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest -f Dockerfile --target=dist --build-arg REMILL_COMMIT_ID=${REMILL_COMMIT_ID} --build-arg UBUNTU_VERSION=${{ matrix.ubuntu }} --build-arg ARCH=amd64 --build-arg LLVM_VERSION=${{ matrix.llvm }}
27 changes: 9 additions & 18 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ ARG CXX_COMMON_VERSION=0.2.12
ARG DISTRO_BASE=ubuntu${UBUNTU_VERSION}
ARG BUILD_BASE=ubuntu:${UBUNTU_VERSION}
ARG LIBRARIES=/opt/trailofbits
ARG BINJA_DECODE_KEY


# Run-time dependencies go here
Expand Down Expand Up @@ -43,11 +42,12 @@ WORKDIR /dependencies


# cxx-common
ADD https://github.com/lifting-bits/cxx-common/releases/download/v${CXX_COMMON_VERSION}/vcpkg_ubuntu-${UBUNTU_VERSION}_llvm-${LLVM_VERSION}_amd64.tar.xz vcpkg_ubuntu-${UBUNTU_VERSION}_llvm-${LLVM_VERSION}_amd64.tar.xz
RUN tar -xJf vcpkg_ubuntu-${UBUNTU_VERSION}_llvm-${LLVM_VERSION}_amd64.tar.xz && \
#ADD https://github.com/lifting-bits/cxx-common/releases/download/v${CXX_COMMON_VERSION}/vcpkg_ubuntu-${UBUNTU_VERSION}_llvm-${LLVM_VERSION}_amd64.tar.xz vcpkg_ubuntu-${UBUNTU_VERSION}_llvm-${LLVM_VERSION}_amd64.tar.xz
RUN curl -L https://github.com/lifting-bits/cxx-common/releases/download/v${CXX_COMMON_VERSION}/vcpkg_ubuntu-${UBUNTU_VERSION}_llvm-${LLVM_VERSION}_amd64.tar.xz \
-o vcpkg_ubuntu-${UBUNTU_VERSION}_llvm-${LLVM_VERSION}_amd64.tar.xz && \
tar -xJf vcpkg_ubuntu-${UBUNTU_VERSION}_llvm-${LLVM_VERSION}_amd64.tar.xz && \
rm vcpkg_ubuntu-${UBUNTU_VERSION}_llvm-${LLVM_VERSION}_amd64.tar.xz


RUN git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" && git config --global user.name "github-actions[bot]"
RUN git clone "https://github.com/lifting-bits/remill.git" remill && cd remill && git checkout ${REMILL_COMMIT_ID}

Expand Down Expand Up @@ -87,6 +87,7 @@ COPY . ./
RUN source ${VIRTUAL_ENV}/bin/activate && \
cmake -G Ninja -B build -S . \
-DANVILL_ENABLE_INSTALL=true \
-DANVILL_ENABLE_TESTS=true \
-Dremill_DIR=${LIBRARIES}/cmake/remill \
-Dsleigh_DIR=${LIBRARIES}/cmake/sleigh \
-DCMAKE_INSTALL_PREFIX:PATH="${LIBRARIES}" \
Expand All @@ -95,6 +96,9 @@ RUN source ${VIRTUAL_ENV}/bin/activate && \
&& \
cmake --build build --target install

# Run Anvill tests
RUN cd build && CTEST_OUTPUT_ON_FAILURE=1 ctest -V

FROM base AS dist
ARG LLVM_VERSION
ARG LIBRARIES
Expand All @@ -109,13 +113,10 @@ COPY --from=build ${LIBRARIES} ${LIBRARIES}

# Target no longer installs at a version

ENTRYPOINT ["anvill-decompile-json"]
ENTRYPOINT ["anvill-decompile-spec"]


FROM dist as binja
ARG BINJA_DECODE_KEY
ARG BINJA_VERSION
ARG BINJA_CHANNEL

ENV VIRTUAL_ENV=/opt/trailofbits/venv

Expand All @@ -124,16 +125,6 @@ RUN apt-get update && \
apt-get install -qqy gpg unzip && \
rm -rf /var/lib/apt/lists/*

COPY ci /dependencies/binja_install

RUN export BINJA_DECODE_KEY="${BINJA_DECODE_KEY}" && \
source ${VIRTUAL_ENV}/bin/activate && \
cd /dependencies/binja_install && \
if [[ "${BINJA_DECODE_KEY}" != "" ]]; then ./install_binja.sh && python3 switcher.py --version_string ${BINJA_VERSION} ${BINJA_CHANNEL}; fi

# Keep this here to sanity check Binary Ninja API Installation & version
RUN python3 --version && python3 -c "import binaryninja; print(binaryninja.core_version())"

COPY scripts/docker-spec-entrypoint.sh /opt/trailofbits/docker-spec-entrypoint.sh
ENTRYPOINT ["/opt/trailofbits/docker-spec-entrypoint.sh"]

Expand Down
Loading

0 comments on commit 3f00eb5

Please sign in to comment.