Arm fixes + upload archives for each build (#38) #88
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and test | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - "v*.*.*" | |
| pull_request: | |
| branches: | |
| - main | |
| permissions: | |
| contents: write | |
| env: | |
| BORING_SSL_COMMIT: d24a38200fef19150eef00cad35b138936c08767 | |
| jobs: | |
| build-and-test-linux: | |
| name: (Linux ${{ matrix.arch }}) Build curl-impersonate and run the tests | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: [x86_64] | |
| include: | |
| - arch: x86_64 | |
| go_arch: amd64 | |
| docker_arch: linux/amd64 | |
| host: x86_64-linux-gnu | |
| capture_interface: eth0 | |
| image: quay.io/pypa/manylinux2014_x86_64:latest | |
| - arch: aarch64 | |
| go_arch: arm64 | |
| docker_arch: linux/arm/v8 | |
| host: aarch64-linux-gnu | |
| capture_interface: eth0 | |
| image: quay.io/pypa/manylinux2014_aarch64:latest | |
| - arch: arm | |
| go_arch: armv6l | |
| docker_arch: linux/arm/v7 | |
| host: arm-linux-gnu | |
| capture_interface: eth0 | |
| image: ghcr.io/bjia56/armv7l-wheel-builder:main | |
| steps: | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - name: Install dependencies | |
| run: | | |
| python_bin_dir=$(docker run ${{ matrix.image }} python3.10 -c 'import sys; import os; print(os.path.dirname(os.path.realpath(sys.executable)))') | |
| echo "runner_uid=$(id -u)" >> $GITHUB_ENV | |
| echo "runner_gid=$(id -g)" >> $GITHUB_ENV | |
| echo "runner_home=$HOME" >> $GITHUB_ENV | |
| docker build -t curl-impersonate-builder -f - . <<EOF | |
| FROM ${{ matrix.image }} | |
| ENV PATH=/usr/local/go/bin:/opt/rh/devtoolset-10/root/usr/bin:$python_bin_dir:\$PATH | |
| ENV HOME=$HOME | |
| ENV CC=/opt/rh/devtoolset-10/root/usr/bin/gcc | |
| ENV CXX=/opt/rh/devtoolset-10/root/usr/bin/g++ | |
| ENV LD=/usr/bin/ld | |
| RUN yum install -y libcurl-devel | |
| RUN python3.10 -m pip install ninja cmake gyp-next --extra-index-url https://bjia56.github.io/armv7l-wheels/ | |
| RUN rm -rf /usr/local/go && \ | |
| curl -o /tmp/go.tar.gz -L https://go.dev/dl/go1.19.10.linux-${{ matrix.go_arch }}.tar.gz && \ | |
| tar -C /usr/local -xzf /tmp/go.tar.gz | |
| RUN rm -f /usr/local/bin/python3 && \ | |
| rm -f /usr/local/bin/python && \ | |
| ln -s /usr/local/bin/python3.10 /usr/local/bin/python3 && \ | |
| ln -s /usr/local/bin/python3.10 /usr/local/bin/python | |
| EOF | |
| - name: Check out the repo | |
| uses: actions/checkout@v2 | |
| - name: Run configure script | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| image: curl-impersonate-builder | |
| options: > | |
| -v ${{ env.runner_home }}:${{ env.runner_home }} | |
| --workdir ${{ github.workspace }} | |
| --user ${{ env.runner_uid }}:${{ env.runner_gid }} | |
| run: | | |
| set -e | |
| mkdir ${{ runner.temp }}/install | |
| ./configure --prefix=${{ runner.temp }}/install \ | |
| --with-ca-path=/etc/ssl/certs \ | |
| --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt | |
| # Cache the build of BoringSSL, which is the longest part of the build | |
| # We must cache the .zip as well, otherwise the Makefile will | |
| # rebuild BoringSSL. This whole thing is a bit hacky, but necessary to | |
| # reduce the insanely long build times. | |
| - name: Cache BoringSSL source | |
| uses: actions/cache@v3 | |
| with: | |
| path: boringssl.zip | |
| key: ${{ runner.os }}-${{ matrix.arch }}-boring-source-${{ env.BORING_SSL_COMMIT }} | |
| - name: Cache BoringSSL build | |
| id: cache-boringssl | |
| uses: actions/cache@v3 | |
| with: | |
| path: boringssl/build | |
| key: ${{ runner.os }}-${{ matrix.arch }}-boring-build-${{ env.BORING_SSL_COMMIT }}-${{ hashFiles('chrome/patches/boringssl*.patch') }} | |
| # Trick the Makefile into skipping the BoringSSL build step | |
| # if it was found in the cache. See Makefile.in | |
| - name: Post BoringSSL cache restore | |
| if: ${{ steps.cache-boringssl.outputs.cache-hit != false }} | |
| run: | | |
| touch boringssl.zip | |
| touch boringssl/.patched | |
| find boringssl/build -type f | xargs touch | |
| - name: Build the Chrome version of curl-impersonate | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| image: curl-impersonate-builder | |
| options: > | |
| -v ${{ env.runner_home }}:${{ env.runner_home }} | |
| --workdir ${{ github.workspace }} | |
| --user ${{ env.runner_uid }}:${{ env.runner_gid }} | |
| run: | | |
| set -e | |
| make chrome-build | |
| make chrome-checkbuild | |
| make chrome-install | |
| - name: Install test dependencies | |
| if: matrix.arch == 'x86_64' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install build-essential pkg-config cmake ninja-build curl autoconf automake libtool | |
| # Chrome version dependencies | |
| sudo apt-get install golang-go | |
| # Needed to compile 'minicurl' | |
| sudo apt-get install libcurl4-openssl-dev | |
| # More dependencies for the tests | |
| sudo apt-get install tcpdump nghttp2-server | |
| - name: Prepare the tests | |
| if: matrix.arch == 'x86_64' | |
| run: | | |
| # Compile 'minicurl' which is used by the tests | |
| gcc -Wall -Werror -o ${{ runner.temp }}/install/bin/minicurl tests/minicurl.c `curl-config --libs` | |
| - uses: actions/setup-python@v4 | |
| if: matrix.arch == 'x86_64' | |
| with: | |
| python-version: '3.10' | |
| - name: Install dependencies for the tests script | |
| if: matrix.arch == 'x86_64' | |
| run: | | |
| pip3 install -r tests/requirements.txt | |
| # For now we can only run the tests when native | |
| # tests run the curl-impersonate binary locally. | |
| - name: Run the tests | |
| if: matrix.arch == 'x86_64' | |
| run: | | |
| cd tests | |
| # sudo is needed for capturing packets | |
| python_bin=$(which python3) | |
| sudo $python_bin -m pytest . --log-cli-level DEBUG --install-dir ${{ runner.temp }}/install --capture-interface ${{ matrix.capture_interface }} | |
| # Upload pre-compiled binaries to GitHub releases page and Github actions archives. | |
| - name: Create tar release files for libcurl-impersonate | |
| run: | | |
| cd ${{ runner.temp }}/install/lib | |
| if [[ "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then | |
| tar -c -z -f ${{ runner.temp }}/libcurl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz libcurl-impersonate* | |
| else | |
| tar -c -z -f ${{ runner.temp }}/libcurl-impersonate.${{ matrix.host }}.tar.gz libcurl-impersonate* | |
| fi | |
| echo "release_file_lib=${{ runner.temp }}/libcurl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz" >> $GITHUB_ENV | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: libcurl-impersonate.${{ matrix.host }}.tar.gz | |
| path: ${{ runner.temp }}/libcurl-impersonate*.tar.gz | |
| - name: Clean build | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| image: curl-impersonate-builder | |
| options: > | |
| -v ${{ env.runner_home }}:${{ env.runner_home }} | |
| --workdir ${{ github.workspace }} | |
| --user ${{ env.runner_uid }}:${{ env.runner_gid }} | |
| run: | | |
| set -e | |
| make chrome-clean | |
| rm -Rf ${{ runner.temp }}/install | |
| mkdir ${{ runner.temp }}/install | |
| # Recompile curl-impersonate statically when doing a release. | |
| - name: Reconfigure statically | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| image: curl-impersonate-builder | |
| options: > | |
| -v ${{ env.runner_home }}:${{ env.runner_home }} | |
| --workdir ${{ github.workspace }} | |
| --user ${{ env.runner_uid }}:${{ env.runner_gid }} | |
| run: | | |
| set -e | |
| ./configure --prefix=${{ runner.temp }}/install --enable-static | |
| - name: Rebuild statically | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| image: curl-impersonate-builder | |
| options: > | |
| -v ${{ env.runner_home }}:${{ env.runner_home }} | |
| --workdir ${{ github.workspace }} | |
| --user ${{ env.runner_uid }}:${{ env.runner_gid }} | |
| run: | | |
| set -e | |
| make chrome-build | |
| make chrome-checkbuild | |
| make chrome-install-strip | |
| - name: Create tar release files for curl-impersonate | |
| run: | | |
| cd ${{ runner.temp }}/install/bin | |
| if [[ "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then | |
| tar -c -z -f ${{ runner.temp }}/curl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz curl-impersonate-chrome curl_* | |
| else | |
| tar -c -z -f ${{ runner.temp }}/curl-impersonate.${{ matrix.host }}.tar.gz curl-impersonate-chrome curl_* | |
| fi | |
| echo "release_file_bin=${{ runner.temp }}/curl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz" >> $GITHUB_ENV | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: curl-impersonate.${{ matrix.host }}.tar.gz | |
| path: ${{ runner.temp }}/curl-impersonate*.tar.gz | |
| - name: Upload release files | |
| uses: softprops/action-gh-release@v1 | |
| if: startsWith(github.ref, 'refs/tags/') | |
| with: | |
| files: | | |
| ${{ env.release_file_lib }} | |
| ${{ env.release_file_bin }} | |
| build-and-test-macos: | |
| name: (MacOS ${{ matrix.arch }}) Build curl-impersonate and run the tests | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [macos-12, macos-14] | |
| include: | |
| - os: macos-12 | |
| arch: x86_64 | |
| host: x86_64-macos | |
| capture_interface: en0 | |
| make: gmake | |
| - os: macos-14 | |
| arch: arm64 | |
| host: arm64-macos | |
| capture_interface: en0 | |
| make: gmake | |
| steps: | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Install macOS dependencies | |
| run: | | |
| brew install pkg-config make cmake ninja autoconf automake libtool | |
| # Chrome version dependencies, install only not installed. | |
| which go || brew install go | |
| # Needed to compile 'minicurl' | |
| brew install curl | |
| # More dependencies for the tests | |
| brew install tcpdump nghttp2 | |
| - name: Check out the repo | |
| uses: actions/checkout@v2 | |
| - name: Install dependencies for the tests script | |
| run: | | |
| pip3 install -r tests/requirements.txt | |
| - name: Build zlib | |
| run: | | |
| curl -LO https://zlib.net/fossils/zlib-1.3.tar.gz | |
| tar xf zlib-1.3.tar.gz | |
| cd zlib-1.3 | |
| CHOST=${{ matrix.host }} ./configure --prefix=${{ runner.temp }}/zlib | |
| make | |
| make install | |
| # Make sure curl will link with libz.so.1 and not libz.so | |
| rm -f ${{ runner.temp }}/zlib/lib/libz.so | |
| - name: Run configure script | |
| run: | | |
| mkdir ${{ runner.temp }}/install | |
| ./configure --prefix=${{ runner.temp }}/install | |
| # Cache the build of BoringSSL, which is the longest part of the build | |
| # We must cache the .zip as well, otherwise the Makefile will | |
| # rebuild BoringSSL. This whole thing is a bit hacky, but necessary to | |
| # reduce the insanely long build times. | |
| - name: Cache BoringSSL source | |
| uses: actions/cache@v3 | |
| with: | |
| path: boringssl.zip | |
| key: ${{ runner.os }}-${{ matrix.arch }}-boring-source-${{ env.BORING_SSL_COMMIT }} | |
| - name: Cache BoringSSL build | |
| id: cache-boringssl | |
| uses: actions/cache@v3 | |
| with: | |
| path: boringssl/build | |
| key: ${{ runner.os }}-${{ matrix.arch }}-boring-build-${{ env.BORING_SSL_COMMIT }}-${{ hashFiles('chrome/patches/boringssl*.patch') }} | |
| # Trick the Makefile into skipping the BoringSSL build step | |
| # if it was found in the cache. See Makefile.in | |
| - name: Post BoringSSL cache restore | |
| if: ${{ steps.cache-boringssl.outputs.cache-hit != false }} | |
| run: | | |
| touch boringssl.zip | |
| touch boringssl/.patched | |
| find boringssl/build -type f | xargs touch | |
| - name: Build the Chrome version of curl-impersonate | |
| run: | | |
| ${{ matrix.make }} chrome-build | |
| ${{ matrix.make }} chrome-checkbuild | |
| ${{ matrix.make }} chrome-install | |
| - name: Prepare the tests | |
| run: | | |
| # Compile 'minicurl' which is used by the tests | |
| gcc -Wall -Werror -o ${{ runner.temp }}/install/bin/minicurl tests/minicurl.c `curl-config --libs` | |
| # For now we can only run the tests when not cross compiling, since the | |
| # tests run the curl-impersonate binary locally. | |
| - name: Run the tests | |
| run: | | |
| cd tests | |
| # sudo is needed for capturing packets | |
| python_bin=$(which python3) | |
| sudo $python_bin -m pytest . --log-cli-level DEBUG --install-dir ${{ runner.temp }}/install --capture-interface ${{ matrix.capture_interface }} | |
| # Upload pre-compiled binaries to GitHub releases page. | |
| - name: Create tar release files for libcurl-impersonate | |
| if: startsWith(github.ref, 'refs/tags/') | |
| run: | | |
| cd ${{ runner.temp }}/install/lib | |
| tar -c -z -f ${{ runner.temp }}/libcurl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz libcurl-impersonate* | |
| echo "release_file_lib=${{ runner.temp }}/libcurl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz" >> $GITHUB_ENV | |
| - name: Clean build | |
| if: startsWith(github.ref, 'refs/tags/') | |
| run: | | |
| ${{ matrix.make }} chrome-clean | |
| rm -Rf ${{ runner.temp }}/install | |
| mkdir ${{ runner.temp }}/install | |
| # Recompile curl-impersonate statically when doing a release. | |
| - name: Reconfigure statically | |
| if: startsWith(github.ref, 'refs/tags/') | |
| run: | | |
| ./configure --prefix=${{ runner.temp }}/install --enable-static | |
| - name: Rebuild statically | |
| if: startsWith(github.ref, 'refs/tags/') | |
| run: | | |
| ${{ matrix.make }} chrome-build | |
| ${{ matrix.make }} chrome-checkbuild | |
| ${{ matrix.make }} chrome-install-strip | |
| - name: Create tar release files for curl-impersonate | |
| if: startsWith(github.ref, 'refs/tags/') | |
| run: | | |
| cd ${{ runner.temp }}/install/bin | |
| tar -c -z -f ${{ runner.temp }}/curl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz curl-impersonate-chrome curl_* | |
| echo "release_file_bin=${{ runner.temp }}/curl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz" >> $GITHUB_ENV | |
| - name: Upload release files | |
| uses: softprops/action-gh-release@v1 | |
| if: startsWith(github.ref, 'refs/tags/') | |
| with: | |
| files: | | |
| ${{ env.release_file_lib }} | |
| ${{ env.release_file_bin }} |