build-and-test-make.yml

name: Build and test

on:
  push:
    branches:
      - main
    tags:
      - "v*.*.*"
  pull_request:
    branches:
      - main

permissions:
  contents: write

env:
  BORING_SSL_COMMIT: d24a38200fef19150eef00cad35b138936c08767

jobs:
  build-and-test-linux:
    name: (Linux ${{ matrix.arch }}) Build curl-impersonate and run the tests
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        arch: [x86_64]
        include:
          - arch: x86_64
            go_arch: amd64
            docker_arch: linux/amd64
            host: x86_64-linux-gnu
            capture_interface: eth0
            image: quay.io/pypa/manylinux2014_x86_64:latest
          - arch: aarch64
            go_arch: arm64
            docker_arch: linux/arm/v8
            host: aarch64-linux-gnu
            capture_interface: eth0
            image: quay.io/pypa/manylinux2014_aarch64:latest
          - arch: arm
            go_arch: armv6l
            docker_arch: linux/arm/v7
            host: arm-linux-gnu
            capture_interface: eth0
            image: ghcr.io/bjia56/armv7l-wheel-builder:main
    steps:
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2

      - name: Install dependencies
        run: |
          python_bin_dir=$(docker run ${{ matrix.image }} python3.10 -c 'import sys; import os; print(os.path.dirname(os.path.realpath(sys.executable)))')
          echo "runner_uid=$(id -u)" >> $GITHUB_ENV
          echo "runner_gid=$(id -g)" >> $GITHUB_ENV
          echo "runner_home=$HOME" >> $GITHUB_ENV
          docker build -t curl-impersonate-builder -f - . <<EOF
            FROM ${{ matrix.image }}
            ENV PATH=/usr/local/go/bin:/opt/rh/devtoolset-10/root/usr/bin:$python_bin_dir:\$PATH
            ENV HOME=$HOME
            ENV CC=/opt/rh/devtoolset-10/root/usr/bin/gcc
            ENV CXX=/opt/rh/devtoolset-10/root/usr/bin/g++
            ENV LD=/usr/bin/ld
            RUN yum install -y libcurl-devel
            RUN python3.10 -m pip install ninja cmake gyp-next --extra-index-url https://bjia56.github.io/armv7l-wheels/
            RUN rm -rf /usr/local/go && \
                curl -o /tmp/go.tar.gz -L https://go.dev/dl/go1.19.10.linux-${{ matrix.go_arch }}.tar.gz && \
                tar -C /usr/local -xzf /tmp/go.tar.gz
            RUN rm -f /usr/local/bin/python3 && \
                rm -f /usr/local/bin/python && \
                ln -s /usr/local/bin/python3.10 /usr/local/bin/python3 && \
                ln -s /usr/local/bin/python3.10 /usr/local/bin/python
          EOF

      - name: Check out the repo
        uses: actions/checkout@v2

      - name: Run configure script
        uses: addnab/docker-run-action@v3
        with:
          image: curl-impersonate-builder
          options: >
            -v ${{ env.runner_home }}:${{ env.runner_home }}
            --workdir ${{ github.workspace }}
            --user ${{ env.runner_uid }}:${{ env.runner_gid }}
          run: |
            set -e
            mkdir ${{ runner.temp }}/install
            ./configure --prefix=${{ runner.temp }}/install \
                        --with-ca-path=/etc/ssl/certs \
                        --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt

      # Cache the build of BoringSSL, which is the longest part of the build
      # We must cache the .zip as well, otherwise the Makefile will
      # rebuild BoringSSL. This whole thing is a bit hacky, but necessary to
      # reduce the insanely long build times.
      - name: Cache BoringSSL source
        uses: actions/cache@v3
        with:
          path: boringssl.zip
          key: ${{ runner.os }}-${{ matrix.arch }}-boring-source-${{ env.BORING_SSL_COMMIT }}

      - name: Cache BoringSSL build
        id: cache-boringssl
        uses: actions/cache@v3
        with:
          path: boringssl/build
          key: ${{ runner.os }}-${{ matrix.arch }}-boring-build-${{ env.BORING_SSL_COMMIT }}-${{ hashFiles('chrome/patches/boringssl*.patch') }}

      # Trick the Makefile into skipping the BoringSSL build step
      # if it was found in the cache. See Makefile.in
      - name: Post BoringSSL cache restore
        if: ${{ steps.cache-boringssl.outputs.cache-hit != false }}
        run: |
          touch boringssl.zip
          touch boringssl/.patched
          find boringssl/build -type f | xargs touch

      - name: Build the Chrome version of curl-impersonate
        uses: addnab/docker-run-action@v3
        with:
          image: curl-impersonate-builder
          options: >
            -v ${{ env.runner_home }}:${{ env.runner_home }}
            --workdir ${{ github.workspace }}
            --user ${{ env.runner_uid }}:${{ env.runner_gid }}
          run: |
            set -e
            make chrome-build
            make chrome-checkbuild
            make chrome-install

      - name: Install test dependencies
        if: matrix.arch == 'x86_64'
        run: |
          sudo apt-get update
          sudo apt-get install build-essential pkg-config cmake ninja-build curl autoconf automake libtool
          # Chrome version dependencies
          sudo apt-get install golang-go
          # Needed to compile 'minicurl'
          sudo apt-get install libcurl4-openssl-dev
          # More dependencies for the tests
          sudo apt-get install tcpdump nghttp2-server

      - name: Prepare the tests
        if: matrix.arch == 'x86_64'
        run: |
          # Compile 'minicurl' which is used by the tests
          gcc -Wall -Werror -o ${{ runner.temp }}/install/bin/minicurl tests/minicurl.c `curl-config --libs`

      - uses: actions/setup-python@v4
        if: matrix.arch == 'x86_64'
        with:
          python-version: '3.10'

      - name: Install dependencies for the tests script
        if: matrix.arch == 'x86_64'
        run: |
          pip3 install -r tests/requirements.txt

      # For now we can only run the tests when native
      # tests run the curl-impersonate binary locally.
      - name: Run the tests
        if: matrix.arch == 'x86_64'
        run: |
          cd tests
          # sudo is needed for capturing packets
          python_bin=$(which python3)
          sudo $python_bin -m pytest . --log-cli-level DEBUG --install-dir ${{ runner.temp }}/install --capture-interface ${{ matrix.capture_interface }}

       # Upload pre-compiled binaries to GitHub releases page.
      - name: Create tar release files for libcurl-impersonate
        if: startsWith(github.ref, 'refs/tags/')
        run: |
            cd ${{ runner.temp }}/install/lib
            tar -c -z -f ${{ runner.temp }}/libcurl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz libcurl-impersonate*
            echo "release_file_lib=${{ runner.temp }}/libcurl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz" >> $GITHUB_ENV

      - name: Clean build
        if: startsWith(github.ref, 'refs/tags/')
        uses: addnab/docker-run-action@v3
        with:
          image: curl-impersonate-builder
          options: >
            -v ${{ env.runner_home }}:${{ env.runner_home }}
            --workdir ${{ github.workspace }}
            --user ${{ env.runner_uid }}:${{ env.runner_gid }}
          run: |
            set -e
            make chrome-clean
            rm -Rf ${{ runner.temp }}/install
            mkdir ${{ runner.temp }}/install

       # Recompile curl-impersonate statically when doing a release.
      - name: Reconfigure statically
        if: startsWith(github.ref, 'refs/tags/') && matrix.arch == 'x86_64'
        uses: addnab/docker-run-action@v3
        with:
          image: curl-impersonate-builder
          options: >
            -v ${{ env.runner_home }}:${{ env.runner_home }}
            --workdir ${{ github.workspace }}
            --user ${{ env.runner_uid }}:${{ env.runner_gid }}
          run: |
            set -e
            ./configure --prefix=${{ runner.temp }}/install --enable-static

      - name: Rebuild statically
        if: startsWith(github.ref, 'refs/tags/')
        uses: addnab/docker-run-action@v3
        with:
          image: curl-impersonate-builder
          options: >
            -v ${{ env.runner_home }}:${{ env.runner_home }}
            --workdir ${{ github.workspace }}
            --user ${{ env.runner_uid }}:${{ env.runner_gid }}
          run: |
            set -e
            make chrome-build
            make chrome-checkbuild
            make chrome-install-strip

      - name: Create tar release files for curl-impersonate
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          cd ${{ runner.temp }}/install/bin
          tar -c -z -f ${{ runner.temp }}/curl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz curl-impersonate-chrome curl_*
          echo "release_file_bin=${{ runner.temp }}/curl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz" >> $GITHUB_ENV

      - name: Upload release files
        uses: softprops/action-gh-release@v1
        if: startsWith(github.ref, 'refs/tags/')
        with:
          files: |
              ${{ env.release_file_lib }}
              ${{ env.release_file_bin }}

  build-and-test-macos:
    name: (MacOS ${{ matrix.arch }}) Build curl-impersonate and run the tests
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [macos-12, macos-14]
        include:
          - os: macos-12
            arch: x86_64
            host: x86_64-macos
            capture_interface: en0
            make: gmake
          - os: macos-14
            arch: arm64
            host: arm64-macos
            capture_interface: en0
            make: gmake
    steps:
      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Install macOS dependencies
        run: |
          brew install pkg-config make cmake ninja autoconf automake libtool
          # Needed to compile 'minicurl'
          brew install curl
          # More dependencies for the tests
          brew install tcpdump nghttp2

      - name: Install go as macOS dependencies
        if: matrix.arch != 'x86_64'
        run: |
          # Chrome version dependencies
          brew install go

      - name: Check out the repo
        uses: actions/checkout@v2

      - name: Install dependencies for the tests script
        run: |
          pip3 install -r tests/requirements.txt

      - name: Build zlib
        run: |
          curl -LO https://zlib.net/fossils/zlib-1.3.tar.gz
          tar xf zlib-1.3.tar.gz
          cd zlib-1.3
          CHOST=${{ matrix.host }} ./configure --prefix=${{ runner.temp }}/zlib
          make
          make install
          # Make sure curl will link with libz.so.1 and not libz.so
          rm -f ${{ runner.temp }}/zlib/lib/libz.so

      - name: Run configure script
        run: |
          mkdir ${{ runner.temp }}/install
          ./configure --prefix=${{ runner.temp }}/install

      # Cache the build of BoringSSL, which is the longest part of the build
      # We must cache the .zip as well, otherwise the Makefile will
      # rebuild BoringSSL. This whole thing is a bit hacky, but necessary to
      # reduce the insanely long build times.
      - name: Cache BoringSSL source
        uses: actions/cache@v3
        with:
          path: boringssl.zip
          key: ${{ runner.os }}-${{ matrix.arch }}-boring-source-${{ env.BORING_SSL_COMMIT }}

      - name: Cache BoringSSL build
        id: cache-boringssl
        uses: actions/cache@v3
        with:
          path: boringssl/build
          key: ${{ runner.os }}-${{ matrix.arch }}-boring-build-${{ env.BORING_SSL_COMMIT }}-${{ hashFiles('chrome/patches/boringssl*.patch') }}

      # Trick the Makefile into skipping the BoringSSL build step
      # if it was found in the cache. See Makefile.in
      - name: Post BoringSSL cache restore
        if: ${{ steps.cache-boringssl.outputs.cache-hit != false }}
        run: |
          touch boringssl.zip
          touch boringssl/.patched
          find boringssl/build -type f | xargs touch

      - name: Build the Chrome version of curl-impersonate
        run: |
          ${{ matrix.make }} chrome-build
          ${{ matrix.make }} chrome-checkbuild
          ${{ matrix.make }} chrome-install

      - name: Prepare the tests
        run: |
          # Compile 'minicurl' which is used by the tests
          gcc -Wall -Werror -o ${{ runner.temp }}/install/bin/minicurl tests/minicurl.c `curl-config --libs`

      # For now we can only run the tests when not cross compiling, since the
      # tests run the curl-impersonate binary locally.
      - name: Run the tests
        run: |
          cd tests
          # sudo is needed for capturing packets
          python_bin=$(which python3)
          sudo $python_bin -m pytest . --log-cli-level DEBUG --install-dir ${{ runner.temp }}/install --capture-interface ${{ matrix.capture_interface }}

       # Upload pre-compiled binaries to GitHub releases page.
      - name: Create tar release files for libcurl-impersonate
        if: startsWith(github.ref, 'refs/tags/')
        run: |
            cd ${{ runner.temp }}/install/lib
            tar -c -z -f ${{ runner.temp }}/libcurl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz libcurl-impersonate*
            echo "release_file_lib=${{ runner.temp }}/libcurl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz" >> $GITHUB_ENV

      - name: Clean build
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          ${{ matrix.make }} chrome-clean
          rm -Rf ${{ runner.temp }}/install
          mkdir ${{ runner.temp }}/install

       # Recompile curl-impersonate statically when doing a release.
      - name: Reconfigure statically
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          ./configure --prefix=${{ runner.temp }}/install --enable-static

      - name: Rebuild statically
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          ${{ matrix.make }} chrome-build
          ${{ matrix.make }} chrome-checkbuild
          ${{ matrix.make }} chrome-install-strip

      - name: Create tar release files for curl-impersonate
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          cd ${{ runner.temp }}/install/bin
          tar -c -z -f ${{ runner.temp }}/curl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz curl-impersonate-chrome curl_*
          echo "release_file_bin=${{ runner.temp }}/curl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz" >> $GITHUB_ENV

      - name: Upload release files
        uses: softprops/action-gh-release@v1
        if: startsWith(github.ref, 'refs/tags/')
        with:
          files: |
              ${{ env.release_file_lib }}
              ${{ env.release_file_bin }}