From 8581c7f52a134a899c8617af00a43c9d24ea1654 Mon Sep 17 00:00:00 2001 From: ADD-SP Date: Fri, 8 Jul 2022 00:22:21 +0800 Subject: [PATCH] chore(*) speed up Github Action --- .github/workflows/codecov.yml | 115 ++++++++++++++++++++++++++++---- .github/workflows/test.yml | 122 +++++++++++++++++++++++++++++----- 2 files changed, 207 insertions(+), 30 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 5bd41a4c..88b392d4 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -7,20 +7,112 @@ defaults: shell: bash jobs: - codecov: + build_deps: + name: Build dependencies runs-on: ubuntu-latest + + env: + BUILD_CACHE_NONCE: 0 + steps: + - name: Set environment variables + run: | + echo "CACHE_ROOT=$HOME/cache-root" >> $GITHUB_ENV + - uses: actions/checkout@v2 with: ref: ${{ github.ref }} + + - name: Checkout SpiderLabs/ModSecurity + uses: actions/checkout@v3 + with: + repository: SpiderLabs/ModSecurity + path: ModSecurity + + - name: Get ModSecurity commit hash + run: | + cd ModSecurity + echo "MODSECURITY_COMMIT_HASH=$(git rev-parse HEAD)" >> $GITHUB_ENV + + - name: Lookup build cache + uses: actions/cache@v3 + id: cache-deps + with: + path: ${{ env.CACHE_ROOT }} + key: ${{ env.MODSECURITY_COMMIT_HASH }}-${{ env.BUILD_CACHE_NONCE }} + + - name: Install dependencies + if: steps.cache-deps.outputs.cache-hit != 'true' + run: | + sudo apt-get --yes update + sudo apt-get install --yes libsodium23 libsodium-dev build-essential zlib1g-dev libpcre3 libpcre3-dev libssl-dev libxslt1-dev libxml2-dev libgeoip-dev libgd-dev libperl-dev libcurl4-openssl-dev + + - name: Download and install ModSecurity + if: steps.cache-deps.outputs.cache-hit != 'true' + run: | + cd ModSecurity + ./build.sh + git submodule init + git submodule update + ./configure --prefix=/usr/local/modsecurity + make -j$(nproc) + sudo make install + + - name: Package dependencies + if: steps.cache-deps.outputs.cache-hit != 'true' + run: | + mkdir -p ${{ env.CACHE_ROOT }} + mkdir -p /tmp/cache/usr/local + sudo cp -r /usr/local/modsecurity /tmp/cache/usr/local + tar -C /tmp/cache/ -cvzf ${{ env.CACHE_ROOT }}/cache.tar.gz . + + codecov: + runs-on: ubuntu-latest + needs: build_deps + + env: + BUILD_CACHE_NONCE: 0 + + steps: + - name: Set environment variables + run: | + echo "CACHE_ROOT=$HOME/cache-root" >> $GITHUB_ENV + - uses: actions/setup-python@v2 with: python-version: '3.x' architecture: 'x64' + + - uses: actions/checkout@v2 + with: + ref: ${{ github.ref }} + + - name: Checkout SpiderLabs/ModSecurity + uses: actions/checkout@v3 + with: + repository: SpiderLabs/ModSecurity + path: ModSecurity + + - name: Get ModSecurity commit hash + run: | + cd ModSecurity + echo "MODSECURITY_COMMIT_HASH=$(git rev-parse HEAD)" >> $GITHUB_ENV + + - name: Lookup build cache + uses: actions/cache@v3 + id: cache-deps + with: + path: ${{ env.CACHE_ROOT }} + key: ${{ env.MODSECURITY_COMMIT_HASH }}-${{ env.BUILD_CACHE_NONCE }} + + - name: Extract the deps + run: sudo tar -C / -xzvf ${{ env.CACHE_ROOT }}/cache.tar.gz + - name: Install dependencies run: | sudo apt-get --yes update sudo apt-get install --yes libsodium23 libsodium-dev build-essential zlib1g-dev libpcre3 libpcre3-dev libssl-dev libxslt1-dev libxml2-dev libgeoip-dev libgd-dev libperl-dev libcurl4-openssl-dev gcovr + - name: Download nginx run: | sudo chmod 777 -R /usr/local/src @@ -29,30 +121,21 @@ jobs: lastversion download nginx:stable mkdir nginx tar zxf nginx-*.tar.gz --directory nginx --strip-components=1 - - name: Download and install modsecurity - run: | - sudo chmod 777 -R /usr/local/src - cd /usr/local/src - git clone https://github.com/SpiderLabs/ModSecurity.git - sudo chmod 777 -R /usr/local/src - cd ModSecurity - ./build.sh - git submodule init - git submodule update - ./configure --prefix=/usr/local/modsecurity - make -j$(nproc) - sudo make install + - name: Download DaveGamble/cJSON.git run: | git clone -b v1.7.15 https://github.com/DaveGamble/cJSON.git lib/cjson + - name: Download troydhanson/uthash run: | git clone -b v2.3.0 https://github.com/troydhanson/uthash.git lib/uthash + - name: Configure nginx run: | cd /usr/local/src/nginx export LIB_MODSECURITY=/usr/local/modsecurity ./configure --add-module=${{ github.workspace }} --with-http_realip_module --with-cc-opt='-fprofile-arcs -ftest-coverage -O0 -g3' --with-ld-opt='-lgcov' + - name: Install nginx run: | cd /usr/local/src/nginx @@ -61,9 +144,11 @@ jobs: sudo useradd nginx -s /sbin/nologin -M sudo chmod 777 -R /usr/local/nginx sudo ln -s /usr/local/nginx/sbin/nginx /usr/local/bin/nginx + - name: Install Test::Nginx run: | sudo cpan Test::Nginx + - name: Test run: | sudo chmod 777 -R /tmp @@ -71,6 +156,7 @@ jobs: export MODULE_TEST_PATH=/tmp/module_test sh ./init.sh exec sudo sh start.sh t/*.t + - name: Generate coverage report run: | sudo chmod -R 777 /usr/local/src @@ -83,6 +169,7 @@ jobs: cp ${{ github.workspace }}/src/* /usr/local/src/empty/ sudo chmod -R 777 ${{ github.workspace }} sudo chmod -R 777 /usr/local/src + - name: Run codacy-coverage-reporter uses: codacy/codacy-coverage-reporter-action@v1 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7939df80..14f0c959 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,26 +30,121 @@ jobs: -H "Accept: application/vnd.github.v3+json" \ https://api.github.com/repos/ADD-SP/ngx_waf/actions/workflows/${{ github.workflow }}.yml/dispatches \ -d '{"ref":"refs/heads/current"}' + + build_deps: + name: Build dependencies + runs-on: ubuntu-latest + + env: + BUILD_CACHE_NONCE: 0 + + steps: + - name: Set environment variables + run: | + echo "CACHE_ROOT=$HOME/cache-root" >> $GITHUB_ENV + + - uses: actions/checkout@v2 + with: + ref: ${{ github.ref }} + + - name: Checkout SpiderLabs/ModSecurity + uses: actions/checkout@v3 + with: + repository: SpiderLabs/ModSecurity + path: ModSecurity + + - name: Get ModSecurity commit hash + run: | + cd ModSecurity + echo "MODSECURITY_COMMIT_HASH=$(git rev-parse HEAD)" >> $GITHUB_ENV + + - name: Lookup build cache + uses: actions/cache@v3 + id: cache-deps + with: + path: ${{ env.CACHE_ROOT }} + key: ${{ env.MODSECURITY_COMMIT_HASH }}-${{ env.BUILD_CACHE_NONCE }} + + - name: Install dependencies + if: steps.cache-deps.outputs.cache-hit != 'true' + run: | + sudo apt-get --yes update + sudo apt-get install --yes libsodium23 libsodium-dev build-essential zlib1g-dev libpcre3 libpcre3-dev libssl-dev libxslt1-dev libxml2-dev libgeoip-dev libgd-dev libperl-dev libcurl4-openssl-dev + + - name: Download and install ModSecurity + if: steps.cache-deps.outputs.cache-hit != 'true' + run: | + cd ModSecurity + ./build.sh + git submodule init + git submodule update + ./configure --prefix=/usr/local/modsecurity + make -j$(nproc) + sudo make install + + - name: Package dependencies + if: steps.cache-deps.outputs.cache-hit != 'true' + run: | + mkdir -p ${{ env.CACHE_ROOT }} + mkdir -p /tmp/cache/usr/local + sudo cp -r /usr/local/modsecurity /tmp/cache/usr/local + tar -C /tmp/cache/ -cvzf ${{ env.CACHE_ROOT }}/cache.tar.gz . + + native: runs-on: ubuntu-latest if: ${{ github.event_name != 'schedule' }} + needs: build_deps + + env: + BUILD_CACHE_NONCE: 0 + strategy: matrix: nginx-version: ['stable', 'mainline'] module-type: ['static', 'dynamic'] pcre-version: ['pcre', 'pcre2'] + steps: - - uses: actions/checkout@v2 - with: - ref: ${{ github.ref }} + - name: Set environment variables + run: | + echo "CACHE_ROOT=$HOME/cache-root" >> $GITHUB_ENV + - uses: actions/setup-python@v2 with: python-version: '3.x' architecture: 'x64' + + - uses: actions/checkout@v2 + with: + ref: ${{ github.ref }} + + - name: Checkout SpiderLabs/ModSecurity + uses: actions/checkout@v3 + with: + repository: SpiderLabs/ModSecurity + path: ModSecurity + + - name: Get ModSecurity commit hash + run: | + cd ModSecurity + echo "MODSECURITY_COMMIT_HASH=$(git rev-parse HEAD)" >> $GITHUB_ENV + + - name: Lookup build cache + uses: actions/cache@v3 + id: cache-deps + with: + path: ${{ env.CACHE_ROOT }} + key: ${{ env.MODSECURITY_COMMIT_HASH }}-${{ env.BUILD_CACHE_NONCE }} + + - name: Extract the deps + run: sudo tar -C / -xzvf ${{ env.CACHE_ROOT }}/cache.tar.gz + - name: Install dependencies run: | sudo apt-get --yes update sudo apt-get install --yes libsodium23 libsodium-dev build-essential zlib1g-dev libpcre3 libpcre3-dev libssl-dev libxslt1-dev libxml2-dev libgeoip-dev libgd-dev libperl-dev libcurl4-openssl-dev + - name: Download nginx-${{ matrix.nginx-version }} run: | sudo chmod 777 -R /usr/local/src @@ -58,6 +153,7 @@ jobs: lastversion download nginx:${{ matrix.nginx-version }} mkdir nginx tar zxf nginx-*.tar.gz --directory nginx --strip-components=1 + - name: Download PCRE if: ${{ matrix.pcre-version == 'pcre' || (matrix.pcre-version == 'pcre2' && matrix.nginx-version == 'stable') }} run: | @@ -67,6 +163,7 @@ jobs: mkdir pcre tar zxf pcre.tar.gz --directory pcre --strip-components=1 sudo chmod 777 -R /usr/local/src + - name: Download PCRE2 if: ${{ matrix.pcre-version == 'pcre2' && matrix.nginx-version == 'mainline' }} run: | @@ -76,25 +173,15 @@ jobs: mkdir pcre tar zxf pcre.tar.gz --directory pcre --strip-components=1 sudo chmod 777 -R /usr/local/src - - name: Download and install modsecurity - run: | - sudo chmod 777 -R /usr/local/src - cd /usr/local/src - git clone https://github.com/SpiderLabs/ModSecurity.git - sudo chmod 777 -R /usr/local/src - cd ModSecurity - ./build.sh - git submodule init - git submodule update - ./configure --prefix=/usr/local/modsecurity - make -j$(nproc) - sudo make install + - name: Download cjson run: | git clone -b v1.7.15 https://github.com/DaveGamble/cJSON.git lib/cjson + - name: Download uthash run: | git clone -b v2.3.0 https://github.com/troydhanson/uthash.git lib/uthash + - name: Configure ${{ matrix.module-type }} run: | cd /usr/local/src/nginx @@ -105,6 +192,7 @@ jobs: fi export LIB_MODSECURITY=/usr/local/modsecurity ./configure ${opt}=${{ github.workspace }} --with-pcre=/usr/local/src/pcre --with-http_realip_module --with-cc-opt='-Wno-unused-but-set-variable -Wno-unused-function -fstack-protector-strong' + - name: Install nginx-${{ matrix.nginx-version }} run: | cd /usr/local/src/nginx @@ -113,9 +201,11 @@ jobs: sudo useradd nginx -s /sbin/nologin -M sudo chmod 777 -R /usr/local/nginx sudo ln -s /usr/local/nginx/sbin/nginx /usr/local/bin/nginx + - name: Install Test::Nginx run: | sudo cpan Test::Nginx + - name: Test run: | sudo chmod 777 -R /tmp