Syncing recent changes. #277
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: [push, pull_request] | |
| env: | |
| GCS_BUCKET: autobuilds.grr-response.com | |
| GCS_BUCKET_OPENAPI: autobuilds-grr-openapi | |
| GCS_LATEST_PATH: _latest_server_deb | |
| DOCKER_REPOSITORY: ghcr.io/google/grr | |
| jobs: | |
| test-devenv: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: check_deps | |
| run: | | |
| sudo apt-get update && sudo apt-get install -y podman curl jq | |
| sudo usermod --add-subuids 500000-565535 --add-subgids 500000-565535 "${USER}" | |
| devenv/devenv.sh check_deps | |
| - name: start | |
| run: | | |
| devenv/devenv.sh start | |
| OK=false | |
| for attempt in $(seq 10); do | |
| curl -su admin:admin http://localhost:4280/api/clients | | |
| sed 1d | | |
| jq -r ".items[].value.client_id.value" | | |
| egrep -e "^C[.][0-9a-f]{16}\$" && { OK=true; break; } | |
| echo "attempt ${attempt}: devenv not ready" | |
| sleep 5 | |
| done | |
| [[ $OK = true ]] | |
| test-ubuntu: | |
| runs-on: ubuntu-22.04 | |
| env: | |
| CHROME_DEB: google-chrome-stable_current_amd64.deb | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install | |
| run: | | |
| free -hmw | |
| lscpu | |
| sudo apt install -y python3-dev python3-pip python3-venv python3-mysqldb | |
| if [[ -z "$(type google-chrome 2>/dev/null)" ]]; then | |
| wget "https://dl.google.com/linux/direct/${CHROME_DEB}" && sudo apt install -y "./${CHROME_DEB}"; | |
| fi | |
| python3 -m venv --system-site-packages "${HOME}/INSTALL" | |
| travis/install.sh | |
| - name: Test | |
| run: | | |
| source "${HOME}/INSTALL/bin/activate" | |
| pip install pytest-xdist==2.2.1 pytest==6.2.5 | |
| # We have 4 vCPUs available, but only use 3 here to avoid timeouts like | |
| # https://ci.appveyor.com/project/grr/grr-ia94e/builds/20483467/messages , | |
| # which happen when tests stall. | |
| pytest --verbose -n 3 grr/ --ignore grr/server/grr_response_server/gui/selenium_tests/ --ignore grr/client/grr_response_client/client_actions/windows/ | |
| # jsTree tests seem to fail on Chrome 71 headless due to https://github.com/GoogleChrome/puppeteer/issues/3463 | |
| if [ $(google-chrome --version | grep -Eo " [0-9]{1,3}") != "71" ]; then (cd grr/server/grr_response_server/gui/static/ && npm run gulp test); fi | |
| build-openapi: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install | |
| run: | | |
| sudo apt install -y python3-dev python3-pip python3-venv python3-mysqldb | |
| python3 -m venv --system-site-packages "${HOME}/INSTALL" | |
| travis/install.sh | |
| - name: Build | |
| run: | | |
| source "${HOME}/INSTALL/bin/activate" | |
| mkdir -p _openapi_artifacts/openapi_description | |
| mkdir -p _openapi_artifacts/documentation | |
| travis/build_api_documentation.sh "_openapi_artifacts/openapi_description/openapi_description.json" "_openapi_artifacts/documentation/openapi_documentation.html" | |
| ls -la _openapi_artifacts/* | |
| - name: Upload OpenAPI to GitHub artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: openapi | |
| path: _openapi_artifacts/ | |
| retention-days: 1 | |
| build-ubuntu: | |
| runs-on: ubuntu-22.04 | |
| env: | |
| GCS_TAG: ubuntu_64bit | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Set up | |
| run: | | |
| sudo apt install fakeroot debhelper libffi-dev libssl-dev | |
| pip install virtualenv | |
| virtualenv "${HOME}/INSTALL" | |
| - name: Build | |
| run: | | |
| travis/install_client_builder.sh | |
| travis/build_templates.sh | |
| ls -la gcs_upload_dir | |
| - name: Upload installers to GitHub artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ubuntu-installers | |
| path: gcs_upload_dir/ | |
| retention-days: 1 | |
| build-osx: | |
| runs-on: macos-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Set up | |
| run: | | |
| pip install --upgrade setuptools virtualenv | |
| virtualenv "${HOME}/INSTALL" | |
| - name: Build installers | |
| run: | | |
| travis/install_client_builder.sh | |
| travis/build_templates.sh | |
| ls -la gcs_upload_dir | |
| - name: Upload installers to GitHub artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: osx-installers | |
| path: gcs_upload_dir/ | |
| retention-days: 1 | |
| build-centos: | |
| runs-on: ubuntu-22.04 | |
| env: | |
| GCS_TAG: centos_64bit | |
| DOCKER_IMG: grrdocker/centos7-python39 | |
| DOCKER_CONTAINER: centos_64bit_container | |
| DOCKER_USER: grrbot | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Build installers | |
| run: | | |
| docker run -dit \ | |
| --volume "${PWD}:/mnt/grr" \ | |
| --workdir /mnt/grr \ | |
| --env DOCKER_USER="${DOCKER_USER}" \ | |
| --env TRAVIS_OS_NAME="linux" \ | |
| --name "${DOCKER_CONTAINER}" \ | |
| "${DOCKER_IMG}" | |
| # Using `bash -l` here and below to make sure devtools | |
| # (including the C++ 14 compatible compiler) are properly | |
| # registered in the environment variables. | |
| docker exec "${DOCKER_CONTAINER}" bash -l travis/set_up_test_user.sh | |
| docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" bash -l -c '/usr/local/bin/python3.9 -m venv "/home/${DOCKER_USER}/INSTALL"' | |
| docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" bash -l travis/install_client_builder.sh | |
| docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" bash -l travis/build_templates.sh | |
| docker exec "${DOCKER_CONTAINER}" rpm -vih gcs_upload_dir/*.rpm | |
| ls -la gcs_upload_dir | |
| - name: Upload installers to GitHub artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: centos-installers | |
| path: gcs_upload_dir/ | |
| retention-days: 1 | |
| build-windows: | |
| runs-on: windows-2022 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Build installers | |
| shell: bash | |
| run: | | |
| set -ex | |
| pip install virtualenv wheel | |
| python -u appveyor/windows_templates/build_windows_templates.py --grr_src=$GITHUB_WORKSPACE --output_dir=$GITHUB_WORKSPACE/output --test_repack_install | |
| mkdir -p gcs_upload_dir | |
| mv -v output*/* gcs_upload_dir | |
| ls -la gcs_upload_dir | |
| - name: Upload installers to GitHub artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: windows-installers | |
| path: gcs_upload_dir/ | |
| retention-days: 1 | |
| build-server-deb: | |
| runs-on: ubuntu-22.04 | |
| env: | |
| GCS_TAG: server_deb | |
| needs: | |
| - build-centos | |
| - build-ubuntu | |
| - build-osx | |
| - build-windows | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Download installers from GitHub artifacts | |
| id: download | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: ~/_artifacts | |
| - name: Set up | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y fakeroot debhelper libffi-dev libssl-dev python3-dev python3-pip python3-venv python3-mysqldb wget openjdk-8-jdk zip git devscripts libmysqlclient-dev dh-virtualenv dh-make libc6-i386 lib32z1 | |
| python3 -m venv --system-site-packages "${HOME}/INSTALL" | |
| "${HOME}/INSTALL/bin/python3" -m pip install --upgrade pip 'setuptools<58.3.1' wheel | |
| - name: Build | |
| run: | | |
| travis/install.sh | |
| mkdir -p grr/config/grr_response_templates/templates | |
| mv -v ~/_artifacts/windows-installers/GRR_*_amd64.msi.zip grr/config/grr_response_templates/templates | |
| mv -v ~/_artifacts/ubuntu-installers/grr_*_amd64.deb.zip grr/config/grr_response_templates/templates | |
| mv -v ~/_artifacts/centos-installers/grr_*_amd64.rpm.zip grr/config/grr_response_templates/templates | |
| mv -v ~/_artifacts/osx-installers/grr_*_amd64.xar.zip grr/config/grr_response_templates/templates | |
| travis/build_local_pyindex.sh | |
| travis/build_server_deb.sh | |
| ls -la gcs_upload_dir | |
| - name: Upload installers to GitHub artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: server-deb | |
| path: gcs_upload_dir/ | |
| retention-days: 1 | |
| test-ubuntu-e2e: | |
| continue-on-error: true # Debug follow up step. | |
| runs-on: ubuntu-22.04 | |
| env: | |
| GRR_ADMIN_PASS: 'e2e_tests' | |
| APPVEYOR_MYSQL_PASS: 'root' | |
| needs: | |
| - build-server-deb | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up MySQL | |
| run: | | |
| printf "\n[mysqld]\nmax_allowed_packet=42M\nlog_bin_trust_function_creators=1\n" | sudo tee -a /etc/mysql/my.cnf | |
| sudo /etc/init.d/mysql start | |
| - name: Download installers from GitHub artifacts | |
| id: download | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: server-deb | |
| path: _artifacts | |
| - name: Install | |
| run: | | |
| free -hmw | |
| lscpu | |
| sudo -EH ./appveyor/e2e_tests/install_mem_usage_cron.sh | |
| sudo -EH ./appveyor/e2e_tests/install_latest_server_deb.sh | |
| - name: Test | |
| run: | | |
| sudo -EH ./appveyor/e2e_tests/run_e2e_tests.sh | |
| sudo -EH ./appveyor/e2e_tests/test_repack.sh | |
| - name: Upload logs and configs to GitHub artifacts | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: e2e-test | |
| path: /var/log/grr | |
| retention-days: 1 | |
| build-push-docker: | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - build-server-deb | |
| # - test-ubuntu-e2e # TODO: Comment back in after debugging is finished. | |
| - test-ubuntu | |
| - build-openapi | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Download installers from GitHub artifacts | |
| id: download | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: server-deb | |
| path: _artifacts | |
| - name: Build Docker image | |
| run: | | |
| export BRANCH=$(echo $GITHUB_REF | cut -d'/' -f 3) | |
| ./appveyor/docker_build/build_docker_image.sh | |
| - if: ${{ github.event_name == 'push' }} | |
| name: Login to GitHub Container registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - if: ${{ github.event_name == 'push' }} | |
| name: Push to GitHub Container registry | |
| run: | | |
| docker push -a ${{ env.DOCKER_REPOSITORY }} | |
| upload: | |
| if: ${{ github.event_name == 'push' }} | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - build-push-docker | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Download installers from GitHub artifacts | |
| id: download | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: _artifacts | |
| - run: | | |
| ls -la _artifacts/*/ | |
| COMMIT_TIME=$(git show -s --date='format-local:%Y-%m-%dT%H:%M:%SZ' --format="%cd" $GITHUB_SHA) | |
| OUTPUT_DIR=gcs_upload_dir/${COMMIT_TIME}_${GITHUB_SHA}/ | |
| echo "OUTPUT_DIR=$OUTPUT_DIR" >> $GITHUB_ENV | |
| mkdir -p $OUTPUT_DIR/centos/ | |
| mv -v _artifacts/centos-installers/* $OUTPUT_DIR/centos | |
| mkdir -p $OUTPUT_DIR/ubuntu/ | |
| mv -v _artifacts/ubuntu-installers/* $OUTPUT_DIR/ubuntu | |
| mkdir -p $OUTPUT_DIR/osx/ | |
| mv -v _artifacts/osx-installers/* $OUTPUT_DIR/osx | |
| mkdir -p $OUTPUT_DIR/windows/ | |
| mv -v _artifacts/windows-installers/* $OUTPUT_DIR/windows | |
| mkdir -p $OUTPUT_DIR/server_deb/ | |
| mv -v _artifacts/server-deb/* $OUTPUT_DIR/server_deb | |
| - name: Authenticate | |
| uses: 'google-github-actions/auth@v1' | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_KEY }} | |
| export_environment_variables: true | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| - name: Upload installers to GCS | |
| uses: google-github-actions/[email protected] | |
| with: | |
| path: gcs_upload_dir/ | |
| destination: ${{ env.GCS_BUCKET }} | |
| # Omit `path` (e.g. /home/runner/deploy/) in final GCS path. | |
| parent: false | |
| - name: Replace ${{ env.GCS_LATEST_PATH }} folder in GCS | |
| run: | | |
| gsutil rm gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/** || true | |
| gsutil cp -r $OUTPUT_DIR/server_deb/* gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/ | |
| - name: Upload OpenAPI to GCS | |
| uses: google-github-actions/[email protected] | |
| with: | |
| path: _artifacts/openapi/ | |
| destination: ${{ env.GCS_BUCKET_OPENAPI }} | |
| # Omit `path` (e.g. /home/runner/deploy/) in final GCS path. | |
| parent: false | |