Skip to content

Commit

Permalink
Update Github actions to build new GRR Docker image
Browse files Browse the repository at this point in the history
The new docker image and docker-compose setup replaces the GRR debian
package. The current e2e testing is temporarily removed as it depends on
the debian package and will be re-introduced after the docker-compose
setup is available.
  • Loading branch information
s-westphal committed Jan 15, 2024
1 parent 727545a commit a11a6c9
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 142 deletions.
126 changes: 32 additions & 94 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -183,122 +183,60 @@ jobs:
path: gcs_upload_dir/
retention-days: 1

build-server-deb:
runs-on: ubuntu-22.04
build-push-docker-image:
env:
GCS_TAG: server_deb
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
runs-on: ubuntu-22.04
needs:
- build-centos
- build-ubuntu
- build-osx
- build-windows
steps:
- uses: actions/checkout@v3
- name: Checkout repository
uses: actions/checkout@v3
- name: Download installers from GitHub artifacts
id: download
uses: actions/download-artifact@v3
uses: actions/download-artifact@v4
with:
pattern: '*installer*'
path: ~/_artifacts
- name: Set up
run: |
sudo apt-get update
sudo apt-get install -y fakeroot debhelper libffi-dev libssl-dev python3-dev python3-pip python3-venv python3-mysqldb wget openjdk-8-jdk zip git devscripts libmysqlclient-dev dh-virtualenv dh-make libc6-i386 lib32z1
python3 -m venv --system-site-packages "${HOME}/INSTALL"
"${HOME}/INSTALL/bin/python3" -m pip install --upgrade pip 'setuptools<58.3.1' wheel
- name: Build
run: |
travis/install.sh
mkdir -p grr/config/grr_response_templates/templates
mv -v ~/_artifacts/windows-installers/GRR_*_amd64.msi.zip grr/config/grr_response_templates/templates
mv -v ~/_artifacts/ubuntu-installers/grr_*_amd64.deb.zip grr/config/grr_response_templates/templates
mv -v ~/_artifacts/centos-installers/grr_*_amd64.rpm.zip grr/config/grr_response_templates/templates
mv -v ~/_artifacts/osx-installers/grr_*_amd64.xar.zip grr/config/grr_response_templates/templates
travis/build_local_pyindex.sh
travis/build_server_deb.sh
ls -la gcs_upload_dir
- name: Upload installers to GitHub artifacts
uses: actions/upload-artifact@v3
- name: Login to GitHub Container registry
if: ${{ github.event_name == 'push' }}
uses: docker/login-action@v3
with:
name: server-deb
path: gcs_upload_dir/
retention-days: 1
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

test-ubuntu-e2e:
continue-on-error: true # Debug follow up step.
runs-on: ubuntu-22.04
env:
GRR_ADMIN_PASS: 'e2e_tests'
APPVEYOR_MYSQL_PASS: 'root'
needs:
- build-server-deb
steps:
- uses: actions/checkout@v3
- name: Set up MySQL
run: |
printf "\n[mysqld]\nmax_allowed_packet=42M\nlog_bin_trust_function_creators=1\n" | sudo tee -a /etc/mysql/my.cnf
sudo /etc/init.d/mysql start
- name: Download installers from GitHub artifacts
id: download
uses: actions/download-artifact@v3
with:
name: server-deb
path: _artifacts
- name: Install
run: |
free -hmw
lscpu
sudo -EH ./appveyor/e2e_tests/install_mem_usage_cron.sh
sudo -EH ./appveyor/e2e_tests/install_latest_server_deb.sh
- name: Test
run: |
sudo -EH ./appveyor/e2e_tests/run_e2e_tests.sh
sudo -EH ./appveyor/e2e_tests/test_repack.sh
- name: Upload logs and configs to GitHub artifacts
uses: actions/upload-artifact@v3
if: always()
- name: Extract metadata (tags, labels) for Docker
if: ${{ github.event_name == 'push' }}
id: meta
uses: docker/metadata-action@v5
with:
name: e2e-test
path: /var/log/grr
retention-days: 1
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}

build-push-docker:
runs-on: ubuntu-22.04
needs:
- build-server-deb
# - test-ubuntu-e2e # TODO: Comment back in after debugging is finished.
- test-ubuntu
- build-openapi
steps:
- uses: actions/checkout@v3
- name: Download installers from GitHub artifacts
id: download
uses: actions/download-artifact@v3
- name: Build and push Docker image
if: ${{ github.event_name == 'push' }}
uses: docker/build-push-action@v5
with:
name: server-deb
path: _artifacts
- name: Build Docker image
run: |
export BRANCH=$(echo $GITHUB_REF | cut -d'/' -f 3)
./appveyor/docker_build/build_docker_image.sh
- if: ${{ github.event_name == 'push' }}
name: Login to GitHub Container registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- if: ${{ github.event_name == 'push' }}
name: Push to GitHub Container registry
run: |
docker push -a ${{ env.DOCKER_REPOSITORY }}
upload:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

upload-artifacts:
if: ${{ github.event_name == 'push' }}
permissions:
contents: 'read'
id-token: 'write'
runs-on: ubuntu-22.04
needs:
- build-push-docker
- build-centos
- build-ubuntu
- build-osx
- build-windows
steps:
- uses: actions/checkout@v3
- name: Download installers from GitHub artifacts
Expand Down
89 changes: 41 additions & 48 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,78 +2,71 @@
#
# See https://hub.docker.com/r/grrdocker/grr/
#
# We have configured Travis to trigger an image build every time a new server
# deb is been uploaded to GCS.
# We have configured Github Actions to trigger an image build every time a new
# a PUSH happens in the GRR github repository.
#
# Run the container with:
# Example: Run the grr admin_ui component:
#
# docker run \
# -e EXTERNAL_HOSTNAME="localhost" \
# -e ADMIN_PASSWORD="demo" \
# -p 0.0.0.0:8000:8000 \
# -p 0.0.0.0:8080:8080 \
# grrdocker/grr
# docker run -it \
# -v $(pwd)/docker_config_files:/configs
# ghcr.io/google/grr:grr-docker-compose
# "-component" "admin_ui"
# "-config" "/configs/server/grr.server.yaml"

FROM mariadb:jammy
FROM ubuntu:22.04 AS builder

LABEL maintainer="[email protected]"

ARG GCS_BUCKET
ARG GRR_COMMIT

ENV GRR_VENV /usr/share/grr-server
ENV DEBIAN_FRONTEND noninteractive
# Buffering output (sometimes indefinitely if a thread is stuck in
# a loop) makes for a non-optimal user experience when containers
# are run in the foreground, so we disable that.
ENV PYTHONUNBUFFERED=0

SHELL ["/bin/bash", "-c"]
ENV PYTHONUNBUFFERED 0

RUN apt-get update && \
apt-get install -y \
debhelper \
default-jre \
dpkg-dev \
git \
libffi-dev \
libssl-dev \
python-is-python3 \
python3-dev \
python3-pip \
python3-venv \
python3-mysqldb \
rpm \
wget \
zip \
python3-mysqldb
build-essential \
linux-headers-generic \
dh-make \
rpm

# Only available when building as part of Github Actions.
COPY _artifacts* /client_templates

ENV VIRTUAL_ENV /usr/share/grr-server
ENV GRR_SOURCE /usr/src/grr

RUN python -m venv --system-site-packages $VIRTUAL_ENV
ENV PATH="$VIRTUAL_ENV/bin:$PATH"

# Limiting setuptools version due to
# https://github.com/pypa/setuptools/issues/3278
# (it behaves incorrectly on Ubuntu 22 on virtualenvs with access to
# globally installed packages).
RUN pip3 install --upgrade 'setuptools<58.3.1' && \
python3 -m venv --system-site-packages $GRR_VENV
RUN pip install wheel nodeenv grpcio-tools==1.60

RUN $GRR_VENV/bin/pip install --upgrade --no-cache-dir pip wheel six setuptools nodeenv && \
$GRR_VENV/bin/nodeenv -p --prebuilt --node=16.13.0 && \
echo '{ "allow_root": true }' > /root/.bowerrc
RUN nodeenv -p --prebuilt --node=16.13.0

# Copy the GRR code over.
ADD . /usr/src/grr
RUN mkdir ${GRR_SOURCE}
ADD . ${GRR_SOURCE}

RUN cd /usr/src/grr && bash -x /usr/src/grr/docker/install_grr_from_gcs.sh
WORKDIR ${GRR_SOURCE}

ENTRYPOINT ["/usr/src/grr/docker/docker-entrypoint.sh"]
RUN cd grr/server/grr_response_server/gui/static && \
npm ci && npm run gulp compile

# Port for the admin UI GUI
EXPOSE 8000
RUN python grr/proto/makefile.py && \
python grr/core/grr_response_core/artifacts/makefile.py

# Port for clients to talk to
EXPOSE 8080
RUN pip install -e grr/proto \
pip install -e grr/core \
pip install -e grr/client \
pip install -e grr/server \
pip install -e grr/client_builder \
pip install -e api_client/python

# Directories used by GRR at runtime, which can be mounted from the host's
# filesystem. Note that volumes can be mounted even if they do not appear in
# this list.
VOLUME ["/usr/share/grr-server/install_data/etc"]
WORKDIR /

CMD ["grr"]
ENTRYPOINT [ "grr_server" ]

0 comments on commit a11a6c9

Please sign in to comment.