From 26587d27866f4587c94e102e10288e142459bd5f Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Wed, 28 Apr 2021 17:46:47 +0100 Subject: [PATCH] initial push of the kurtosis_upgrade --- .ci/run_e2e_tests.sh | 52 ------- .dockerignore | 5 +- .github/workflows/e2e.yml | 25 ++++ .github/workflows/publish_image.sh | 45 ++++++ .github/workflows/run_e2e_tests.sh | 56 +++++++ .kurtosis/build-and-run-core.sh | 140 ++++++++++++++++++ .kurtosis/kurtosis.sh | 226 +++++++++++++++++++++++++++++ .travis.yml | 8 - Dockerfile | 28 ++-- scripts/build.sh | 26 ++-- scripts/build_avalanche.sh | 19 ++- scripts/build_coreth.sh | 35 ++--- scripts/build_local_image.sh | 23 ++- scripts/constants.sh | 17 +++ scripts/versions.sh | 7 + 15 files changed, 594 insertions(+), 118 deletions(-) delete mode 100755 .ci/run_e2e_tests.sh create mode 100644 .github/workflows/e2e.yml create mode 100755 .github/workflows/publish_image.sh create mode 100755 .github/workflows/run_e2e_tests.sh create mode 100755 .kurtosis/build-and-run-core.sh create mode 100755 .kurtosis/kurtosis.sh create mode 100644 scripts/constants.sh create mode 100644 scripts/versions.sh diff --git a/.ci/run_e2e_tests.sh b/.ci/run_e2e_tests.sh deleted file mode 100755 index c0b2106abe36..000000000000 --- a/.ci/run_e2e_tests.sh +++ /dev/null @@ -1,52 +0,0 @@ -SCRIPTS_PATH=$(cd $(dirname "${BASH_SOURCE[0]}"); pwd) -SRC_PATH=$(dirname "${SCRIPTS_PATH}") - -# Early auth to avoid limit rating -if [[ -z ${DOCKER_USERNAME} ]]; then - echo "Skipping E2E Tests for untrusted build" - exit 0 -else - echo "$DOCKER_PASS" | docker login --username "$DOCKER_USERNAME" --password-stdin -fi - -# Build the runnable Avalanche docker image -bash "${SRC_PATH}"/scripts/build_image.sh -AVALANCHE_IMAGE_REPO=$(docker image ls --format="{{.Repository}}" | head -n 1) -AVALANCHE_IMAGE_TAG=$(docker image ls --format="{{.Tag}}" | head -n 1) -AVALANCHE_IMAGE="$AVALANCHE_IMAGE_REPO:$AVALANCHE_IMAGE_TAG" -echo "Using Avalanche Image: $AVALANCHE_IMAGE" - -DOCKER_REPO="avaplatform" -BYZANTINE_IMAGE="$DOCKER_REPO/avalanche-byzantine:v0.2.0-rc.1" -TEST_SUITE_IMAGE="$DOCKER_REPO/avalanche-testing:v0.11.0-rc.2" - -# Kurtosis Environment Parameters -KURTOSIS_CORE_CHANNEL="1.0.3" -INITIALIZER_IMAGE="kurtosistech/kurtosis-core_initializer:${KURTOSIS_CORE_CHANNEL}" -API_IMAGE="kurtosistech/kurtosis-core_api:${KURTOSIS_CORE_CHANNEL}" -PARALLELISM=4 - -docker pull "${BYZANTINE_IMAGE}" -docker pull "$TEST_SUITE_IMAGE" - -SUITE_EXECUTION_VOLUME="avalanche-test-suite_${AVALANCHE_IMAGE_TAG}_$(date +%s)" -docker volume create "${SUITE_EXECUTION_VOLUME}" - -# Docker only allows you to have spaces in the variable if you escape them or use a Docker env file -CUSTOM_ENV_VARS_JSON="CUSTOM_ENV_VARS_JSON={\"AVALANCHE_IMAGE\":\"${AVALANCHE_IMAGE}\",\"BYZANTINE_IMAGE\":\"${BYZANTINE_IMAGE}\"}" - -echo "${CUSTOM_ENV_VARS_JSON}" -echo "${KURTOSIS_API_IMAGE}" -echo "${INITIALIZER_IMAGE}" -docker run \ - --mount "type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock" \ - --mount "type=volume,source=${SUITE_EXECUTION_VOLUME},target=/suite-execution" \ - --env "${CUSTOM_ENV_VARS_JSON}" \ - --env "TEST_SUITE_IMAGE=${TEST_SUITE_IMAGE}" \ - --env "SUITE_EXECUTION_VOLUME=${SUITE_EXECUTION_VOLUME}" \ - --env "KURTOSIS_API_IMAGE=${API_IMAGE}" \ - --env "PARALLELISM=${PARALLELISM}" \ - `# In Bash, this is how you feed arguments exactly as-is to a child script (since ${*} loses quoting and ${@} trips set -e if no arguments are passed)` \ - `# It basically says, "if and only if ${1} exists, evaluate ${@}"` \ - ${1+"${@}"} \ - "${INITIALIZER_IMAGE}" diff --git a/.dockerignore b/.dockerignore index 3e23d7e4be83..3c0dd2fef296 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,10 +1,13 @@ .ci -.git .github .gitignore .golangci.yml + .idea .vscode LICENSE *.md + +Dockerfile + diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 000000000000..3bb2ce1529c5 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,25 @@ +name: Test + build Image + +on: + push: + branches: + - kurtosis_upgrade + +jobs: + e2e_plus_publish: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - name: Run Kurtosis Tests + env: # Secret as an environment variable + KURTOSIS_CLIENT_ID: ${{ secrets.kurtosis_client_id }} + KURTOSIS_CLIENT_SECRET: ${{ secrets.kurtosis_client_secret }} + DOCKER_USERNAME: ${{ secrets.docker_username }} + DOCKER_PASS: ${{ secrets.docker_pass }} + run: ./run_e2e_tests.sh + + - name: Publish image to DockerHub + env: # Secret as an environment variable + DOCKER_USERNAME: ${{ secrets.docker_username }} + DOCKER_PASS: ${{ secrets.docker_pass }} + run: .github/workflows/publish_image.sh diff --git a/.github/workflows/publish_image.sh b/.github/workflows/publish_image.sh new file mode 100755 index 000000000000..9e9477177e72 --- /dev/null +++ b/.github/workflows/publish_image.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +# Skip if this is not on the main public repo or +# if this is not a trusted build (Docker Credentials are not set) +#if [[ $TRAVIS_REPO_SLUG != "ava-labs/avalanchego" || -z "$DOCKER_USERNAME" ]]; then +# exit 0; +#fi + +# Directory above this script +AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) + +# Load the versions +source "$AVALANCHE_PATH"/scripts/versions.sh + +# Load the constants +source "$AVALANCHE_PATH"/scripts/constants.sh + +echo "Path: $AVALANCHE_PATH" +current_branch=$(git rev-parse --abbrev-ref HEAD) + +if [[ $current_branch == "master" ]]; then + echo "Tagging current avalanchego image as $dockerhub_repo:latest" + docker_image=$dockerhub_repo:latest +fi + +if [[ $current_branch == "dev" ]]; then + echo "Tagging current avalanchego image as $dockerhub_repo:dev" + docker_image=$dockerhub_repo:dev +fi + +if [[ $current_branch != "" ]]; then + echo "Tagging current avalanchego image as $dockerhub_repo:current_branch" + docker_image=$dockerhub_repo:current_branch +fi + +echo "Pushing: $docker_image" + +#echo "$DOCKER_PASS" | docker login --username "$DOCKER_USERNAME" --password-stdin +# +## pushing image with tags +#docker push $docker_image diff --git a/.github/workflows/run_e2e_tests.sh b/.github/workflows/run_e2e_tests.sh new file mode 100755 index 000000000000..b531440c3bfe --- /dev/null +++ b/.github/workflows/run_e2e_tests.sh @@ -0,0 +1,56 @@ +set -o errexit +set -o nounset +set -o pipefail + +root_dirpath="$(cd "$(dirname "${0}")" && cd ../../ && pwd)" +kurtosis_core_dirpath="${root_dirpath}/.kurtosis" + +echo "root_dirpath: ${root_dirpath}" +echo "kurtosis_core_dirpath: ${kurtosis_core_dirpath}" + +# Define avalanche byzantine version to use +avalanche_byzantine_image="avaplatform/avalanche-byzantine:v0.2.0-rc.1" +# Define avalanche testing version to use +avalanche_testing_image="avaplatform/avalanche-testing:kurtosis_upgrade" + +# Fetch the images +# If Docker Credentials are not available fail +if [[ -z ${DOCKER_USERNAME} ]]; then + echo "Skipping Byzantine Tests because Docker Credentials were not present." + exit 1 +else + echo "$DOCKER_PASS" | docker login --username "$DOCKER_USERNAME" --password-stdin + docker pull ${avalanche_testing_image} + docker pull ${avalanche_byzantine_image} +fi + +# Setting the build ID +GIT_COMMIT_ID=$( git rev-list -1 HEAD ) + +# Build current avalanchego +"${root_dirpath}"/scripts/build_image.sh + +# Target built version to use in avalanche-testing +# Todo hook this with the e2e matchup +#avalanche_image_tag=$(docker image ls --format="{{.Tag}}" | head -n 1) +avalanche_image_tag="dev" +avalanche_image="avaplatform/avalanchego:${avalanche_image_tag}" + +echo "Running Avalanche Image: ${avalanche_image}" +echo "Running Avalanche Image Tag: ${avalanche_image_tag}" +echo "Running Avalanche Testing Image: ${avalanche_testing_image}" +echo "Running Avalanche Byzantine Image: ${avalanche_byzantine_image}" +echo "Git Commit ID : ${GIT_COMMIT_ID}" + + +# >>>>>>>> avalanche-testing custom parameters <<<<<<<<<<<<< +custom_params_json="{ + \"isKurtosisCoreDevMode\": false, + \"avalanchegoImage\":\"${avalanche_image}\", + \"avalanchegoByzantineImage\":\"${avalanche_byzantine_image}\" +}" +# >>>>>>>> avalanche-testing custom parameters <<<<<<<<<<<<< + +bash "${kurtosis_core_dirpath}/kurtosis.sh" \ + --custom-params "${custom_params_json}" \ + "${avalanche_testing_image}" diff --git a/.kurtosis/build-and-run-core.sh b/.kurtosis/build-and-run-core.sh new file mode 100755 index 000000000000..663d65006d0c --- /dev/null +++ b/.kurtosis/build-and-run-core.sh @@ -0,0 +1,140 @@ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# This script is intended to be wrapped with a thin script passing in testsuite-specific +# variables (e.g. the name of the testsuite image) +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +set -euo pipefail + +# ====================== CONSTANTS ======================================================= +BUILD_ACTION="build" +RUN_ACTION="run" +BOTH_ACTION="all" +HELP_ACTION="help" + +# ====================== ARG PARSING ======================================================= +show_help_and_exit() { + echo "$(basename ${0}) action suite_image_name repo_dirpath dockerfile_filepath [run_arg1] [run_arg2]..." + echo "" + echo " This script will optionally a) build your Kurtosis testsuite into a Docker image and/or b) run it via a call to the kurtosis.sh script" + echo "" + echo " action Determines the action this script will execute ('${HELP_ACTION}' to print this message, '${BUILD_ACTION}' to only build the testsuite" + echo " image, '${RUN_ACTION}' to just run the testsuite image, and '${BOTH_ACTION}' to both build and run the testsuite)" + echo " suite_image The name to give the built Docker image containing the testsuite" + echo " repo_dirpath Path to the root of the repo containing the testsuite to build" + echo " dockerfile_filepath Filepath to the Dockerfile for building the testsuite Docker image" + echo " wrapper_filepath Filepath to the kurtosis.sh wrapper script" + echo " run_arg Argument to pass to kurtosis.sh when running the testsuite (use '--help' here to see all options)" + echo "" + echo " To see the args the kurtosis.sh script accepts for the 'run' phase, call '$(basename ${0}) all --help'" + echo "" + exit 1 # Exit with error code so we dont't get spurious CI passes +} + +if [ "${#}" -lt 5 ]; then + show_help_and_exit +fi + +action="${1:-}" +shift 1 + +suite_image="${1:-}" +shift 1 + +repo_dirpath="${1:-}" +shift 1 + +dockerfile_filepath="${1:-}" +shift 1 + +wrapper_filepath="${1:-}" +shift 1 + +do_build=true +do_run=true +case "${action}" in + ${HELP_ACTION}) + show_help_and_exit + ;; + ${BUILD_ACTION}) + do_build=true + do_run=false + ;; + ${RUN_ACTION}) + do_build=false + do_run=true + ;; + ${BOTH_ACTION}) + do_build=true + do_run=true + ;; + *) + echo "Error: Action argument must be one of '${HELP_ACTION}', '${BUILD_ACTION}', '${RUN_ACTION}', or '${BOTH_ACTION}'" >&2 + exit 1 + ;; +esac + +# ====================== ARG VALIDATION =================================================== +if [ -z "${suite_image}" ]; then + echo "Error: Suite image name cannot be empty" >&2 + exit 1 +fi +if [ -z "${repo_dirpath}" ]; then + echo "Error: Repo dirpath cannot be empty" >&2 + exit 1 +fi +if ! [ -d "${repo_dirpath}" ]; then + echo "Error: Repo dirpath '${repo_dirpath}' is not a directory" >&2 + exit 1 +fi +if [ -z "${dockerfile_filepath}" ]; then + echo "Error: Dockerfile filepath cannot be empty" >&2 + exit 1 +fi +if ! [ -f "${dockerfile_filepath}" ]; then + echo "Error: Dockerfile filepath '${dockerfile_filepath}' is not a file" >&2 + exit 1 +fi +if [ -z "${wrapper_filepath}" ]; then + echo "Error: Wrapper filepath cannot be empty" >&2 + exit 1 +fi +if ! [ -f "${wrapper_filepath}" ]; then + echo "Error: Wrapper filepath '${dockerfile_filepath}' is not a file" >&2 + exit 1 +fi + +# ====================== MAIN LOGIC ======================================================= +# Captures the first of tag > branch > commit +git_ref="$(cd "${repo_dirpath}" && git describe --tags --exact-match 2> /dev/null || git symbolic-ref -q --short HEAD || git rev-parse --short HEAD)" +if [ "${git_ref}" == "" ]; then + echo "Error: Could not determine a Git ref to use for the Docker tag; is the repo a Git directory?" >&2 + exit 1 +fi + +docker_tag="$(echo "${git_ref}" | sed 's,[/:],_,g')" +if [ "${docker_tag}" == "" ]; then + echo "Error: Sanitizing Git ref to get a Docker tag for the testsuite yielded an empty string" >&2 + exit 1 +fi + +if "${do_build}"; then + if ! [ -f "${repo_dirpath}"/.dockerignore ]; then + echo "Error: No .dockerignore file found in root of repo '${repo_dirpath}'; this is required so Docker caching is enabled and your testsuite builds remain quick" >&2 + exit 1 + fi + + echo "Building '${suite_image}' Docker image..." + if ! docker build -t "${suite_image}:${docker_tag}" -f "${dockerfile_filepath}" "${repo_dirpath}"; then + echo "Error: Docker build of the testsuite failed" >&2 + exit 1 + fi +fi + +if "${do_run}"; then + # The funky ${1+"${@}"} incantation is how you you feed arguments exactly as-is to a child script in Bash + # ${*} loses quoting and ${@} trips set -e if no arguments are passed, so this incantation says, "if and only if + # ${1} exists, evaluate ${@}" + if ! bash "${wrapper_filepath}" ${1+"${@}"} "${suite_image}:${docker_tag}"; then + echo "Error: Testsuite execution failed" + exit 1 + fi +fi diff --git a/.kurtosis/kurtosis.sh b/.kurtosis/kurtosis.sh new file mode 100755 index 000000000000..a3d1cd85e3bb --- /dev/null +++ b/.kurtosis/kurtosis.sh @@ -0,0 +1,226 @@ +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# +# Do not modify this file! It will get overwritten when you upgrade Kurtosis! +# +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +set -euo pipefail + + + +# ============================================================================================ +# Constants +# ============================================================================================ +# The directory where Kurtosis will store files it uses in between executions, e.g. access tokens +# Can make this configurable if needed +KURTOSIS_DIRPATH="${HOME}/.kurtosis" + +KURTOSIS_CORE_TAG="1.8" +KURTOSIS_DOCKERHUB_ORG="kurtosistech" +INITIALIZER_IMAGE="${KURTOSIS_DOCKERHUB_ORG}/kurtosis-core_initializer:${KURTOSIS_CORE_TAG}" +API_IMAGE="${KURTOSIS_DOCKERHUB_ORG}/kurtosis-core_api:${KURTOSIS_CORE_TAG}" + +POSITIONAL_ARG_DEFINITION_FRAGMENTS=2 + + + +# ============================================================================================ +# Arg Parsing +# ============================================================================================ +function print_help_and_exit() { + echo "" + echo "$(basename "${0}") [--custom-params custom_params_json] [--client-id client_id] [--client-secret client_secret] [--help] [--kurtosis-log-level kurtosis_log_level] [--list] [--parallelism parallelism] [--tests test_names] [--test-suite-log-level test_suite_log_level] test_suite_image" + echo "" + echo " --custom-params custom_params_json JSON string containing arbitrary data that will be passed as-is to your testsuite, so it can modify its behaviour based on input (default: {})" + echo " --client-id client_id An OAuth client ID which is needed for running Kurtosis in CI, and should be left empty when running Kurtosis on a local machine" + echo " --client-secret client_secret An OAuth client secret which is needed for running Kurtosis in CI, and should be left empty when running Kurtosis on a local machine" + echo " --help Display this message" + echo " --kurtosis-log-level kurtosis_log_level The log level that all output generated by the Kurtosis framework itself should log at (panic|fatal|error|warning|info|debug|trace) (default: info)" + echo " --list Rather than running the tests, lists the tests available to run" + echo " --parallelism parallelism The number of texts to execute in parallel (default: 4)" + echo " --tests test_names List of test names to run, separated by ',' (default or empty: run all tests)" + echo " --test-suite-log-level test_suite_log_level A string that will be passed as-is to the test suite container to indicate what log level the test suite container should output at; this string should be meaningful to the test suite container because Kurtosis won't know what logging framework the testsuite uses (default: info)" + echo " test_suite_image The Docker image containing the testsuite to execute" + + echo "" + exit 1 # Exit with an error code, so that if it gets accidentally called in parent scripts/CI it fails loudly +} + + + +# ============================================================================================ +# Arg Parsing +# ============================================================================================ +client_id="" +client_secret="" +custom_params_json="{}" +do_list="false" +kurtosis_log_level="info" +parallelism="4" +show_help="false" +test_names="" +test_suite_image="" +test_suite_log_level="info" + + + +POSITIONAL=() +while [ ${#} -gt 0 ]; do + key="${1}" + case "${key}" in + + --custom-params) + + custom_params_json="${2}" + shift # Shift to clear out the flag + shift # Shift again to clear out the value + ;; + + --client-id) + + client_id="${2}" + shift # Shift to clear out the flag + shift # Shift again to clear out the value + ;; + + --client-secret) + + client_secret="${2}" + shift # Shift to clear out the flag + shift # Shift again to clear out the value + ;; + + --help) + show_help="true" + shift # Shift to clear out the flag + + ;; + + --kurtosis-log-level) + + kurtosis_log_level="${2}" + shift # Shift to clear out the flag + shift # Shift again to clear out the value + ;; + + --list) + do_list="true" + shift # Shift to clear out the flag + + ;; + + --parallelism) + + parallelism="${2}" + shift # Shift to clear out the flag + shift # Shift again to clear out the value + ;; + + --tests) + + test_names="${2}" + shift # Shift to clear out the flag + shift # Shift again to clear out the value + ;; + + --test-suite-log-level) + + test_suite_log_level="${2}" + shift # Shift to clear out the flag + shift # Shift again to clear out the value + ;; + + -*) + echo "ERROR: Unrecognized flag '${key}'" >&2 + exit 1 + ;; + *) + POSITIONAL+=("${1}") + shift + ;; + esac +done + +if "${show_help}"; then + print_help_and_exit +fi + +# Restore positional parameters and assign them to variables +set -- "${POSITIONAL[@]}" +test_suite_image="${1:-}" + + + + + +# ============================================================================================ +# Arg Validation +# ============================================================================================ +if [ "${#}" -ne 1 ]; then + echo "ERROR: Expected 1 positional variables but got ${#}" >&2 + print_help_and_exit +fi + +if [ -z "$test_suite_image" ]; then + echo "ERROR: Variable 'test_suite_image' cannot be empty" >&2 + exit 1 +fi + + + +# ============================================================================================ +# Main Logic +# ============================================================================================# Because Kurtosis X.Y.Z tags are normalized to X.Y so that minor patch updates are transparently +# used, we need to pull the latest API & initializer images +echo "Pulling latest versions of API & initializer image..." +if ! docker pull "${INITIALIZER_IMAGE}"; then + echo "WARN: An error occurred pulling the latest version of the initializer image (${INITIALIZER_IMAGE}); you may be running an out-of-date version" >&2 +else + echo "Successfully pulled latest version of initializer image" +fi +if ! docker pull "${API_IMAGE}"; then + echo "WARN: An error occurred pulling the latest version of the API image (${API_IMAGE}); you may be running an out-of-date version" >&2 +else + echo "Successfully pulled latest version of API image" +fi + +# Kurtosis needs a Docker volume to store its execution data in +# To learn more about volumes, see: https://docs.docker.com/storage/volumes/ +sanitized_image="$(echo "${test_suite_image}" | sed 's/[^a-zA-Z0-9_.-]/_/g')" +suite_execution_volume="$(date +%Y-%m-%dT%H.%M.%S)_${sanitized_image}" +if ! docker volume create "${suite_execution_volume}" > /dev/null; then + echo "ERROR: Failed to create a Docker volume to store the execution files in" >&2 + exit 1 +fi + +if ! mkdir -p "${KURTOSIS_DIRPATH}"; then + echo "ERROR: Failed to create the Kurtosis directory at '${KURTOSIS_DIRPATH}'" >&2 + exit 1 +fi + +docker run \ + `# The Kurtosis initializer runs inside a Docker container, but needs to access to the Docker engine; this is how to do it` \ + `# For more info, see the bottom of: http://jpetazzo.github.io/2015/09/03/do-not-use-docker-in-docker-for-ci/` \ + --mount "type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock" \ + \ + `# Because the Kurtosis initializer runs inside Docker but needs to persist & read files on the host filesystem between execution,` \ + `# the container expects the Kurtosis directory to be bind-mounted at the special "/kurtosis" path` \ + --mount "type=bind,source=${KURTOSIS_DIRPATH},target=/kurtosis" \ + \ + `# The Kurtosis initializer image requires the volume for storing suite execution data to be mounted at the special "/suite-execution" path` \ + --mount "type=volume,source=${suite_execution_volume},target=/suite-execution" \ + \ + `# Keep these sorted alphabetically` \ + --env CLIENT_ID="${client_id}" \ + --env CLIENT_SECRET="${client_secret}" \ + --env CUSTOM_PARAMS_JSON="${custom_params_json}" \ + --env DO_LIST="${do_list}" \ + --env KURTOSIS_API_IMAGE="${API_IMAGE}" \ + --env KURTOSIS_LOG_LEVEL="${kurtosis_log_level}" \ + --env PARALLELISM="${parallelism}" \ + --env SUITE_EXECUTION_VOLUME="${suite_execution_volume}" \ + --env TEST_NAMES="${test_names}" \ + --env TEST_SUITE_IMAGE="${test_suite_image}" \ + --env TEST_SUITE_LOG_LEVEL="${test_suite_log_level}" \ + \ + "${INITIALIZER_IMAGE}" diff --git a/.travis.yml b/.travis.yml index 901ae1bc5caf..040098114c7c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,14 +18,6 @@ jobs: script: - ".ci/lint.sh" - - stage: build - os: linux - dist: bionic - script: - - ".ci/build_and_test.sh" - - ".ci/run_e2e_tests.sh" - - ".ci/after_success.sh" - - stage: build os: osx osx_image: xcode11.4 diff --git a/Dockerfile b/Dockerfile index 43f361f2fbb8..7983ab62f55e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,23 @@ -# syntax=docker/dockerfile:experimental +# ============= Compilation Stage ================ +FROM golang:1.15.5-alpine AS builder +RUN apk add --no-cache bash git make gcc musl-dev linux-headers git ca-certificates -ARG AVALANCHEGO_COMMIT -FROM golang:1.15.5-buster +WORKDIR /build +# Copy and download dependencies using go mod +COPY go.mod . +COPY go.sum . +RUN go mod download -ARG AVALANCHEGO_COMMIT +# Copy the code into the container +COPY . . -RUN mkdir -p /go/src/github.com/ava-labs +# Build avalanchego and plugins +RUN ./scripts/build.sh -WORKDIR $GOPATH/src/github.com/ava-labs/ -COPY . avalanchego +# ============= Cleanup Stage ================ +FROM alpine:3.13 AS execution -WORKDIR $GOPATH/src/github.com/ava-labs/avalanchego -RUN export AVALANCHEGO_COMMIT=$AVALANCHEGO_COMMIT -RUN ./scripts/build.sh +WORKDIR /run -RUN ln -sv $GOPATH/src/github.com/ava-labs/avalanchego/ /avalanchego +# Copy the executables into the container +COPY --from=builder /build/build/ . diff --git a/scripts/build.sh b/scripts/build.sh index a9b5f93af6f4..ed47c615a88e 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -4,23 +4,27 @@ set -o errexit set -o nounset set -o pipefail -# Download dependencies -echo "Downloading dependencies..." -go mod download +# Directory above this script +AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) -# Set GOPATH -GOPATH="$(go env GOPATH)" +# Load the versions +source "$AVALANCHE_PATH"/scripts/versions.sh -AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) # Directory above this script -BUILD_DIR=$AVALANCHE_PATH/build # Where binaries go -PLUGIN_DIR="$BUILD_DIR/plugins" # Where plugin binaries (namely coreth) go +# Load the constants +source "$AVALANCHE_PATH"/scripts/constants.sh +# Download dependencies +echo "Downloading dependencies..." +go mod download -"$AVALANCHE_PATH/scripts/build_avalanche.sh" +# Build avalanchego +"$AVALANCHE_PATH"/scripts/build_avalanche.sh -"$AVALANCHE_PATH/scripts/build_coreth.sh" +# Build coreth +"$AVALANCHE_PATH"/scripts/build_coreth.sh -if [[ -f "$BUILD_DIR/avalanchego" && -f "$PLUGIN_DIR/evm" ]]; then +# Exit build successfully if the binaries are created +if [[ -f "$build_dir/avalanchego" && -f "$plugin_dir/evm" ]]; then echo "Build Successful" exit 0 else diff --git a/scripts/build_avalanche.sh b/scripts/build_avalanche.sh index 78c14c63ac67..31c63165cc70 100755 --- a/scripts/build_avalanche.sh +++ b/scripts/build_avalanche.sh @@ -4,14 +4,17 @@ set -o errexit set -o nounset set -o pipefail -# Set GOPATH -GOPATH="$(go env GOPATH)" +# Directory above this script +AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) -AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) # Directory above this script -BUILD_DIR=$AVALANCHE_PATH/build # Where binaries go +# Load the versions +source "$AVALANCHE_PATH"/scripts/versions.sh -GIT_COMMIT=${AVALANCHEGO_COMMIT:-$( git rev-list -1 HEAD )} +# Load the constants +source "$AVALANCHE_PATH"/scripts/constants.sh -# Build aVALANCHE -echo "Building Avalanche..." -go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -o "$BUILD_DIR/avalanchego" "$AVALANCHE_PATH/main/"*.go +git_commit=${AVALANCHEGO_COMMIT:-$( git rev-list -1 HEAD )} + +# Build AVALANCHE +echo "Building Avalanche @ ${git_commit} ..." +go build -ldflags "-X main.GitCommit=$git_commit" -o "$build_dir/avalanchego" "$AVALANCHE_PATH/main/"*.go diff --git a/scripts/build_coreth.sh b/scripts/build_coreth.sh index c4fcf2cb71fd..6c8b00d604d1 100755 --- a/scripts/build_coreth.sh +++ b/scripts/build_coreth.sh @@ -4,36 +4,33 @@ set -o errexit set -o nounset set -o pipefail -# Set GOPATH -GOPATH="$(go env GOPATH)" -CURRENT_DIR="$(pwd)" +# Directory above this script +AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) -AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) # Directory above this script -BUILD_DIR="$AVALANCHE_PATH/build" # Where binaries go -PLUGIN_DIR="$BUILD_DIR/plugins" # Where plugin binaries (namely coreth) go -BINARY_PATH="$PLUGIN_DIR/evm" +# Load the versions +source "$AVALANCHE_PATH"/scripts/versions.sh -CORETH_VER="v0.4.2-rc.4" - -CORETH_PATH="$GOPATH/pkg/mod/github.com/ava-labs/coreth@$CORETH_VER" +# Load the constants +source "$AVALANCHE_PATH"/scripts/constants.sh +# check if there's args defining different coreth source and build paths if [[ $# -eq 2 ]]; then - CORETH_PATH=$1 - BINARY_PATH=$2 + coreth_path=$1 + evm_path=$2 elif [[ $# -eq 0 ]]; then - if [[ ! -d "$CORETH_PATH" ]]; then - go get "github.com/ava-labs/coreth@$CORETH_VER" + if [[ ! -d "$coreth_path" ]]; then + go get "github.com/ava-labs/coreth@$coreth_version" fi else echo "Invalid arguments to build coreth. Requires either no arguments (default) or two arguments to specify coreth directory and location to add binary." exit 1 fi -# Build Coreth, which is run as a subprocess -echo "Building Coreth..." -cd "$CORETH_PATH" -go build -o "$BINARY_PATH" "plugin/"*.go -cd "$CURRENT_DIR" +# Build Coreth +echo "Building Coreth @ ${coreth_version} ..." +cd "$coreth_path" +go build -o "$evm_path" "plugin/"*.go +cd "$AVALANCHE_PATH" # Building coreth + using go get can mess with the go.mod file. go mod tidy diff --git a/scripts/build_local_image.sh b/scripts/build_local_image.sh index 5189e7e64583..1dca27bc38e8 100755 --- a/scripts/build_local_image.sh +++ b/scripts/build_local_image.sh @@ -4,14 +4,21 @@ set -o errexit set -o nounset set -o pipefail -# Build image from current local source -SRC_DIR="$(dirname "${BASH_SOURCE[0]}")" -AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) # Directory above this script -DOCKERHUB_REPO="avaplatform/avalanchego" +# Directory above this script +AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) + +# Load the versions +source "$AVALANCHE_PATH"/scripts/versions.sh + +# Load the constants +source "$AVALANCHE_PATH"/scripts/constants.sh # WARNING: this will use the most recent commit even if there are un-committed changes present -FULL_COMMIT_HASH="$(git --git-dir="$AVALANCHE_PATH/.git" rev-parse HEAD)" -COMMIT_HASH="${FULL_COMMIT_HASH::8}" +full_commit_hash="$(git --git-dir="$AVALANCHE_PATH/.git" rev-parse HEAD)" +commit_hash="${full_commit_hash::8}" + +avalanche_image_tag=$(docker image ls --format="{{.Tag}}" | head -n 1) -echo "Building Docker Image: $DOCKERHUB_REPO:$COMMIT_HASH" -docker build -t "$DOCKERHUB_REPO:$COMMIT_HASH" "$AVALANCHE_PATH" -f "$AVALANCHE_PATH/Dockerfile" --build-arg AVALANCHEGO_COMMIT="$FULL_COMMIT_HASH" +echo "Building Docker Image: $dockerhub_repo:$commit_hash" +echo "Building Docker Image: $dockerhub_repo:$avalanche_image_tag" +docker build -t "$dockerhub_repo:$commit_hash" -t "$dockerhub_repo:${avalanche_image_tag}" "$AVALANCHE_PATH" -f "$AVALANCHE_PATH/Dockerfile" diff --git a/scripts/constants.sh b/scripts/constants.sh new file mode 100644 index 000000000000..4311977bece7 --- /dev/null +++ b/scripts/constants.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Set the PATHS +GOPATH="$(go env GOPATH)" +coreth_path="$GOPATH/pkg/mod/github.com/ava-labs/coreth@$coreth_version" + +# Where AvalancheGo binary goes +build_dir="$AVALANCHE_PATH/build" + +# Where plugin binaries (namely coreth) go +plugin_dir="$build_dir/plugins" +evm_path="$plugin_dir/evm" + +# Avalabs docker hub +dockerhub_repo="avaplatform/avalanchego" +avalanche_image_name="avalanchego" + diff --git a/scripts/versions.sh b/scripts/versions.sh new file mode 100644 index 000000000000..069eab9607de --- /dev/null +++ b/scripts/versions.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + + +# Set up the versions to be used +# Don't export them as their used in the context of other calls +coreth_version=${CORETH_VERSION:-'v0.4.2-rc.4'} +go_ethereum=${GO_ETHEREUM:-'v1.9.21'}