devtool

#!/usr/bin/env bash

# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0

# Firecracker devtool
#
# Use this script to build and test Firecracker.
#
# TL;DR
# Make sure you have Docker installed and properly configured
# (http://docker.com). Then,
#   building: `./devtool build`
#     Then find the binaries under build/debug/
#   testing: `./devtool test`
#     Will run the entire test battery; will take several minutes to complete.
#   deep-dive: `./devtool shell`
#     Open a shell prompt inside the container. Then build or test (or do
#     anything, really) manually.
#
# Still TL;DR: have Docker; ./devtool build; ./devtool test; ./devtool help.
#
#
# Both building and testing are done inside a Docker container. Please make sure
# you have Docker up and running on your system (see http:/docker.com) and your
# user has permission to run Docker containers.
#
# The Firecracker sources dir will be bind-mounted inside the development
# container (under /firecracker) and any files generated by the build process
# will show up under the build/ dir.  This includes the final binaries, as well
# as any intermediate or cache files.
#
# By default, all devtool commands run the container transparently, removing
# it after the command completes. Any persisting files will be stored under
# build/.
# If, for any reason, you want to access the container directly, please use
# `devtool shell`. This will perform the initial setup (bind-mounting the
# sources dir, setting privileges) and will then drop into a BASH shell inside
# the container.
#
# Building:
#   Run `./devtool build`.
#   By default, the debug binaries are built and placed under build/debug/.
#   To build the release version, run `./devtool build --release` instead.
#   You can then find the binaries under build/release/.
#
# Testing:
#   Run `./devtool test`.
#   This will run the entire integration test battery. The testing system is
#   based on pytest (http://pytest.org).
#
# Opening a shell prompt inside the development container:
#   Run `./devtool shell`.
#
# Additional information:
#   Run `./devtool help`.
#
#
# TODO:
#   - Cache test binaries, preserving them across `./devtool test` invocations.
#     At the moment, Firecracker is rebuilt everytime a test is run.
#   - List tests by parsing the `pytest --collect-only` output.
#   - Implement test filtering with `./devtool test --filter <filter>`
#   - Find an easier way to run individual tests on existing binaries.
#   - Add a `./devtool run` command to set up and run Firecracker.
#   - Add a `./devtool diag` command to help with troubleshooting, by checking
#     the most common failure conditions.
#   - Look into caching the Cargo registry within the container and if that
#     would help with reproducible builds (in addition to pinning Cargo.lock)

# Development container image (without tag)
DEVCTR_IMAGE_NO_TAG="public.ecr.aws/firecracker/fcuvm"

# Development container tag
DEVCTR_IMAGE_TAG="v44"

# Development container image (name:tag)
# This should be updated whenever we upgrade the development container.
# (Yet another step on our way to reproducible builds.)
DEVCTR_IMAGE="${DEVCTR_IMAGE_NO_TAG}:${DEVCTR_IMAGE_TAG}"

# Naming things is hard
MY_NAME="Firecracker $(basename "$0")"

# Full path to the Firecracker tools dir on the host.
FC_TOOLS_DIR=$(cd "$(dirname "$0")" && pwd)

# Full path to the Firecracker sources dir on the host.
FC_ROOT_DIR=$(cd "${FC_TOOLS_DIR}/.." && pwd)

# Full path to the build dir on the host.
FC_BUILD_DIR="${FC_ROOT_DIR}/build"

# Full path to devctr dir on the host.
FC_DEVCTR_DIR="${FC_ROOT_DIR}/tools/devctr"

# Path to the linux kernel directory on the host.
KERNEL_DIR="${FC_ROOT_DIR}/.kernel"

# Full path to the cargo registry dir on the host. This appears on the host
# because we want to persist the cargo registry across container invocations.
# Otherwise, any rust crates from crates.io would be downloaded again each time
# we build or test.
CARGO_REGISTRY_DIR="${FC_BUILD_DIR}/cargo_registry"

# Full path to the cargo git registry on the host. This serves the same purpose
# as CARGO_REGISTRY_DIR, for crates downloaded from GitHub repos instead of
# crates.io.
CARGO_GIT_REGISTRY_DIR="${FC_BUILD_DIR}/cargo_git_registry"

# Full path to the cargo target dir on the host.
CARGO_TARGET_DIR="${FC_BUILD_DIR}/cargo_target"

# Full path to the seccompiler cargo target dir on the host.
CARGO_SECCOMPILER_TARGET_DIR="${FC_BUILD_DIR}/seccompiler"

# Full path to the rebase-snap cargo target dir on the host.
CARGO_REBASE_SNAP_TARGET_DIR="${FC_BUILD_DIR}/rebase-snap"

# Full path to the Firecracker sources dir, as bind-mounted in the container.
CTR_FC_ROOT_DIR="/firecracker"

# Full path to the build dir, as bind-mounted in the container.
CTR_FC_BUILD_DIR="${CTR_FC_ROOT_DIR}/build"

# Full path to the cargo target dir, as bind-mounted in the container.
CTR_CARGO_TARGET_DIR="$CTR_FC_BUILD_DIR/cargo_target"

# Full path to the seccompiler cargo target dir, as bind-mounted in the container.
CTR_CARGO_SECCOMPILER_TARGET_DIR="$CTR_FC_BUILD_DIR/seccompiler"

# Full path to the rebase-snap cargo target dir, as bind-mounted in the container.
CTR_CARGO_REBASE_SNAP_TARGET_DIR="$CTR_FC_BUILD_DIR/rebase-snap"

# Full path to the microVM images cache dir
CTR_MICROVM_IMAGES_DIR="$CTR_FC_BUILD_DIR/img"

# Full path to the poetry tmp directory on the container, used for holding
# the lock and toml files.
CTR_POETRY_TMP_DIR="/tmp/poetry"

# Full path to the public key mapping on the guest
PUB_KEY_PATH=/root/.ssh/id_rsa.pub

# Full path to the private key mapping on the guest
PRIV_KEY_PATH=/root/.ssh/id_rsa

# Path to the linux kernel directory, as bind-mounted in the container.
CTR_KERNEL_DIR="${CTR_FC_ROOT_DIR}/.kernel"

# Global options received by $0
# These options are not command-specific, so we store them as global vars
OPT_UNATTENDED=false

# Get the target prefix to avoid repeated calls to uname -m
TARGET_PREFIX="$(uname -m)-unknown-linux-"

DEFAULT_TEST_SESSION_ROOT_PATH=/srv
DEFAULT_RAMDISK_PATH=/mnt/devtool-ramdisk


# Send a decorated message to stdout, followed by a new line
#
say() {
    [ -t 1 ] && [ -n "$TERM" ] \
        && echo "$(tput setaf 2)[$MY_NAME]$(tput sgr0) $*" \
        || echo "[$MY_NAME] $*"
}

# Send a decorated message to stdout, without a trailing new line
#
say_noln() {
    [ -t 1 ] && [ -n "$TERM" ] \
        && echo -n "$(tput setaf 2)[$MY_NAME]$(tput sgr0) $*" \
        || echo "[$MY_NAME] $*"
}

# Send a text message to stderr
#
say_err() {
    [ -t 2 ] && [ -n "$TERM" ] \
        && echo -e "$(tput setaf 1)[$MY_NAME] $*$(tput sgr0)" 1>&2 \
        || echo -e "[$MY_NAME] $*" 1>&2
}

# Send a warning-highlighted text to stdout
say_warn() {
    [ -t 1 ] && [ -n "$TERM" ] \
        && echo "$(tput setaf 3)[$MY_NAME] $*$(tput sgr0)" \
        || echo "[$MY_NAME] $*"
}

# Exit with an error message and (optional) code
# Usage: die [-c <error code>] <error message>
#
die() {
    code=1
    [[ "$1" = "-c" ]] && {
        code="$2"
        shift 2
    }
    say_err "$@"
    exit $code
}

# Exit with an error message if the last exit code is not 0
#
ok_or_die() {
    code=$?
    [[ $code -eq 0 ]] || die -c $code "$@"
}

# Check if Docker is available and exit if it's not.
# Upon returning from this call, the caller can be certain Docker is available.
#
ensure_docker() {
    NEWLINE=$'\n'
    output=$(which docker 2>&1)
    ok_or_die "Docker not found. Aborting." \
        "Please make sure you have Docker (http://docker.com) installed" \
        "and properly configured.${NEWLINE}" \
        "Error: $?, command output: ${output}"

    output=$(docker ps 2>&1)
    ok_or_die "Error accessing Docker. Please make sure the Docker daemon" \
        "is running and that you are part of the docker group.${NEWLINE}" \
        "Error: $?, command output: ${output}${NEWLINE}" \
        "For more information, see" \
        "https://docs.docker.com/install/linux/linux-postinstall/"
}

# Run a command and retry multiple times if it fails. Once it stops
# failing return to normal execution. If there are "retry count"
# failures, set the last error code.
# $1 - command
# $2 - retry count
# $3 - sleep interval between retries
retry_cmd() {
    command=$1
    retry_cnt=$2
    sleep_int=$3

    {
        $command
    } || {
        # Command failed, substract one from retry_cnt
        retry_cnt=$((retry_cnt - 1))

        # If retry_cnt is larger than 0, sleep and call again
        if [ "$retry_cnt" -gt 0 ]; then
            echo "$command failed, retrying..."
            sleep "$sleep_int"
            retry_cmd "$command" "$retry_cnt" "$sleep_int"
        fi
    }
}

# Attempt to download our Docker image. Exit if that fails.
# Upon returning from this call, the caller can be certain our Docker image is
# available on this system.
#
ensure_devctr() {

    # We depend on having Docker present.
    ensure_docker

    # Check if we have the container image available locally. Attempt to
    # download it, if we don't.
    [[ $(docker images -q "$DEVCTR_IMAGE" | wc -l) -gt 0 ]] || {
        say "About to pull docker image $DEVCTR_IMAGE"
        get_user_confirmation || die "Aborted."

        # Run docker pull 5 times in case it fails - sleep 3 seconds
        # between attempts
        retry_cmd "docker pull $DEVCTR_IMAGE" 5 3

        ok_or_die "Error pulling docker image. Aborting."
    }
}

# Check if /dev/kvm exists. Exit if it doesn't.
# Upon returning from this call, the caller can be certain /dev/kvm is
# available.
#
ensure_kvm() {
    [[ -c /dev/kvm ]] || die "/dev/kvm not found. Aborting."
}

# Make sure the build/ dirs are available. Exit if we can't create them.
# Upon returning from this call, the caller can be certain the build/ dirs exist.
#
ensure_build_dir() {
    for dir in "$FC_BUILD_DIR" "$CARGO_TARGET_DIR" \
               "$CARGO_REGISTRY_DIR" "$CARGO_GIT_REGISTRY_DIR"; do
        create_dir "$dir"
    done
}

build_fc_bin_path() {
    target="$1"
    profile="$2"
    echo "$CARGO_TARGET_DIR/$target/$profile/firecracker"
}

build_jailer_bin_path() {
    target="$1"
    profile="$2"
    echo "$CARGO_TARGET_DIR/$target/$profile/jailer"
}

build_seccomp_bin_path() {
    target="$1"
    profile="$2"
    echo "$CARGO_SECCOMPILER_TARGET_DIR/$target/$profile/seccompiler-bin"
}

build_rebase_snap_bin_path() {
    target="$1"
    profile="$2"
    echo "$CARGO_REBASE_SNAP_TARGET_DIR/$target/$profile/rebase-snap"
}

ensure_release_binaries_exist() {
    target=$1
    profile=$2
    firecracker_bin_path=$( build_fc_bin_path "$target" "$profile")
    jailer_bin_path=$( build_jailer_bin_path "$target" "$profile")
    seccompiler_bin_path=$( build_seccomp_bin_path "$target" "$profile")
    rebase_snap_bin_path=$( build_rebase_snap_bin_path "$target" "$profile")

    { [ -f "$firecracker_bin_path" ] && [ -f "$jailer_bin_path" ] && [ -f "$seccompiler_bin_path" ] && \
    [ -f "$rebase_snap_bin_path" ]; } || \
    die "Missing release binaries. Needed files:\n" \
    "* $firecracker_bin_path\n" \
    "* $jailer_bin_path\n" \
    "* $seccompiler_bin_path\n" \
    "* $rebase_snap_bin_path\n" \
    "To build the binaries, run:\n\t$0 build --$profile"
}

# Fix build/ dir permissions after a privileged container run.
# Since the privileged container runs as root, any files it creates will be
# owned by root. This fixes that by recursively changing the ownership of build/
# to the current user.
#
cmd_fix_perms() {
    # Yes, running Docker to get elevated privileges, just to chown some files
    # is a dirty hack.
    run_devctr \
        -- \
        chown -R "$(id -u):$(id -g)" "$CTR_FC_BUILD_DIR"
}

# Builds the development container from its Dockerfile.
#
cmd_build_devctr() {
    arch=$(uname -m)
    docker_file_name="Dockerfile.$arch"
    build_args="--build-arg TMP_POETRY_DIR=$CTR_POETRY_TMP_DIR"

    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")      { cmd_help; exit 1; } ;;
            "-n"|"--no-python-package-upgrade")
                shift
                build_args="$build_args --build-arg POETRY_LOCK_PATH=tools/devctr/poetry.lock"
                ;;
            "--")               { shift; break;     } ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    docker build -t "$DEVCTR_IMAGE_NO_TAG" -f "$FC_DEVCTR_DIR/$docker_file_name" $build_args .

    # Copy back the lockfile, since a new dependency or version would have
    # updated it.
    copy_poetry_lockfile
}

# Prompt the user for confirmation before proceeding.
# Args:
#   $1  prompt text.
#       Default: Continue? (y/n)
#   $2  confirmation input.
#       Default: y
# Return:
#   exit code 0 for successful confirmation
#   exit code != 0 if the user declined
#
get_user_confirmation() {

    # Pass if running unattended
    [[ "$OPT_UNATTENDED" = true ]] && return 0

    # Fail if STDIN is not a terminal (there's no user to confirm anything)
    [[ -t 0 ]] || return 1

    # Otherwise, ask the user
    #
    msg=$([ -n "$1" ] && echo -n "$1" || echo -n "Continue? (y/n) ")
    yes=$([ -n "$2" ] && echo -n "$2" || echo -n "y")
    say_noln "$msg"
    read c && [ "$c" = "$yes" ] && return 0
    return 1
}

# Validate the user supplied version number.
# It must start with 3 groups of integers separated by dot and
# must not contain `wip` or `dirty`.
validate_version() {
    declare version_regex="^([0-9]+\.){2}[0-9]+"
    version="$1"

    if [ -z "$version" ]; then
        die "Version cannot be empty."
    elif [[ ! "$version" =~ $version_regex ]]; then
        die "Invalid version number: $version. Version should start with \$Major.\$Minor.\$Build, see
        https://github.com/firecracker-microvm/firecracker/blob/main/docs/RELEASE_POLICY.md for more information."
    elif [[ "$version" == *"wip"* ]] || [[ "$version" == *"dirty"* ]]; then
        die "Invalid version number: $version. Version should not contain \`wip\` or \`dirty\`."
    fi
}

# Validate that the repo targetted for a release exists.
#
validate_repo() {
    owner="$1"
    repo="$2"
    if [ -z "$owner" ]; then
        die "GitHub owner cannot be empty."
    fi
    git_link="https://api.github.com/repos/$owner/$repo"
    status_code=$(curl --write-out %{http_code} --silent --output /dev/null $git_link)
    if [[ "$status_code" -ne 200 ]] ; then
        die "GitHub repo $git_link is not valid. Please provide a valid user/org and repository."
    fi
}

# Validate there is no tag currently associated with the version provided.
#
check_release_tag() {
    say "Fetching remote tags..."
    git fetch upstream --tags

    if git tag | grep v$1 >/dev/null 2>&1; then
        say_err "Seems that tag provided is already associated with a release! "
        say "Will skip drafting a new release for now."
        return 1
    fi
    return 0
}

# Validate the user supplied kernel version number.
# It must be composed of 2 groups of integers separated by dot, with an optional third group.
validate_kernel_version() {
    local version_regex="^([0-9]+.)[0-9]+(.[0-9]+)?$"
    version="$1"

    if [ -z "$version" ]; then
        die "Kernel version cannot be empty."
    elif [[ ! "$version" =~ $version_regex ]]; then
        die "Invalid version number: $version (expected: \$Major.\$Minor.\$Patch(optional))."
    fi

}

# Compose the text for a new release tag using the information in the changelog,
# between the two specified releases. Section headers (`###`) are removed.
# Args:
#   $1  previous version.
#   $2  new version.
#
compose_tag_text() {
    release_text=$(compose_release_text "$1" "$2") || die "Could not compose release description."
    echo "$release_text" | sed "s/^###\s//g"
}

# Compose the text for a new release using the information in the changelog,
# between the two specified releases.
# Args:
#   $1  previous version.
#   $2  new version.
#
compose_release_text() {
    declare curr_ver="$1"
    declare prev_ver="$2"
    declare changelog="$FC_ROOT_DIR/CHANGELOG.md"

    # Patterns for the sections in the changelog corresponding to the versions.
    pat_curr="^##\s\[$curr_ver\]"
    pat_prev="^##\s\[$prev_ver\]"
    # Extract the section enclosed between the 2 headers and strip off the first
    # 2 and last 2 lines (one is blank and one contains the header `## [A.B.C]`).
    # Then, replace `-` with `*` and remove section headers.
    sed "/$pat_curr/,/$pat_prev/!d" "$changelog" \
      | sed '1,2d;$d' \
      | sed "s/^-/*/g"
}

get_prev_release() {
    declare version=$1
    declare pat_release="^## \[([0-9]+\.){2}[0-9]+.*\]$"
    declare changelog="$FC_ROOT_DIR/CHANGELOG.md"

    grep -q "\[$version\]" "$changelog"
    ok_or_die "No changelog entry for release $version can be found. Make sure you have run ./devtool prepare_release."
    # We work with the assumption that the changelog has already been updated
    # and contains a header (and a corresponding section) for the new release.

    # Step 1: Get all release numbers.
    all_releases=($(grep -E "$pat_release" "$changelog"))
    # Step 2: Trim out headers (`##`).
    all_releases=(${all_releases[@]//##*})

    # Step 3: Walk the array until we come across the desired release number,
    # then pick up the next one. Since the latest releases are at the top of the
    # changelog, the next one in line will be the previous one chronologically.
    # The array now contains all the release numbers in the changelog, enclosed
    # in square brackets.
    for release in "${all_releases[@]}"; do
        if [ -n "$found" ]; then
            # Trim out square brackets.
            prev_version=$(echo "$release" | awk -F"[][]" "{print \$2}")
            break
        elif [ "$release" == "[$version]" ]; then
            found=1
        fi
    done

    if [ -z "$prev_version" ]; then
        die "Could not find a previous release for v$version."
    fi

    echo $prev_version
}

# Auxiliary function for creating the JSON for calling into
# GitHub's API for posting a release.
generate_release_post_data() {
  cat <<EOF
{
  "tag_name": "v$1",
  "name": "Firecracker v$1",
  "body": "$3",
  "draft": $draft,
  "prerelease": $2
}
EOF
}

# Call into GitHub's API to draft a release.
post_release() {
    version="$1"
    user="$2"
    repo="$3"
    token="$4"
    prerelease="$5"
    release_text="$6"
    # Will always create a draft release.
    draft="true"

    # We need to replace the newlines from the changelog content with
    # literal newlines (i.e \n).
    # Second pass: escape quotation marks.
    json_changelog_content=$(echo "$release_text" | awk '{printf "%s\\n", $0}' | sed 's/\"/\\"/g')
    API_JSON="$(generate_release_post_data $version $prerelease "$json_changelog_content")"
    API_RESPONSE_STATUS=$(curl -H "Authorization: token $token" --data "$API_JSON" -s -i https://api.github.com/repos/"$user"/"$repo"/releases)

    if [[ ! "$API_RESPONSE_STATUS" == *"HTTP/2 201"* ]]; then
        die "Could not post release: $API_RESPONSE_STATUS"
    fi
}

get_branch() {
    echo `git rev-parse --abbrev-ref HEAD`
}

# Helper function to run the dev container.
# Usage: run_devctr <docker args> -- <container args>
# Example: run_devctr --privileged -- bash -c "echo 'hello world'"
run_devctr() {
    docker_args=()
    ctr_args=()
    docker_args_done=false
    while [[ $# -gt 0 ]]; do
        [[ "$1" = "--" ]] && {
            docker_args_done=true
            shift
            continue
        }
        [[ $docker_args_done = true ]] && ctr_args+=("$1") || docker_args+=("$1")
        shift
    done

    # If we're running in a terminal, pass the terminal to Docker and run
    # the container interactively
    [[ -t 0 ]] && docker_args+=("-i")
    [[ -t 1 ]] && docker_args+=("-t")

    # Try to pass these environments from host into container for network proxies
    proxies=(http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY)
    for i in "${proxies[@]}"; do
        if [[ ! -z ${!i} ]]; then
            docker_args+=("--env") && docker_args+=("$i=${!i}")
        fi
    done

    # Finally, run the dev container
    # Use 'z' on the --volume parameter for docker to automatically relabel the
    # content and allow sharing between containers.
    docker run "${docker_args[@]}" \
        --rm \
        --volume /dev:/dev \
        --volume "$FC_ROOT_DIR:$CTR_FC_ROOT_DIR:z" \
        --env OPT_LOCAL_IMAGES_PATH="$(dirname "$CTR_MICROVM_IMAGES_DIR")" \
        --env PYTHONDONTWRITEBYTECODE=1 \
        "$DEVCTR_IMAGE" "${ctr_args[@]}"
}

# Helper function to test that the argument provided is a valid path to a SSH key.
#
test_key() {
    ssh-keygen -lf "$1" &>/dev/null
    ret=$?
    [ $ret -ne 0 ] && die "$1 is not a valid key file."
}

# Copies the ```poetry.lock``` file to the host, upgrading the versions if
# requested.
#
copy_poetry_lockfile() {
    # defined in Dockerfile
    lock_file_location_on_host="$FC_DEVCTR_DIR/poetry.lock"
    image_id=$(docker images -q "$DEVCTR_IMAGE_NO_TAG" | head -n 1)
    dummy_container_name=$(uuidgen)
    dummy_container_id=$(docker create --name "$dummy_container_name" "$image_id" bash)

    docker cp \
        "$dummy_container_id":"$CTR_POETRY_TMP_DIR"/poetry.lock \
        "$lock_file_location_on_host"

    docker rm -f "$dummy_container_name"
}

create_dir() {
    # Create a dir for the provided path.
    dir="$1"
    mkdir -p "$dir" || die "Error: cannot create dir $dir"
        [ -x "$dir" ] && [ -w "$dir" ] || \
            {
                say "Wrong permissions for $dir. Attempting to fix them ..."
                chmod +x+w "$dir"
            } || \
            die "Error: wrong permissions for $dir. Should be +x+w"
}

# `$0 help`
# Show the detailed devtool usage information.
#
cmd_help() {
    echo ""
    echo "Firecracker $(basename $0)"
    echo "Usage: $(basename $0) [<args>] <command> [<command args>]"
    echo ""
    echo "Global arguments"
    echo "    -y, --unattended         Run unattended. Assume the user would always"
    echo "                             answer \"yes\" to any confirmation prompt."
    echo ""
    echo "Available commands:"
    echo ""
    echo "    build [--debug|--release] [-l|--libc musl|gnu] [-- [<cargo args>]]"
    echo "        Build the Firecracker binaries."
    echo "        Firecracker is built using the Rust build system (cargo). All arguments after --"
    echo "        will be passed through to cargo."
    echo "        --debug               Build the debug binaries. This is the default."
    echo "        --release             Build the release binaries."
    echo "        -l, --libc musl|gnu   Choose the libc flavor against which Firecracker will"
    echo "                              be linked. Default is musl."
    echo "        --ssh-keys            Provide the paths to the public and private SSH keys on the host"
    echo "                              (in this particular order) required for the git authentication."
    echo "                              It is mandatory that both keys are specified."
    echo ""
    echo "    build_devctr [--no-python-package-upgrade]"
    echo "        Builds the development container from its Dockerfile."
    echo "        -n, --no-python-package-upgrade  Do not update python packages."
    echo ""
    echo "    build_kernel -c|--config [-n|--nproc]"
    echo "        Builds a kernel image custom-tailored for our CI."
    echo "        -c, --config  Path to the config file."
    echo "        -n, --nproc  Number of cores to use for building kernel."

    help_build_release_archive

    echo "    build_rootfs -s|--size [--partuuid]"
    echo "        Builds a rootfs image custom-tailored for use in our CI."
    echo "        -s, --size      Size of the rootfs image. Defaults to 300MB.
                                  The format is the same as that of 'truncates'."
    echo "        -p, --partuuid  Whether to build a partuuid image."
    echo ""
    echo "    checkenv"
    echo "        Performs prerequisites checks needed to execute firecracker."
    echo ""
    echo "    ci"
    echo "        Run a continuous integration test run that executes the integration tests and"
    echo "        checks that the release process works."
    echo ""
    echo "    create_snapshot_artifacts"
    echo "        Runs a tool that generates snapshot artifacts for supported kernel versions."
    echo "        Snapshot mem and state files are saved under \`snapshot_artifacts/\` directory"
    echo ""
    echo "    distclean"
    echo "        Clean up the build tree and remove the docker container."
    echo ""
    echo "    fix_perms"
    echo "        Fixes permissions when devtool dies in the middle of a privileged session."
    echo ""
    echo "    fmt"
    echo "        Auto-format all Rust source files, to match the Firecracker requirements."
    echo "        This should be used as the last step in every commit, to ensure that the"
    echo "        Rust style tests pass."
    echo ""
    echo "    generate_syscall_tables <version>"
    echo "        Generates the syscall tables for seccompiler, according to a given kernel version."
    echo "        Release candidate (rc) linux versions are not allowed."
    echo "        Outputs a rust file for each supported arch: src/seccompiler/src/syscall_table/{arch}.rs"
    echo "        Supported architectures: x86_64 and aarch64."
    echo ""
    echo "    install [-p|--path] [--debug|--release]"
    echo "      Install firecracker, jailer and seccomp binaries to /usr/local/bin or a given path."
    echo "      Only the musl linked binaries are supported."
    echo "        --path                Install binaries to a specified path."
    echo "        --debug               Install the debug binaries."
    echo "        --release             Install the release binaries. This is the default."
    echo ""
    echo "    help"
    echo "        Display this help message."
    echo ""
    echo "    prepare_release <version>"
    echo "        Prepare a new Firecracker release by updating the version number, crate "
    echo "        dependencies and credits."
    echo ""

    help_release

    echo ""
    echo "    shell [--privileged]"
    echo "        Launch the development container and open an interactive BASH shell."
    echo "        -p, --privileged    Run the container as root, in privileged mode."
    echo "                            Running Firecracker via the jailer requires elevated"
    echo "                            privileges, though the build phase does not."
    echo ""
    echo "    tag <version>"
    echo "        Create a git tag for the specified version. The tag message will contain "
    echo "        the contents of CHANGELOG.md enclosed between the header corresponding to "
    echo "        the specified version and the one corresponding to the previous version."
    echo ""
    echo "    test [-- [<pytest args>]]"
    echo "        Run the Firecracker integration tests."
    echo "        The Firecracker testing system is based on pytest. All arguments after --"
    echo "        will be passed through to pytest."
    echo "        -c, --cpuset-cpus cpulist    Set a dedicated cpulist to be used by the tests."
    echo "        -m, --cpuset-mems memlist    Set a dedicated memlist to be used by the tests."
    echo "        -r, --ramdisk size[k|m|g]    Use a ramdisk of `size` MB for
                                               the entire test session (e.g
                                               stored artifacts, Firecracker
                                               binaries, logs/metrics FIFOs
                                               and test created device files)."
    echo ""
    echo "    strip"
    echo "        Strip debug symbols from the Firecracker release binaries."
    echo ""

    help_upload_assets
}

help_release() {
    echo "    release -v <val> -u <val> [-t <val> -r <val> -m <val> -p]"
    echo "        Draft a GitHub release by calling into its API."
    echo "        Firstly, a tag will be created for the release number provided, together with a"
    echo "        description obtained from the CHANGELOG file. The tag will be pushed to a provided remote."
    echo "        Secondly, the script will call into GitHub's API for programatically posting a draft release."
    echo "        For authentication reasons, one needs to pass a GitHub API access token."
    echo "        You can find instructions on how to create one: https://github.blog/2013-05-16-personal-api-tokens/."
    echo "        -v, --version             The targeted release version."
    echo "        -o, --owner               The GitHub owner of the targeted repo."
    echo "        -t, --token               A GitHub Access token with "repo" permissions"
    echo "        -b, --branch              Branch or commit hash to create the tag to."
    echo "        -r, --repo                The name of the repository where the release will be posted."
    echo "        -m, --remote              The name of the remote where the release tag will be pushed."
    echo "        -p, --prerelease          Marks the release as a pre-release."
}

help_upload_assets() {
    echo "    upload_assets -v <val> [-t <val> -r <val> -a <asset1> -a <asset2>]"
    echo "        Uploads assets mentioned through -a|--asset arguments to the draft release for the version specified."
    echo "        This script creates a list of assets for all asset names provided through -a|--asset arguments."
    echo "        If the version provided does not have an associated draft release in the repo, the script exits."
    echo "        If the draft release is found, it proceeds to upload assets one by one."
    echo "        For authentication reasons, one needs to pass a GitHub API access token."
    echo "        You can find instructions on how to create one: https://github.blog/2013-05-16-personal-api-tokens/."
    echo "        -v, --version             The targeted draft release version."
    echo "        -o, --owner               The GitHub owner of the targeted repo."
    echo "        -t, --token               A GitHub Access token with "repo" permissions"
    echo "        -r, --repo                The name of the repository where the draft release exists."
    echo "        -a, --asset               Path to asset to be uploaded."
}

help_build_release_archive() {
    echo "    build_release_archive -v <val>"
    echo "        Building the release archive involves the following steps:"
    echo "            1. Running integration tests."
    echo "            2. Building release binaries and stripping them of debug symbols."
    echo "            3. Verifying artifacts' version against release version targeted."
    echo "            4. Packing all artifacts into \`release-v{version}-{arch}\` directory."
    echo "            5. Creating firecracker-v{version}-{arch} release archive."
    echo "        -v, --version             The targeted release version."
}

# `$0 build` - build Firecracker
# Please see `$0 help` for more information.
#
cmd_build() {

    # By default, we'll build the debug binaries.
    profile="debug"
    libc="musl"

    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")  { cmd_help; exit 1;     } ;;
            "--debug")      { profile="debug";      } ;;
            "--release")    { profile="release";    } ;;
            "--ssh-keys")
                shift
                [[ -z "$1" ]] && \
                    die "Please provide the path to the public SSH key."
                [[ ! -f "$1" ]]  && die "The public key file does not exist: $1."
                test_key "$1"
                host_pub_key_path="$1"
                shift
                [[ -z "$1" ]] && \
                    die "Please provide the path to the private SSH key."
                [[ ! -f "$1" ]]  && die "The private key file does not exist: $1."
                test_key "$1"
                host_priv_key_path="$1"
                ;;
            "-l"|"--libc")
                shift
                [[ "$1" =~ ^(musl|gnu)$ ]] || \
                    die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"."
                libc="$1"
                ;;
            "--")           { shift; break;         } ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    target="$TARGET_PREFIX${libc}"

    # Check prerequisites
    ensure_devctr
    ensure_build_dir

    say "Starting build ($profile, $libc) ..."

    # Cargo uses the debug profile by default. If we're building the release
    # binaries, we need to pass an extra argument to cargo.
    cargo_args=("$@")

    # Add the default target if we did not get that argument in the build command.
    add_default_target=true
    for flag in "${@}"; do
        if [[ "$flag" == "--" ]]; then
            break
        elif [[ "$flag" == "--target" || "$flag" =~ --target=.* ]]; then
            add_default_target=false
        fi
    done

    if [ "$add_default_target" = true ]; then
        cargo_args+=(--target "$target")
    fi

    [ $profile = "release" ] && cargo_args+=("--release")

    # Map the public and private keys to the guest if they are specified.
    [ ! -z "$host_pub_key_path" ] && [ ! -z "$host_priv_key_path" ] &&
        extra_args="--volume $host_pub_key_path:$PUB_KEY_PATH:z \
                    --volume $host_priv_key_path:$PRIV_KEY_PATH:z"

    # Artificially trigger a re-run of the build script,
    # to make sure that `firecracker --version` reports the latest changes.
    touch "$FC_ROOT_DIR/build.rs"

    # Run the cargo build process inside the container.
    # We don't need any special privileges for the build phase, so we run the
    # container as the current user/group.

    # Build seccompiler-bin.
    run_devctr \
        --user "$(id -u):$(id -g)" \
        --workdir "$CTR_FC_ROOT_DIR" \
        ${extra_args} \
        -- \
        cargo build -p seccompiler --bin seccompiler-bin \
            --target-dir "$CTR_CARGO_SECCOMPILER_TARGET_DIR" \
            "${cargo_args[@]}"
    ret=$?

    [ $ret -ne 0 ] && return $ret

    # Build rebase-snap.
    run_devctr \
        --user "$(id -u):$(id -g)" \
        --workdir "$CTR_FC_ROOT_DIR" \
        ${extra_args} \
        -- \
        cargo build -p rebase-snap \
            --target-dir "$CTR_CARGO_REBASE_SNAP_TARGET_DIR" \
            "${cargo_args[@]}"
    ret=$?

    # Build Firecracker.
    run_devctr \
        --user "$(id -u):$(id -g)" \
        --workdir "$CTR_FC_ROOT_DIR" \
        ${extra_args} \
        -- \
        cargo build \
            --target-dir "$CTR_CARGO_TARGET_DIR" \
            "${cargo_args[@]}"
    ret=$?

    [ $ret -ne 0 ] && return $ret

    # Build jailer only in case of musl for compatibility reasons.
    if [ "$libc" == "musl" ];then
        run_devctr \
            --user "$(id -u):$(id -g)" \
            --workdir "$CTR_FC_ROOT_DIR" \
            ${extra_args} \
            -- \
            cargo build -p jailer \
                --target-dir "$CTR_CARGO_TARGET_DIR" \
                "${cargo_args[@]}"

    fi

    ret=$?

    # If `cargo build` was successful, output a message.
    [ $ret -eq 0 ] && {
        cargo_bin_dir="$CARGO_TARGET_DIR/$target/$profile"
        seccompiler_bin_dir="$CARGO_SECCOMPILER_TARGET_DIR/$target/$profile"
        rebase_snap_bin_dir="$CARGO_REBASE_SNAP_TARGET_DIR/$target/$profile"

        # Seccompiler has a different build folder, we need to output two
        # messages.
        say "Build successful."
        say "Firecracker and Jailer binaries placed under $cargo_bin_dir"
        say "Seccompiler-bin binary placed under $seccompiler_bin_dir"
        say "Rebase_snap binary placed under $rebase_snap_bin_dir"
    }

    return $ret
}

cmd_distclean() {
    # List of folders to remove.
    dirs=("build" "test_results")

    for dir in "${dirs[@]}"; do
        if [ -d "$dir" ]; then
            say "Removing $dir"
            rm -rf "$dir"
        fi
    done

    # Remove devctr if it exists
    if [ $(docker images -q "$DEVCTR_IMAGE" | wc -l) -eq "1" ]; then
        say "Removing $DEVCTR_IMAGE"
        docker rmi -f "$DEVCTR_IMAGE"
    fi
}

# Suppress "referencing arguments but none are ever passed" error, because
# strip command accepts only --help argument.
# shellcheck disable=SC2120
cmd_strip() {
    profile="release"
    target="$TARGET_PREFIX""musl"

    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")  { cmd_help; exit 1;     } ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    # Check prerequisites
    ensure_devctr
    ensure_build_dir
    ensure_release_binaries_exist $target $profile

    say "Starting stripping the debug symbols for $profile binaries built against $target target."
    strip_flags="--strip-debug"
    say "Strip flags: $strip_flags."

    run_devctr \
      --user "$(id -u):$(id -g)" \
      -- \
      strip $strip_flags\
        "$CTR_CARGO_TARGET_DIR/$target/$profile/firecracker" \
        "$CTR_CARGO_TARGET_DIR/$target/$profile/jailer" \
        "$CTR_CARGO_SECCOMPILER_TARGET_DIR/$target/$profile/seccompiler-bin" \
        "$CTR_CARGO_REBASE_SNAP_TARGET_DIR/$target/$profile/rebase-snap"
    ret=$?

    [ $ret -eq 0 ] && {
        say "Stripping was successful."
        say "Stripped Firecracker and Jailer binaries placed under $CARGO_TARGET_DIR/$target/$profile."
        say "Stripped seccompiler-bin binary placed under $CARGO_SECCOMPILER_TARGET_DIR/$target/$profile."
        say "Stripped rebase-snap binary placed under $CARGO_REBASE_SNAP_TARGET_DIR/$target/$profile."
    }

    return $ret
}

cmd_build_release_archive() {
    target="$TARGET_PREFIX""musl"
    profile="release"
    seccomp_json="resources/seccomp/$target.json"

    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")      { help_build_release_archive; exit 1; } ;;
            "-v"|"--version")   { version="$2"; shift 2;              } ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
        esac
    done

    validate_version "$version"
    say "Will start preparing release archive for v$version ..."
    get_user_confirmation || die "Aborted."
    release_suffix="v$version-$(uname -m)"
    archive_name="firecracker-$release_suffix.tgz"
    archive_checksum="$archive_name.sha256.txt"
    release_dir="release-$release_suffix"

    # Build release binaries and strip them from debug symbols.
    ( cmd_build --release && cmd_strip ) || die "Aborted."

    # Create release directory or overwrite if it already exists.
    rm -rf "$release_dir" && mkdir "$release_dir"

    # Copy release artifacts to release directory.
    say "Populating release artifacts directory..."
    bin_paths=( "$(build_fc_bin_path "$target" "$profile")"
                "$(build_jailer_bin_path "$target" "$profile")"
                "$(build_seccomp_bin_path "$target" "$profile")"
                "$(build_rebase_snap_bin_path "$target" "$profile")" )

    for bin_path in "${bin_paths[@]}"; do
        add_bin_artifact "$release_dir" "$bin_path" "$release_suffix"
    done

    add_swagger_artifact "$release_dir"
    add_file_artifact "$release_dir" "$seccomp_json" "seccomp-filter-$release_suffix.json"
    for file in "LICENSE" "NOTICE" "THIRD-PARTY"; do
        add_file_artifact "$release_dir" "$file"
    done

    find $release_dir -type f |sort |xargs sha256sum > SHA256SUMS.$(uname -m)

    # Run tests to obtain test reports dir.
    cmd_test || cleanup_release_dir "$release_dir" "Tests failed."
    add_folder_artifact "$release_dir" "test_results" "$release_suffix"

    # Create release archive.
    say "Creating release archive..."
    tar -czf "$archive_name" "$release_dir"
    say "Done. Archive $archive_name successfully created."
    say "Calculating checksum for release archive..."
    sha256sum "$archive_name"  | awk '{print $1}' > "$archive_checksum"
    say "Done. Saved checksum in $archive_checksum."
}

cleanup_release_dir() {
    release_dir="$1"
    error_msg="$2"

    rm -rf "$release_dir"
    die "$error_msg"
}

check_file_existence() {
    artifact="$1"
    if [ ! -f "$artifact" ]; then
        die "Artifact $artifact does not exist!"
    fi
}

add_swagger_artifact() {
    local release_dir="$1"
    local swagger_path="$FC_ROOT_DIR/src/api_server/swagger/firecracker.yaml"

    check_file_existence "$swagger_path"

    # Validate swagger version against target version.
    swagger_ver=$(get_swagger_version "$swagger_path")
    if [ ! "$swagger_ver" == "$version" ]; then
        die "Swagger version: $swagger_ver does not match release version: $version."
    fi

    copy_release_artifact "$swagger_path" "$release_dir/firecracker_spec-v$version.yaml"
}

add_file_artifact() {
    local release_dir="$1"
    local path="$2"
    local name="${3:-"$path"}"

    check_file_existence "$path"
    copy_release_artifact "$path" "$release_dir/$name"
}

add_bin_artifact() {
    local release_dir="$1"
    local path="$2"
    local release_suffix="$3"

    check_file_existence "$path"

    # Validate binary version against target version.
    output=$("$path" --version)
    bin_version=$( echo "$output" | head -1 | grep -oP ' v\K.*')
    if [[ ! "$bin_version" == "$version" ]]; then
        die "Artifact $path's version: $bin_version does not match release version $version."
    fi

    formatted_name="$(basename "$path")-$release_suffix"
    copy_release_artifact "$path" "$release_dir/$formatted_name"
}

add_folder_artifact() {
    local release_dir="$1"
    local path="$2"
    local release_suffix="$3"

    # Ensure the directory exists.
    if [ ! -d "$path" ]; then
        die "Artifact $path does not exist!"
    fi
    formatted_name="$(basename "$path")-$release_suffix"
    copy_release_artifact "$path" "$release_dir/$formatted_name"
}

copy_release_artifact() {
    local from_path="$1"
    local to_path="$2"

    say "Copying release artifact from $from_path to $to_path."
    if [[ -f "$from_path" ]]; then
        cp "$from_path" "$to_path"
    elif [[ -d "$from_path" ]]; then
        cp -r "$from_path" "$to_path"
    fi
}

get_swagger_version() {
    local file="$1"
    grep -oP 'version: \K.*' "$file"
}

mount_ramdisk() {
    local ramdisk_size="$1"
    umount_ramdisk
    DEFAULT_RAMDISK_PATH=$(sudo mktemp -d /mnt/devtool-ramdisk.XXXXXX)
    ok_or_die "Could not create ramdisk directory"
    sudo mkdir -p ${DEFAULT_RAMDISK_PATH} && \
    sudo mount -t tmpfs -o size=${ramdisk_size} tmpfs ${DEFAULT_RAMDISK_PATH}
    ok_or_die "Failed to mount ramdisk to ${DEFAULT_RAMDISK_PATH}. Check the permission."
    sudo mkdir -p ${DEFAULT_RAMDISK_PATH}/srv
    sudo mkdir -p  ${DEFAULT_RAMDISK_PATH}/tmp

    say "Using ramdisk: ${DEFAULT_RAMDISK_PATH}"
}

umount_ramdisk() {
    if [ ! -e "${DEFAULT_RAMDISK_PATH}" ]; then
        return 0
    fi
    if [ ! -d "${DEFAULT_RAMDISK_PATH}" ]; then
        die "${DEFAULT_RAMDISK_PATH} is not a directory."
    fi
    if [ ! -w "${DEFAULT_RAMDISK_PATH}" ]; then
        die "Failed to unmount ${DEFAULT_RAMDISK_PATH}. Check the permission."
    fi
    sudo umount ${DEFAULT_RAMDISK_PATH} &>/dev/null
    sudo rmdir ${DEFAULT_RAMDISK_PATH} &>/dev/null
}

# `$0 test` - run integration tests
# Please see `$0 help` for more information.
#
cmd_test() {

    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")      { cmd_help; exit 1; } ;;
            "-c"|"--cpuset-cpus")
                shift
                local cpuset_cpus="$1"
                ;;
            "-m"|"--cpuset-mems")
                shift
                local cpuset_mems="$1"
                ;;
            "-r"|"--ramdisk")
                shift
                local ramdisk_size="$1"
                local ramdisk=true
                ;;
            "--")               { shift; break;     } ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    # Check prerequisites.
    ensure_kvm
    ensure_devctr
    ensure_build_dir

    # If we got to here, we've got all we need to continue.
    say "$(date -u +'%F %H:%M:%S %Z')"
    say "Starting test run ..."

    if [[ $ramdisk = true ]]; then
        mount_ramdisk ${ramdisk_size}
        ramdisk_args="--env TMPDIR=${DEFAULT_TEST_SESSION_ROOT_PATH}/tmp --volume ${DEFAULT_RAMDISK_PATH}:${DEFAULT_TEST_SESSION_ROOT_PATH}"
    fi

    # Testing (running Firecracker via the jailer) needs root access,
    # in order to set-up the Firecracker jail (manipulating cgroups, net
    # namespaces, etc).
    # We need to run a privileged container to get that kind of access.
    run_devctr \
        --privileged \
        --security-opt seccomp=unconfined \
        --ulimit core=0 \
        --ulimit nofile=4096:4096 \
        --ulimit memlock=-1:-1 \
        --workdir "$CTR_FC_ROOT_DIR/tests" \
        --cpuset-cpus="$cpuset_cpus" \
        --cpuset-mems="$cpuset_mems" \
        -v /boot:/boot \
        ${ramdisk_args} \
        -- \
        pytest "$@"

    ret=$?


    if [[ $ramdisk = true ]]; then
        umount_ramdisk
    fi

    # Running as root would have created some root-owned files under the build
    # dir. Let's fix that.
    cmd_fix_perms

    return $ret
}

# `$0 shell` - drop to a shell prompt inside the dev container
# Please see `$0 help` for more information.
#
cmd_shell() {

    # By default, we run the container as the current user.
    privileged=false

    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")          { cmd_help; exit 1; } ;;
            "-p"|"--privileged")    { privileged=true;  } ;;
            "-r"|"--ramdisk")
                  shift
                  local ramdisk_size="$1"
                  local ramdisk=true
                  ;;
              "--")               { shift; break;     } ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    # Make sure we have what we need to continue.
    ensure_devctr
    ensure_build_dir

    if [[ $ramdisk = true ]]; then
        mount_ramdisk ${ramdisk_size}
        ramdisk_args="--volume ${DEFAULT_RAMDISK_PATH}:${DEFAULT_TEST_SESSION_ROOT_PATH}"
    fi

    if [[ $privileged = true ]]; then
        # If requested, spin up a privileged container.
        #
        say "Dropping to a privileged shell prompt ..."
        say "Note: $FC_ROOT_DIR is bind-mounted under $CTR_FC_ROOT_DIR"
        say_warn "You are running as root; any files that get created under" \
            "$CTR_FC_ROOT_DIR will be owned by root."
        run_devctr \
            --privileged \
            --ulimit nofile=4096:4096 \
            --ulimit memlock=-1:-1 \
            --security-opt seccomp=unconfined \
            --workdir "$CTR_FC_ROOT_DIR" \
            -v /boot:/boot \
            ${ramdisk_args} \
            -- \
            bash
        ret=$?

        # Running as root may have created some root-owned files under the build
        # dir. Let's fix that.
        #
        cmd_fix_perms
    else
        say "Dropping to shell prompt as user $(whoami) ..."
        say "Note: $FC_ROOT_DIR is bind-mounted under $CTR_FC_ROOT_DIR"
        say_warn "You won't be able to run Firecracker via the jailer," \
            "but you can still build it."
        say "You can use \`$0 shell --privileged\` to get a root shell."

        [ -w /dev/kvm ] || \
            say_warn "WARNING: user $(whoami) doesn't have permission to" \
                "access /dev/kvm. You won't be able to run Firecracker."

        run_devctr \
            --user "$(id -u):$(id -g)" \
            --ulimit nofile=4096:4096 \
            --ulimit memlock=-1:-1 \
            --device=/dev/kvm:/dev/kvm \
            --workdir "$CTR_FC_ROOT_DIR" \
            --env PS1="$(whoami)@\h:\w\$ " \
            -- \
            bash --norc
        ret=$?
    fi

    if [[ $ramdisk = true ]]; then
        umount_ramdisk
    fi

    return $ret
}


cmd_create_snapshot_artifacts() {
    privileged=true

    # Make sure we have what we need to continue.
    ensure_devctr
    ensure_build_dir

    say "Creating snapshot artifacts ..."
    run_devctr \
        --privileged \
        --ulimit nofile=4096:4096 \
        --ulimit memlock=-1:-1 \
        --security-opt seccomp=unconfined \
        --workdir "$CTR_FC_ROOT_DIR" \
        ${ramdisk_args} \
        -- \
        python3 tools/create_snapshot_artifact/main.py
    ret=$?

    # Running as root may have created some root-owned files under the build
    # dir. Let's fix that.
    cmd_fix_perms

    return $ret
}


# Auto-format all source code, to match the Firecracker requirements. For the
# moment, this is just a wrapper over `cargo fmt --all`
# Example: `devtool fmt`
#
cmd_fmt() {

    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")      { cmd_help; exit 1; } ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    ensure_devctr

    say "Applying rustfmt ..."
    run_devctr \
        --user "$(id -u):$(id -g)" \
        --workdir "$CTR_FC_ROOT_DIR" \
        -- \
        cargo fmt --all
}


# Prepare a Firecracker release by updating the version, changelog
# and credits. This is to be run before `devtool release`.
# Example: `devtool prepare_release 0.42.0`
#
cmd_prepare_release() {

    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")      { cmd_help; exit 1;    } ;;
            *)                  { version="$1"; break; } ;;
        esac
        shift
    done

    validate_version "$version"

    # We'll be needing the dev container later on.
    ensure_devctr

    # The cargo registry dir needs to be there for `cargo update`.
    ensure_build_dir

    # Get current version from the swagger spec.
    swagger="$FC_ROOT_DIR/src/api_server/swagger/firecracker.yaml"
    curr_ver=$(get_swagger_version "$swagger")

    say "Updating from $curr_ver to $version ..."
    get_user_confirmation || die "Aborted."

    # Update version in files.
    files_to_change=("$swagger"                                 \
                     "$FC_ROOT_DIR/src/firecracker/Cargo.toml"  \
                     "$FC_ROOT_DIR/src/jailer/Cargo.toml"       \
                     "$FC_ROOT_DIR/src/rebase-snap/Cargo.toml"  \
                     "$FC_ROOT_DIR/src/seccompiler/Cargo.toml")
    say "Updating source files:"
    for file in "${files_to_change[@]}"; do
        say "- $file"
        # Dirty hack to make this work on both macOS/BSD and Linux.
        sed -i="" "s/$curr_ver/$version/g" "$file"
        rm -f "${file}="
    done

    # Run `cargo check` to update firecracker and jailer versions in
    # `Cargo.lock`.
    say "Updating lockfile..."
    run_devctr \
        --user "$(id -u):$(id -g)" \
        --workdir "$CTR_FC_ROOT_DIR" \
        -- \
        cargo check
    ok_or_die "cargo check failed."

    # Update credits.
    say "Updating credits..."
    "$FC_TOOLS_DIR/update-credits.sh"

    # Update changelog.
    say "Updating changelog..."
    sed -i="" "s/\[Unreleased\]/\[$version\]/g" "$FC_ROOT_DIR/CHANGELOG.md"
    rm -f "$FC_ROOT_DIR/CHANGELOG.md="
}

# Publish a Firecracker draft release to GitHub.
# For this one needs to pass a GitHub API access token.
# You can find instructions on how to create one: https://github.blog/2013-05-16-personal-api-tokens/.
# If the tag associated with the intended release already exists or if the user has not specified
# a GitHub API token, the script exits.
# Example: `devtool release -v 0.42.0 -b main -t token`
#
cmd_release() {
    local prerelease="false"
    local repo="firecracker"
    local remote="origin"

    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")      { help_release; exit 1; } ;;
            "-v"|"--version")
                shift
                local version="$1"
                ;;
            "-t"|"--token")
                shift
                local token="$1"
                ;;
            "-b"|"--branch")
                shift
                local branch="$1"
                ;;
            "-o"|"--owner")
                shift
                local owner="$1"
                ;;
            "-r"|"--repo")
                shift
                repo="$1"
                ;;
            "-m"|"--remote")
                shift
                remote="$1"
                ;;
            "-p"|"--prerelease")
                shift
                prerelease="true"
                ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    validate_version "$version"
    validate_repo "$owner" "$repo"

    # If branch was not provided, obtain the current branch.
    if [[ -z "$branch" ]]; then
      branch=$(get_branch)
    fi
    say "Will start creating a draft release for branch $branch ..."
    get_user_confirmation || die "Aborted."

    # Check that the release tag associated with the provided version
    # does not already exist. If the tag exists, we will not try to
    # post a release on GitHub.
    check_release_tag "$version" || skip_github_release=true

    git checkout "$branch" || die "Could not checkout branch $branch."

    if [[ -z "$skip_github_release" ]]; then
        if [[ -z "$token" ]]; then
            say_err "In order to draft a GitHub release you need to pass a Github API Access Token!"
        else
            prev_ver=$(get_prev_release "$version") || die "Could not obtain previous release version."

            # Start by creating a local tag and associate to it a description.
            say "Creating local tag..."
            create_local_tag "$version" "$prev_ver" "$branch" || die "Could not create local tag v$version."

            # From now on, any function that will fail will also need to revert creation of local tag.
            say "Continue with remote tag push?"
            get_user_confirmation || die "Local tag not pushed to remote $remote."
            push_local_tag "$version" "$remote" || delete_local_tag "$version" "Failed to push local tag v$version."

            say "Continue with posting a GitHub Release?"
            get_user_confirmation || delete_remote_tag "$version" "$remote" "Could not post release to GitHub."

            release_text=$(compose_release_text "$version" "$prev_ver")
            post_release "$version" "$owner" "$repo" "$token" "$prerelease" "$release_text" ||
            delete_remote_tag "$version" "$remote" "Could not post release to GitHub."
            say "Draft release created successful. Go to https://github.com/$owner/$repo/releases to check it out!"
    fi
fi
}

# Create a tag for the specified release.
# The tag text will be composed from the changelog contents enclosed between the
# specified release number and the previous one.
create_local_tag() {
    version="$1"
    prev_ver="$2"
    branch="$3"

    # Create tag.
    say "Obtaining tag description for local tag v$version..."
    tag_text=$(compose_tag_text "$version" "$prev_ver") || die "Could not compose tag description."
    say "Tag description for v$version:"
    echo "$tag_text"
    say "Continue with tag creation?"
    get_user_confirmation || die "Local v$version not created."

    git tag -a v"$version" "$branch" -m "$tag_text"
    ok_or_die "Local tag v$version not created."
    say "Local tag v$version created."
}

delete_local_tag() {
    version="$1"
    err_msg="$2"
    say "Reverting local tag created by us..."
    git tag -d v"$version"
    die "$2"
}

delete_remote_tag() {
    version="$1"
    remote="$2"
    say "Reverting remote tag created by us..."
    git push --delete "$remote" v"$version"
    delete_local_tag "$1" "$3"
}

push_local_tag() {
    version="$1"
    remote="$2"
    say "Pushing local tag v$version to $remote..."
    git push "$remote" v"$version"
}

# Upload Firecracker release assets to the latest draft release on GitHub.
# For this one needs to pass a GitHub API access token.
# You can find instructions on how to create one: https://github.blog/2013-05-16-personal-api-tokens/.
# If the draft release has not been created or it exists, but the associated tag does not match
# the intended release, the script exits.
# Example: `devtool upload_archives -v 0.42.0 -t token -a archive.tgz`
#
cmd_upload_assets() {
    local repo="firecracker"
    local remote="origin"
    local owner="firecracker-microvm"

    # List of assets to upload.
    local assets=()

    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")      { help_upload_assets; exit 1; } ;;
            "-v"|"--version")
                shift
                local version="$1"
                ;;
            "-t"|"--token")
                shift
                local token="$1"
                ;;
            "-o"|"--owner")
                shift
                owner="$1"
                ;;
            "-r"|"--repo")
                shift
                repo="$1"
                ;;
            "-a"|"--asset")
                shift
                assets+=( "$1" )
                ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    validate_version "$version"
    validate_repo "$owner" "$repo"

    say "Will start uploading assets: ${assets[*]} to draft release v$version..."
    get_user_confirmation || die "Aborted."

    # Get URL for the draft release.
    say "Getting upload URL for the draft release..."
    url=$(get_upload_url "$token" "$owner" "$repo" "$version")
    if [[ ! $? == 0 ]]; then
      die "Abort."
    fi

    say "Uploading assets to draft release..."
    for asset in "${assets[@]}"; do
      upload_asset "$token" "$asset" "$url"
    done
    say "Assets successfully uploaded to draft release. Go to https://github.com/$owner/$repo/releases to check them out!"
}

# Call into GitHub's API to upload asset.
upload_asset() {
  token="$1"
  asset="$2"
  url="$3"

  say "Uploading asset $asset to draft release..."
  get_user_confirmation || die "Could not upload asset to draft release."

  content_type="Content-Type: multipart/form-data"
  asset_upload_url="$url/assets?name=$(basename $asset)"
  API_RESPONSE=$(curl --data-binary @$asset -H "Authorization: token $token" -H "$content_type" -s -i $asset_upload_url)
  if [[ ! "$API_RESPONSE" == *"HTTP/2 201"* ]]; then
    die "Could not upload release asset $asset: $API_RESPONSE"
  fi

  say "Asset $(basename $asset) uploaded to draft release."
}

# Call into GitHub's API to fetch upload URL.
get_upload_url() {
  token="$1"
  owner="$2"
  repo="$3"
  version="$4"

  # Fetch all releases for repo.
  API_RESPONSE=$(curl -X GET -H "Authorization: token $token" -s -i https://api.github.com/repos/$owner/$repo/releases)
  if [[ ! "$API_RESPONSE" == *"HTTP/2 200"* ]]; then
    die "Could not fetch release list: $API_RESPONSE"
  fi

  # Get URL for the latest draft release created.
  draft_release_url=$(echo "${API_RESPONSE}" | grep releases | grep -m 1 -o -P '(?<=\"url\": \").*(?=\",)')

  # Fetch release metadata for the previously computed URL.
  API_RESPONSE=$(curl -X GET -H "Authorization: token $token" -s -i "$draft_release_url")
  if [[ ! "$API_RESPONSE" == *"HTTP/2 200"* ]]; then
    die "Could not fetch release from URL: $draft_release_url"
  fi
  # Verify that the release is indeed draft, fail otherwise.
  if [[ ! "$API_RESPONSE" == *"\"draft\": true"* ]]; then
    die "Could not find draft release."
  fi
  # Check associated release tag.
  if [[ ! "$API_RESPONSE" == *"\"tag_name\": \"v$version\""* ]]; then
    die "Tag name associated to the release does not match version: v$version."
  fi

  # All checks have passed, now return upload URL by substituting
  # `api.github.com` host with `uploads.github.com`.
  echo "$draft_release_url" | sed -r 's/api/uploads/g'
}

# Check if able to run firecracker.
# ../docs/getting-started.md#prerequisites

ensure_kvm_rw () {
    [[ -c /dev/kvm && -w /dev/kvm && -r /dev/kvm ]] || \
        say_err "FAILED: user $(whoami) doesn't have permission to" \
                "access /dev/kvm."
}

check_kernver () {
    KERN_MAJOR=4
    KERN_MINOR=14
    (uname -r | awk -v MAJOR=$KERN_MAJOR -v MINOR=$KERN_MINOR '{ split($0,kver,".");
    if( (kver[1] + (kver[2] / 100) ) <  MAJOR + (MINOR/100) )
    {
      exit 1;
    } }') ||
    say_err "FAILED: Kernel version must be >= $KERN_MAJOR.$KERN_MINOR"
}

# Check Production Host Setup
# ../docs/prod-host-setup.md

check_SMT () {
    (grep -q "^forceoff$\|^notsupported$" \
      /sys/devices/system/cpu/smt/control) ||
    say_warn "WARNING: Hyperthreading ENABLED."
}

check_KPTI () {
    (grep -q "^Mitigation: PTI$" \
      /sys/devices/system/cpu/vulnerabilities/meltdown) || \
    say_warn "WARNING: KPTI NOT SUPPORTED"
}

check_KSM () {
    (grep -q "^0$" /sys/kernel/mm/ksm/run) || \
    say_warn "WARNING: KSM ENABLED"
}

check_IBPB_IBRS () {
    (grep -q "^Mitigation: Full generic retpoline, IBPB, IBRS_FW$"\
      /sys/devices/system/cpu/vulnerabilities/spectre_v2) || \
    say_warn "WARNING: retpoline, IBPB, IBRS: DISABLED."
}

check_L1TF () {
    declare -a CONDITIONS=("Mitigation: PTE Inversion" "VMX: cache flushes")
    for cond in "${CONDITIONS[@]}";
    do (grep -q "$cond" /sys/devices/system/cpu/vulnerabilities/l1tf) ||
       say_warn "WARNING: $cond: DISABLED";
    done
}

check_swap () {
    (grep -q "swap.img" /proc/swaps ) && \
    say_warn "WARNING: SWAP ENABLED"
}

check_SSBD () {
    arch=$(uname -m)
    if [ "$arch" = "aarch64" ]; then
        local param="ssbd=force-on"
    elif [ "$arch" = "x86_64" ]; then
        local param="spec_store_bypass_disable=on"
    fi

    ssbd_sysfs_file="/sys/devices/system/cpu/vulnerabilities/spec_store_bypass"

    if [ -f "$ssbd_sysfs_file" ]; then
        (grep -q "^Vulnerable" $ssbd_sysfs_file) && \
        say_warn "WARNING: SSBD mitigation is either globally disabled or"\
            "system does not support mitigation via prctl or seccomp. Try"\
            "enabling it system-wide, using the \`${param}\` boot parameter."
    else
        say_warn "WARNING: SSBD mitigation not supported on this kernel."\
            "View the prod-host-setup.md for more details."
    fi
}

check_EPT() {
    if [ "$arch" = "x86_64" ]; then
        (grep -q "Y" /sys/module/kvm_intel/parameters/ept) || \
        say_warn "WARNING: EPT DISABLED. Performance will be affected."
    fi
}

check_vm() {
    if [ $(dmesg | grep -c -i "hypervisor detected") -gt 0 ]; then
        say_warn "WARNING: you are running in a virtual machine." \
    "Firecracker is not well tested under nested virtualization."
    fi
}

cmd_checkenv() {
    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")      { cmd_help; exit 1; } ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
        ;;
        esac
        shift
    done
    PROD_DOC="../docs/prod-host-setup.md"
    QUICKSTART="../docs/getting-started.md#prerequisites"
    say "Checking prerequisites for running Firecracker."
    say "Please check $QUICKSTART in case of any error."
    ensure_kvm_rw
    check_kernver
    check_vm
    say "Checking Host Security Configuration."
    say "Please check $PROD_DOC in case of any error."
    check_KSM
    check_IBPB_IBRS
    check_L1TF
    check_SMT
    check_swap
    check_SSBD
    check_EPT
}

generate_syscall_table_x86_64() {
    path_to_rust_file="$FC_ROOT_DIR/src/seccompiler/src/syscall_table/x86_64.rs"

    echo "$header" > $path_to_rust_file

    # the table for x86_64 is nicely formatted here: linux/arch/x86/entry/syscalls/syscall_64.tbl
    cat linux/arch/x86/entry/syscalls/syscall_64.tbl | grep -v "^#" | grep -v -e '^$' |\
        awk '{print $2,$3,$1}' | grep -v "^x32" |\
        awk '{print "    map.insert(\""$2"\".to_string(), "$3");"}' | sort >> $path_to_rust_file

    echo "$footer" >> $path_to_rust_file

    say "Generated at: $path_to_rust_file"
}

generate_syscall_table_aarch64() {
    path_to_rust_file="$FC_ROOT_DIR/src/seccompiler/src/syscall_table/aarch64.rs"

    # filter for substituting `#define`s that point to other macros;
    # values taken from linux/include/uapi/asm-generic/unistd.h
    replace+='s/__NR3264_fadvise64/223/;'
    replace+='s/__NR3264_fcntl/25/;'
    replace+='s/__NR3264_fstatat/79/;'
    replace+='s/__NR3264_fstatfs/44/;'
    replace+='s/__NR3264_fstat/80/;'
    replace+='s/__NR3264_ftruncate/46/;'
    replace+='s/__NR3264_lseek/62/;'
    replace+='s/__NR3264_sendfile/71/;'
    replace+='s/__NR3264_statfs/43/;'
    replace+='s/__NR3264_truncate/45/;'
    replace+='s/__NR3264_mmap/222/;'

    echo "$header" > $path_to_rust_file

    # run the gcc command in the Docker container (to make sure that we have gcc installed)
    # the aarch64 syscall table is not located in a .tbl file, like x86; we run gcc's
    # pre-processor to extract the numeric constants from header files.
    run_devctr \
        --user "$(id -u):$(id -g)" \
        --workdir "$CTR_KERNEL_DIR" \
        -- \
            gcc -Ilinux/include/uapi -E -dM -D__ARCH_WANT_RENAMEAT\
                -D__BITS_PER_LONG=64\
                linux/arch/arm64/include/uapi/asm/unistd.h |\
                grep "#define __NR_" | grep -v "__NR_syscalls" |\
                grep -v "__NR_arch_specific_syscall" |\
                awk -F '__NR_' '{print $2}' |\
                sed $replace |\
                awk '{ print "    map.insert(\""$1"\".to_string(), "$2");" }' |\
                sort -d >> $path_to_rust_file
    ret=$?

    [ $ret -ne 0 ] && return $ret

    echo "$footer" >> $path_to_rust_file

    say "Generated at: $path_to_rust_file"
}

cmd_generate_syscall_tables() {
    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")      { cmd_help; exit 1;    } ;;
            *)                  { kernel_version="$1"; break; } ;;
        esac
        shift
    done

    validate_kernel_version "$kernel_version"

    kernel_major=v$(echo ${kernel_version} | cut -d . -f 1).x
    kernel_baseurl=https://www.kernel.org/pub/linux/kernel/${kernel_major}
    kernel_archive=linux-${kernel_version}.tar.xz

    ensure_devctr

    # Create the kernel clone directory
    rm -rf "$KERNEL_DIR"
    create_dir "$KERNEL_DIR"
    cd "$KERNEL_DIR"

    say "Fetching linux kernel..."

    # Get sha256 checksum.
    curl -fsSLO ${kernel_baseurl}/sha256sums.asc && \
    kernel_sha256=$(grep ${kernel_archive} sha256sums.asc | cut -d ' ' -f 1)
    # Get kernel archive.
    curl -fsSLO "$kernel_baseurl/$kernel_archive" && \
    # Verify checksum.
    echo "${kernel_sha256}  ${kernel_archive}" | sha256sum -c - && \
    # Decompress the kernel source.
    xz -d "${kernel_archive}" && \
    cat linux-${kernel_version}.tar | tar -x && mv linux-${kernel_version} linux

    ret=$?
    [ $ret -ne 0 ] && return $ret

    # rust file header
    read -r -d '' header << EOM
// Copyright $(date +"%Y") Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

// This file is auto-generated by \`tools/devtool generate_syscall_tables\`.
// Do NOT manually edit!
// Generated at: $(date)
// Kernel version: $kernel_version

use std::collections::HashMap;

pub(crate) fn make_syscall_table(map: &mut HashMap<String, i64>) {
EOM

    # rust file footer
    read -r -d '' footer << EOM
}

EOM

    # generate syscall table for x86_64
    say "Generating table for x86_64..."
    generate_syscall_table_x86_64 $header $footer

    # generate syscall table for aarch64
    say "Generating table for aarch64..."
    generate_syscall_table_aarch64 $header $footer

    ret=$?
    [ $ret -ne 0 ] && return $ret
}

cmd_install() {
    # By default we install release/musl binaries.
    profile="release"
    target="$TARGET_PREFIX""musl"
    install_path="/usr/local/bin"

    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help") { cmd_help; exit 1; } ;;
            "-p"|"--path")
                shift;
                install_path=$1;
                ;;
            "--debug")      { profile="debug";      } ;;
            "--release")    { profile="release";    } ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    # Check that the binaries exist first
    ensure_release_binaries_exist $target $profile

    say "Installing firecracker in $install_path"
    install -m 755 "$( build_fc_bin_path "$target" "$profile")" "$install_path"

    say "Installing jailer in $install_path"
    install -m 755 "$( build_jailer_bin_path "$target" "$profile")" "$install_path"

    say "Installing seccomp in $install_path"
    install -m 755 "$( build_seccomp_bin_path "$target" "$profile")" "$install_path"

    say "Installing rebase-snap in $install_path"
    install -m 755 "$( build_rebase_snap_bin_path "$target" "$profile")" "$install_path"
}

# Build a Firecracker CI compatible kernel image.
# Example: `./tools/devtool build_kernel -c resources/guest_configs/microvm-kernel-arm64-4.14.config`
#
cmd_build_kernel() {
    nprocs=$(getconf _NPROCESSORS_ONLN)

    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")      { cmd_help; exit 1; } ;;
            "-c"|"--config")
                shift
                local KERNEL_CFG="$1"
                ;;
            "-n"|"--nproc")
                shift
                nprocs="$1"
                ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    # It is mandatory to provide a valid config file for building the kernel.
    [ -z "$KERNEL_CFG" ] && die "You need to provide the path to a valid kernel config!"

    arch=$(uname -m)
    if [ "$arch" = "x86_64" ]; then
        target="vmlinux"
        cfg_pattern="x86"
        format="elf"
    elif [ "$arch" = "aarch64" ]; then
        target="Image"
        cfg_pattern="arm64"
        format="pe"
    else
        die "Unsupported architecture!"
    fi

    kernel_dir_host="$FC_BUILD_DIR"/kernel
    kernel_dir_ctr="$CTR_FC_BUILD_DIR"/kernel
    create_dir "$kernel_dir_host"

    # Extract the kernel version from the config file provided as parameter.
    KERNEL_VERSION=$(cat "$KERNEL_CFG" | grep -Po "^# Linux\/$cfg_pattern (([0-9]+.)[0-9]+)" | cut -d ' ' -f 3)
    validate_kernel_version "$KERNEL_VERSION"

    recipe_commit="b551cccc405a73a6d316c0c09dfe0b3e7a73ba3f"
    recipe_url="https://raw.githubusercontent.com/rust-vmm/vmm-reference/$recipe_commit/resources/kernel/make_kernel.sh"
    run_devctr \
        --user "$(id -u):$(id -g)" \
        --workdir "$kernel_dir_ctr" \
        -- /bin/bash -c "curl -LO "$recipe_url" && source make_kernel.sh && extract_kernel_srcs "$KERNEL_VERSION""

    cp "$KERNEL_CFG" "$kernel_dir_host/linux-$KERNEL_VERSION/.config"

    KERNEL_BINARY_NAME="vmlinux-$KERNEL_VERSION-$arch.bin"
    run_devctr \
        --user "$(id -u):$(id -g)" \
        --workdir "$kernel_dir_ctr" \
        -- /bin/bash -c "source make_kernel.sh && make_kernel "$kernel_dir_ctr/linux-$KERNEL_VERSION" $format $target "$nprocs" "$KERNEL_BINARY_NAME""

    say "Successfully built kernel!"
    say "Kernel binary placed in: $kernel_dir_host/linux-$KERNEL_VERSION/$KERNEL_BINARY_NAME"
}

# `./devtool build_rootfs -s 500M`
# Build a rootfs of custom size.
#
cmd_build_rootfs() {
    # Default size for the resulting rootfs image is 300M.
    SIZE="300M"
    FROM_CTR=ubuntu:18.04
    flavour="bionic"
    ROOTFS_DIR=/firecracker/build/rootfs
    MNT_DIR=$ROOTFS_DIR/mnt

    # Parse the command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")      { cmd_help; exit 1; } ;;
            "-s"|"--size")
                shift
                SIZE="$1"
                ;;
            "-p"|"--partuuid")
                shift
                PARTUUID=1
                ;;
            "-m"|"--mount")
                shift
                MNT_DIR="$1"
                ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    rootfs_dir_host="$FC_BUILD_DIR/rootfs"
    rootfs_dir_ctr="$CTR_FC_BUILD_DIR/rootfs"
    resources_dir_ctr="$CTR_FC_ROOT_DIR/resources/tests"
    ROOTFS_NAME="$flavour.rootfs.ext4"
    ROOTFS_PARTUUID_NAME="$flavour.rootfs_with_partuuid.ext4"

    create_dir "$rootfs_dir_host"

    run_devctr \
        --workdir "$CTR_FC_ROOT_DIR" \
        -- /bin/bash -c "gcc -o  $rootfs_dir_ctr/init $resources_dir_ctr/init.c && \
        gcc -o  $rootfs_dir_ctr/fillmem $resources_dir_ctr/fillmem.c && \
        gcc -o  $rootfs_dir_ctr/readmem $resources_dir_ctr/readmem.c"

    img_file="$rootfs_dir_host/$ROOTFS_NAME"

    # Check for pre-existence of rootfs image. See if we are allowed
    # to delete it. If not, abort.
    if [ -f "$img_file" ]; then
        say "Rootfs image $img_file already exists. Do you want us to delete it for you?"
        get_user_confirmation || die "Aborted."
        rm -f "$img_file"
    fi

    truncate -s "$SIZE" "$img_file"
    mkfs.ext4 -F "$img_file"
    docker run --env ROOTFS_DIR="$ROOTFS_DIR" --env MNT_DIR="$MNT_DIR" --env ROOTFS_NAME=$ROOTFS_NAME --env PARTUUID=$PARTUUID --privileged --rm -i -v "$FC_ROOT_DIR:/firecracker"  "$FROM_CTR" bash -s <<'EOF'
img_file="$ROOTFS_DIR/$ROOTFS_NAME"
resource_dir="/firecracker/resources/tests"
mnt_dir="$MNT_DIR"
dirs="bin etc home lib lib64 opt root sbin usr"

source $resource_dir/setup_rootfs.sh

mkdir -p $mnt_dir
echo "Mounting $img_file on $mnt_dir ..."
mount $img_file $mnt_dir

prepare_fc_rootfs "$ROOTFS_DIR" "$resource_dir"

# If we want to create a PARTUUID no need to install all packages (i.e cpuid, iperf3).
if [ -z "$PARTUUID" ]; then
    setup_specialized_rootfs "$ROOTFS_DIR" "$resource_dir"
    dirs="$dirs dev"
fi

# Copy everything we need to the bind-mounted rootfs image file
for d in $dirs; do tar c "/$d" | tar x -C $mnt_dir; done
umount $mnt_dir

EOF

    if [ -n "$PARTUUID" ]; then
        say "Now creating partuuid enabled rootfs..."

        run_devctr \
        --privileged \
        --workdir "$FC_DEVCTR_DIR" \
        -- /bin/bash -c "source "$CTR_FC_ROOT_DIR/resources/tests/setup_rootfs.sh" && create_partuuid_rootfs "$rootfs_dir_ctr/$ROOTFS_NAME" "$rootfs_dir_ctr/$ROOTFS_PARTUUID_NAME"" || die "Error while building partuuid enabled root!"
        label=$(blkid "$rootfs_dir_host/$ROOTFS_PARTUUID_NAME" | cut -d ' ' -f 2 | cut -d '=' -f 2)
        say "Created partuuid enabled disk with partition label: $label!"
        rm "$img_file"
        img_file="$rootfs_dir_host/$ROOTFS_PARTUUID_NAME"
    fi

    say "Successfully built rootfs!"
    say "Rootfs image binary and private key placed in $img_file and $rootfs_dir_host/ssh/id_rsa, respectively!"
}

main() {

    if [ $# = 0 ]; then
    die "No command provided. Please use \`$0 help\` for help."
    fi

    # Parse main command line args.
    #
    while [ $# -gt 0 ]; do
        case "$1" in
            -h|--help)              { cmd_help; exit 1;     } ;;
            -y|--unattended)        { OPT_UNATTENDED=true;  } ;;
            -*)
                die "Unknown arg: $1. Please use \`$0 help\` for help."
            ;;
            *)
                break
            ;;
        esac
        shift
    done

    # $1 is now a command name. Check if it is a valid command and, if so,
    # run it.
    #
    declare -f "cmd_$1" > /dev/null
    ok_or_die "Unknown command: $1. Please use \`$0 help\` for help."

    cmd=cmd_$1
    shift

    # $@ is now a list of command-specific args
    #
    $cmd "$@"
}

main "$@"