grenade.sh

#!/usr/bin/env bash

# ``grenade.sh`` is an OpenStack upgrade test harness to exercise the
# OpenStack upgrade process.  It uses DevStack to perform the initial
# OpenStack install.

# Grenade assumes it is running on the system that will be hosting the
# upgrade processes

# ``grenade.sh [-b] [-t] [-s stop-label] [-q]``
#
# ``-b``    Run only the base part
# ``-t``    Run only the target part (assumes a base run is in place)
# ``-q``    Quiet mode
# ``-s stop-label`` is the name of the step after which the script will stop.
# This is useful for debugging upgrades.



# ``GRENADE_DIR`` is set once by the top level grenade.sh and exported
# so that all subsequent scripts can find their way back to the
# grenade root directory. No other scripts should set this variable
export GRENADE_DIR=$(cd $(dirname "$0") && pwd)

# Source the bootstrapping facilities
#
# Grenade attempts to reuse as much content from devstack on the
# target side as possible, but we need enough of our own code to get
# there.
#
# ``grenaderc`` is a set of X=Y declarations that don't need *any* of
# the devstack functions to work
#
# ``inc/bootstrap`` is the most minimal amount of functions that
# grenade needs to get going. This includes things like echo
# functions, trueorfalse, and the functions related to git cloning, so
# that we can get our devstack trees.
export DSTOOLS_VERSION=${DSTOOLS_VERSION:-0.4.0}
source $GRENADE_DIR/grenaderc
source $GRENADE_DIR/inc/bootstrap

while getopts bqs:t c; do
    case $c in
        b)
            RUN_TARGET=False
            ;;
        q)
            VERBOSE=False
            ;;
        s)
            STOP=$2
            ;;
        t)
            RUN_BASE=False
            ;;
    esac
done
shift `expr $OPTIND - 1`


# Create all the base directory structures needed for the rest of the
# environment to run.
#
# This will give you an STACK_ROOT tree that you expect, and that your
# normal user owns for follow on activities.
sudo mkdir -p $BASE_RELEASE_DIR $TARGET_RELEASE_DIR
sudo chown -R `whoami` $STACK_ROOT

# Logging
# =======

# TODO(sdague): should this extract into ``inc/bootstrap``?
#
# Set up logging
# Set ``LOGFILE`` to turn on logging
# Append '.xxxxxxxx' to the given name to maintain history
# where 'xxxxxxxx' is a representation of the date the file was created
if [[ -n "$LOGFILE" ]]; then
    LOGDAYS=${LOGDAYS:-7}
    TIMESTAMP_FORMAT=${TIMESTAMP_FORMAT:-"%F-%H%M%S"}
    CURRENT_LOG_TIME=$(date "+$TIMESTAMP_FORMAT")

    # First clean up old log files.  Use the user-specified ``LOGFILE``
    # as the template to search for, appending '.*' to match the date
    # we added on earlier runs.
    LOGDIR=$(dirname "$LOGFILE")
    LOGNAME=$(basename "$LOGFILE")
    echo "Creating $LOGDIR...."

    sudo mkdir -p $LOGDIR
    sudo chown -R `whoami` $LOGDIR
    find $LOGDIR -maxdepth 1 -name $LOGNAME.\* -mtime +$LOGDAYS -exec rm {} \;
    LOGFILE=$LOGFILE.${CURRENT_LOG_TIME}
    SUMFILE=$LOGFILE.${CURRENT_LOG_TIME}.summary

    # Redirect output according to config
    # Copy stdout to fd 3
    exec 3>&1
    if [[ "$VERBOSE" == "True" ]]; then
        echo "Running in verbose mode:"
        echo "  Full logs found at => ${LOGFILE}"
        echo "  Summary logs at => ${SUMFILE}"
        # Redirect stdout/stderr to tee to write the log file
        exec 1> >( ./tools/outfilter.py -v -o "${LOGFILE}" ) 2>&1
        # Set up a second fd for output
        exec 6> >( ./tools/outfilter.py -o "${SUMFILE}" )
    else
        echo "Running in summary mode:"
        echo "  Full logs found at => ${LOGFILE}"
        echo "  Summary logs at => ${SUMFILE}"
        # Set fd 1 and 2 to primary logfile
        exec 1> >( ./tools/outfilter.py -o "${LOGFILE}") 2>&1
        # Set fd 6 to summary logfile and stdout
        exec 6> >( ./tools/outfilter.py -v -o "${SUMFILE}" >&3)
    fi

    echo_summary "grenade.sh log $LOGFILE"
    # Specified logfile name always links to the most recent log
    ln -sf $LOGFILE $LOGDIR/$LOGNAME
    ln -sf $SUMFILE $LOGDIR/$LOGNAME.summary
else
    # Set up output redirection without log files
    # Copy stdout to fd 3
    exec 3>&1
    if [[ "$VERBOSE" != "yes" ]]; then
        # Throw away stdout and stderr
        exec 1>/dev/null 2>&1
    fi
    # Always send summary fd to original stdout
    exec 6> >( ./tools/outfilter.py -v -o "${SUMFILE}" >&3)
fi

# Set up logging of screen windows
# Set ``SCREEN_LOGDIR`` to turn on logging of screen windows to the
# directory specified in ``SCREEN_LOGDIR``, we will log to the file
# ``screen-$SERVICE_NAME-$TIMESTAMP.log`` in that dir and have a link
# ``screen-$SERVICE_NAME.log`` to the latest log file.
# Logs are kept for as long specified in ``LOGDAYS``.
if [[ -n "$SCREEN_LOGDIR" ]]; then

    # We make sure the directory is created.
    if [[ -d "$SCREEN_LOGDIR" ]]; then
        # We cleanup the old logs
        find $SCREEN_LOGDIR -maxdepth 1 -name screen-\*.log -mtime +$LOGDAYS -exec rm {} \;
    else
        sudo mkdir -p $SCREEN_LOGDIR
        sudo chown -R `whoami` $SCREEN_LOGDIR
    fi
fi


# This script exits on an error so that errors don't compound and you see
# only the first error that occurred.
set -o errexit

# Print the commands being run so that we can see the command that triggers
# an error.  It is also useful for following allowing as the install occurs.
set -o xtrace


# Devstack Phase 2 initialization
# ===============================
#
# We now have enough infrastructure in grenade.sh to go and get *both*
# SOURCE and TARGET devstack trees. After which point we 'pivot' onto
# the TARGET devstack functions file, then source the rest of the
# grenade settings. This should let us run the bulk of grenade.

if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then
    # Get both devstack trees, so that BASE_DEVSTACK_DIR, and
    # TARGET_DEVSTACK_DIR are now fully populated.
    fetch_devstacks
else
    devstacks_setup_environment
fi

# Source the rest of the Grenade functions. For convenience
# ``$GRENADE_DIR/functions`` implicitly sources
# ``$TARGET_DEVSTACK_DIR/functions``. So this line can't happen until
# we have the devstacks pulled down.
source $GRENADE_DIR/functions

# We now have the 'short_source' function available, so setup our PS4 variable
export PS4='+ $(short_source):   '

# Many calls inside of devstack functions reference $TOP_DIR, which is
# the root of devstack. We export $TOP_DIR to all child processes here
# to be the TARGET_DEVSTACK_DIR.
#
# If you want a script to use functions off of BASE_DEVSTACK_DIR (like
# the shutdown phase) you *must* explicitly reset TOP_DIR in those
# scripts.
export TOP_DIR=$TARGET_DEVSTACK_DIR

# Install 'Base' Build of OpenStack
# =================================

source $TARGET_DEVSTACK_DIR/inc/meta-config
# Oh the complexity of bootstrapping. We need to populate enabled
# services from what devstack-gate might have set it as.
extract_localrc_section $BASE_DEVSTACK_DIR/local.conf \
                        $BASE_DEVSTACK_DIR/localrc \
                        $BASE_DEVSTACK_DIR/.localrc.auto

# Collect the ENABLED_SERVICES from the base directory, this is what
# we are starting with.
ENABLED_SERVICES=$(set +o xtrace &&
                   source $BASE_DEVSTACK_DIR/stackrc &&
                   echo $ENABLED_SERVICES)

# Fetch all the grenade plugins which were registered in ``pluginrc``
# via the ``enable_grenade_plugin`` stanza. This must be done before
# settings are loaded, but has to be this late to give access to all
# the devstack functions.
fetch_grenade_plugins

# Load the ``settings`` files for all the in tree ``projects/``. This
# registers all the projects in order that we're going to be upgrading
# when the time is right.
load_settings

if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then
    # Nova should use singleconductor as Grenade doesn't
    # setup multi-cell rabbit for now
    devstack_localrc base "CELLSV2_SETUP=singleconductor"
    devstack_localrc target "CELLSV2_SETUP=singleconductor"
fi

# And ensure that we setup the target localrc.auto, because stack.sh
# isn't run there. This has to be run after load_settings because
# plugins might change the service list during this phase.

extract_localrc_section $TARGET_DEVSTACK_DIR/local.conf \
                        $TARGET_DEVSTACK_DIR/localrc \
                        $TARGET_DEVSTACK_DIR/.localrc.auto

# Run the base install of the environment
if [[ "$RUN_BASE" == "True" ]]; then

    # Initialize grenade_db local storage, used for resource tracking
    init_grenade_db

    if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then
        echo_summary "Running base stack.sh"
        cd $BASE_DEVSTACK_DIR
        GIT_BASE=$GIT_BASE ./stack.sh
        stop $STOP stack.sh 10

        echo_summary "Running post-stack.sh"
        if [[ -e $GRENADE_DIR/post-stack.sh ]]; then
            cd $GRENADE_DIR

            # By the time we get here the sub nodes are already setup with localrc files
            # as those are transferred in devstack-gate even before grenade.sh is called
            # We hack the ./post-stack.sh to inject what we need. if we don't set
            # CELLSV2_SETUP, the default devstack assumes "superconductor" and fails.
            export SUB_NODE_ENV_VARS="CELLSV2_SETUP=singleconductor"
            sed -i 's/stdbuf/$SUB_NODE_ENV_VARS stdbuf/' ./post-stack.sh
            cat ./post-stack.sh

            ./post-stack.sh
            stop $STOP post-stack.sh 15
            echo_summary "Completed post-stack.sh"
        fi
    fi

    # Cache downloaded instances
    # --------------------------

    echo_summary "Caching downloaded images"
    mkdir -p $BASE_RELEASE_DIR/images
    echo "Images: $IMAGE_URLS"
    for image_url in ${IMAGE_URLS//,/ }; do
        IMAGE_FNAME=`basename "$image_url"`
        if [[ -r $BASE_DEVSTACK_DIR/files/$IMAGE_FNAME ]]; then
            rsync -a $BASE_DEVSTACK_DIR/files/$IMAGE_FNAME $BASE_RELEASE_DIR/images
        fi
    done
    # NOTE(sileht): If glance is not enabled the directory cannot exists.
    if [[ -d $BASE_DEVSTACK_DIR/files/images ]] ; then
        rsync -a $BASE_DEVSTACK_DIR/files/images/ $BASE_RELEASE_DIR/images
    fi
    stop $STOP image-cache 20

    # Operation
    # ---------

    # Validate the install
    if [[ "$BASE_RUN_SMOKE" == "True" ]]; then
        echo_summary "Running base smoke test"
        run_tempest $BASE_RELEASE_DIR
        if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then
            # once we are done, copy our created artifacts to the target
            if [[ -e $TARGET_RELEASE_DIR/tempest ]]; then
                rsync -a $BASE_RELEASE_DIR/tempest/.tox/ $TARGET_RELEASE_DIR/tempest/.tox/
                if [[ -d $BASE_RELEASE_DIR/tempest/.testrepository ]]; then
                    rsync -a $BASE_RELEASE_DIR/tempest/.testrepository/ $TARGET_RELEASE_DIR/tempest/.testrepository/
                elif [[ -d $BASE_RELEASE_DIR/tempest/.stestr ]]; then
                    rsync -a $BASE_RELEASE_DIR/tempest/.stestr/ $TARGET_RELEASE_DIR/tempest/.stestr/
                fi
            fi
        fi
    fi
    stop $STOP base-smoke 110

    # Early Create resources, used largely for network setup
    resources early_create

    # Create resources
    resources create

    # Verify the resources were created
    resources verify pre-upgrade

    # Save some stuff before we shut that whole thing down
    echo_summary "Saving current state information"
    $GRENADE_DIR/save-state
    stop $STOP save-state 130

    # Shut down running code
    echo_summary "Shutting down all services on base devstack..."
    shutdown_services

    # Verify the resources still exist after the shutdown
    resources verify_noapi pre-upgrade
fi


# Upgrades
# ========

if [[ "$RUN_TARGET" == "True" ]]; then
    # Clone all devstack plugins on the new side, because we're not
    # running a full stack.sh
    fetch_plugins

    # Get target devstack tree ready for services to be run from it,
    # including trying to reuse any existing files we pulled during
    # the base run.
    echo_summary "Preparing the target devstack environment"
    $GRENADE_DIR/prep-target

    # upgrade all the projects in order
    echo "Upgrade projects: $UPGRADE_PROJECTS"
    for project in $UPGRADE_PROJECTS; do
        echo "Upgrading project: $project"
        upgrade_service $project
    done

    # Upgrade Tempest
    if [[ "$ENABLE_TEMPEST" == "True" ]]; then
        echo_summary "Running upgrade-tempest"
        $GRENADE_DIR/upgrade-tempest || die $LINENO "Failure in upgrade-tempest"
        stop $STOP upgrade-tempest 290
    fi

    # Upgrade Tests
    # =============

    # Verify the resources still exist after the upgrade
    resources verify post-upgrade

    # Validate the upgrade
    # This is used for testing grenade locally, and should not be used in the
    # gate. Instead, grenade.sh runs smoke tests on the old cloud above, but
    # smoke tests are run on the upgraded cloud by the gate script after
    # grenade.sh has finished. If this option is enabled in the gate, tempest
    # will be run twice against the new cloud.
    if [[ "$TARGET_RUN_SMOKE" == "True" ]]; then
        echo_summary "Running tempest smoke tests"
        run_tempest $TARGET_RELEASE_DIR
        stop $STOP run-smoke 330
    fi

    # Save databases
    # --------------
    save_data $TARGET_RELEASE $TARGET_DEVSTACK_DIR

    # Cleanup the resources
    resources destroy

fi


# Fin
# ===

echo_summary "Grenade has completed the pre-programmed upgrade scripts."
# Indicate how long this took to run (bash maintained variable ``SECONDS``)
echo_summary "grenade.sh completed in $SECONDS seconds."