From 409edd42a5b2f7bc5ad8529880ae9653601d6d4d Mon Sep 17 00:00:00 2001 From: Henner Zeller Date: Wed, 27 Oct 2021 01:15:54 -0700 Subject: [PATCH 1/2] Make smoke-test output easier to read. * Make sure the output of the run is not cluttered with bash outputs of '..progname received signal .. aborted' * Highlight some outputs with bold and red for quicker visual navigation. * Prepare for the time we might want to use ASAN: set an environment variable for it to output a useful exit code. Signed-off-by: Henner Zeller --- .github/bin/smoke-test.sh | 45 ++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/.github/bin/smoke-test.sh b/.github/bin/smoke-test.sh index cb7c7cf86..f1d00405d 100755 --- a/.github/bin/smoke-test.sh +++ b/.github/bin/smoke-test.sh @@ -28,19 +28,33 @@ # crash Verible, we're good. ### -set -u +# Suppress '... aborted' messages bash would print when a tool crashes. +# Comment out to see syntax errors in bash while working on script. +exec 2>/dev/null -readonly BASE_TEST_DIR=/tmp/test/verible-smoke-test +set -u # Be strict: only allow using a variable after it is assigned + +TMPDIR="${TMPDIR:-/tmp}" +readonly BASE_TEST_DIR=${TMPDIR}/test/verible-smoke-test + +# Some terminal codes to highlight +readonly TERM_RED="\033[1;31m" +readonly TERM_BOLD="\033[1m" +readonly TERM_RESET="\033[0m" # Build all the installable binaries that we're going to use below. -# Ideally, we'd use --config=asan, but unfortunately, asan exits with exit -# code 1, so we can't distinguish from regular 1 exit :( -# -# (TODO: maybe consider asan and capture the output and grep for -# "AddressSanitizer" string if we get an exit code of 1.) +# TODO: Consider running with +# * address sanitizer (best result with libc++). CAVE: slow +# * running with -D_GLIBCXX_DEBUG (also see #1056) bazel build :install-binaries + readonly BINARY_BASE_DIR=bazel-bin/verilog/tools +# In case the binaries are run with ASAN: +# By default, failing asan binaries exit with exit code 1. +# Let's change it to something that we can distinguish from 'normal operation' +export ASAN_OPTIONS="exitcode=140" + readonly VERIBLE_TOOLS_TO_RUN="syntax/verible-verilog-syntax \ lint/verible-verilog-lint \ formatter/verible-verilog-format \ @@ -116,12 +130,13 @@ KnownProjectToolIssue[project:opentitan]="#917 #1002 #1003" # First parameter : project name # Second parameter: name of file containing a list of {System}Verilog files function run_smoke_test() { - local PROJECT_FILE_LIST=/tmp/filelist.$$.list + local PROJECT_FILE_LIST=${TMPDIR}/filelist.$$.list + local TOOL_OUT=${TMPDIR}/tool.$$.out local PROJECT_NAME=$1 local FILELIST=$2 local result=0 - echo "== Running verible on $PROJECT_NAME with $(wc -l < ${FILELIST}) files" + echo -e "== Running verible on ${TERM_BOLD}${PROJECT_NAME}${TERM_RESET} with $(wc -l < ${FILELIST}) files ==" for tool in $VERIBLE_TOOLS_TO_RUN ; do printf "%-20s %-32s\n" ${PROJECT_NAME} ${tool} @@ -138,7 +153,7 @@ function run_smoke_test() { file_param=${single_file} fi - ${BINARY_BASE_DIR}/${tool} ${EXTRA_PARAM} ${file_param} >/dev/null 2>&1 + ${BINARY_BASE_DIR}/${tool} ${EXTRA_PARAM} ${file_param} > ${TOOL_OUT} 2>&1 local EXIT_CODE=$? # A regular error exit code we accept as normal operation of the tool if @@ -148,7 +163,10 @@ function run_smoke_test() { # and >= 128 (128 + signal-number) # https://www.gnu.org/software/bash/manual/html_node/Exit-Status.html if [ $EXIT_CODE -ge 126 ]; then - echo ${BINARY_BASE_DIR}/${tool} ${EXTRA_PARAM} ${file_param} + if [[ $tool == *-project ]]; then + file_param="<(echo $single_file)" # make easy to reproduce + fi + echo -e "${TERM_RED}${BINARY_BASE_DIR}/${tool} ${EXTRA_PARAM} ${file_param} ${TERM_RESET}" waive_file_key="${short_tool_name}:${single_file}" waive_project_key="${short_tool_name}:${PROJECT_NAME}" if [[ -v KnownIssue[${waive_file_key}] ]]; then @@ -159,14 +177,15 @@ function run_smoke_test() { echo " --> Known issue 🐞🐞 possibly one of ${KnownProjectToolIssue[${waive_project_key}]}" else # This is an so far unknown issue - echo "😱 ${single_file}: crash exit code $EXIT_CODE for $tool" + echo "::error ::😱 ${single_file}: crash exit code $EXIT_CODE for $tool" + head -10 ${TOOL_OUT} # Might be useful in this case result=$((${result} + 1)) fi fi done < ${FILELIST} done # for tool - rm -f ${PROJECT_FILE_LIST} + rm -f ${PROJECT_FILE_LIST} ${TOOL_OUT} return ${result} } From 07d9fe8e62d2b04108e3b03e21d29305e9335848 Mon Sep 17 00:00:00 2001 From: Henner Zeller Date: Thu, 28 Oct 2021 01:27:05 -0700 Subject: [PATCH 2/2] Address review comments. Signed-off-by: Henner Zeller --- .github/bin/smoke-test.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/bin/smoke-test.sh b/.github/bin/smoke-test.sh index f1d00405d..fbb8bbdb2 100755 --- a/.github/bin/smoke-test.sh +++ b/.github/bin/smoke-test.sh @@ -38,9 +38,9 @@ TMPDIR="${TMPDIR:-/tmp}" readonly BASE_TEST_DIR=${TMPDIR}/test/verible-smoke-test # Some terminal codes to highlight -readonly TERM_RED="\033[1;31m" -readonly TERM_BOLD="\033[1m" -readonly TERM_RESET="\033[0m" +readonly TERM_RED=$'\033[1;31m' +readonly TERM_BOLD=$'\033[1m' +readonly TERM_RESET=$'\033[0m' # Build all the installable binaries that we're going to use below. # TODO: Consider running with @@ -136,7 +136,7 @@ function run_smoke_test() { local FILELIST=$2 local result=0 - echo -e "== Running verible on ${TERM_BOLD}${PROJECT_NAME}${TERM_RESET} with $(wc -l < ${FILELIST}) files ==" + echo "::group::== Running verible on ${TERM_BOLD}${PROJECT_NAME}${TERM_RESET} with $(wc -l < ${FILELIST}) files ==" for tool in $VERIBLE_TOOLS_TO_RUN ; do printf "%-20s %-32s\n" ${PROJECT_NAME} ${tool} @@ -166,7 +166,7 @@ function run_smoke_test() { if [[ $tool == *-project ]]; then file_param="<(echo $single_file)" # make easy to reproduce fi - echo -e "${TERM_RED}${BINARY_BASE_DIR}/${tool} ${EXTRA_PARAM} ${file_param} ${TERM_RESET}" + echo "${TERM_RED}${BINARY_BASE_DIR}/${tool} ${EXTRA_PARAM} ${file_param} ${TERM_RESET}" waive_file_key="${short_tool_name}:${single_file}" waive_project_key="${short_tool_name}:${PROJECT_NAME}" if [[ -v KnownIssue[${waive_file_key}] ]]; then