Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add Bash trap framework basics, refactored stack trace #8527

Merged
merged 2 commits into from
Jun 22, 2016
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion hack/build-base-images.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ set -o pipefail
STARTTIME=$(date +%s)
OS_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${OS_ROOT}/hack/lib/init.sh"
os::log::install_errexit
os::log::stacktrace::install

# Go to the top of the tree.
cd "${OS_ROOT}"
Expand Down
2 changes: 1 addition & 1 deletion hack/build-cross.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ set -o pipefail
STARTTIME=$(date +%s)
OS_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${OS_ROOT}/hack/lib/init.sh"
os::log::install_errexit
os::log::stacktrace::install

platforms=( "${OS_CROSS_COMPILE_PLATFORMS[@]}" )
if [[ -n "${OS_ONLY_BUILD_PLATFORMS-}" ]]; then
Expand Down
2 changes: 1 addition & 1 deletion hack/build-go.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ set -o pipefail
STARTTIME=$(date +%s)
OS_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${OS_ROOT}/hack/lib/init.sh"
os::log::install_errexit
os::log::stacktrace::install

# only works on Linux for now, all other platforms must build binaries themselves
if [[ -z "$@" ]]; then
Expand Down
2 changes: 1 addition & 1 deletion hack/build-images.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ STARTTIME=$(date +%s)
OS_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${OS_ROOT}/hack/lib/init.sh"
source "${OS_ROOT}/contrib/node/install-sdn.sh"
os::log::install_errexit
os::log::stacktrace::install

# Go to the top of the tree.
cd "${OS_ROOT}"
Expand Down
2 changes: 1 addition & 1 deletion hack/build-release.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ set -o pipefail
STARTTIME=$(date +%s)
OS_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${OS_ROOT}/hack/lib/init.sh"
os::log::install_errexit
os::log::stacktrace::install

# Go to the top of the tree.
cd "${OS_ROOT}"
Expand Down
2 changes: 1 addition & 1 deletion hack/cherry-pick.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ set -o pipefail

OS_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${OS_ROOT}/hack/lib/init.sh"
os::log::install_errexit
os::log::stacktrace::install

# Go to the top of the tree.
cd "${OS_ROOT}"
Expand Down
16 changes: 13 additions & 3 deletions hack/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -249,16 +249,27 @@ function os::build::build_static_binaries() {
}
readonly -f os::build::build_static_binaries

# Build binaries targets specified
# Build binary targets specified
#
# Input:
# $@ - targets and go flags. If no targets are set then all binaries targets
# are built.
# OS_BUILD_PLATFORMS - Incoming variable of targets to build for. If unset
# then just the host architecture is built.
function os::build::build_binaries() {
local -a binaries=( "$@" )
# Create a sub-shell so that we don't pollute the outer environment
(
( os::build::internal::build_binaries "${binaries[@]+"${binaries[@]}"}" )
}

# Build binary targets specified. Should always be run in a sub-shell so we don't leak GOBIN
#
# Input:
# $@ - targets and go flags. If no targets are set then all binaries targets
# are built.
# OS_BUILD_PLATFORMS - Incoming variable of targets to build for. If unset
# then just the host architecture is built.
os::build::internal::build_binaries() {
# Check for `go` binary and set ${GOPATH}.
os::build::setup_env

Expand Down Expand Up @@ -325,7 +336,6 @@ function os::build::build_binaries() {
"$(dirname ${test})"
done
done
)
}
readonly -f os::build::build_binaries

Expand Down
2 changes: 1 addition & 1 deletion hack/gen-bootstrap-bindata.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ set -o pipefail
STARTTIME=$(date +%s)
OS_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${OS_ROOT}/hack/lib/init.sh"
os::log::install_errexit
os::log::stacktrace::install

EXAMPLES=examples
OUTPUT_PARENT=${OUTPUT_ROOT:-$OS_ROOT}
Expand Down
2 changes: 1 addition & 1 deletion hack/gen-swagger-docs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ set -o pipefail
OS_ROOT=$(dirname "${BASH_SOURCE}")/..
cd "${OS_ROOT}"
source "${OS_ROOT}/hack/lib/init.sh"
os::log::install_errexit
os::log::stacktrace::install

pushd "${OS_ROOT}/hack/swagger-doc" > /dev/null
gradle gendocs --info
Expand Down
2 changes: 1 addition & 1 deletion hack/install-etcd.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ set -o pipefail

OS_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${OS_ROOT}/hack/lib/init.sh"
os::log::install_errexit
os::log::stacktrace::install

etcd_version=$(go run ${OS_ROOT}/tools/godepversion/godepversion.go ${OS_ROOT}/Godeps/Godeps.json github.com/coreos/etcd/etcdserver)

Expand Down
100 changes: 100 additions & 0 deletions hack/lib/log/stacktrace.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
#!/bin/bash
#
# This library contains an implementation of a stack trace for Bash scripts.

# os::log::stacktrace::install installs the stacktrace as a handler for the ERR signal if one
# has not already been installed and sets `set -o errtrace` in order to propagate the handler
# If the ERR trap is not initialized, installing this plugin will initialize it.
#
# Globals:
# None
# Arguments:
# None
# Returns:
# - export OS_USE_STACKTRACE
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here and elsewhere, you list library-internal state-tracking variables as returns, which seems like it's stretching the definition of a "return". Does it really make sense to list such things as returns?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The Google (? it's been a while) guide asks to list external state changes that a function creates other than the exit code it returns with. Since this is a globally scoped variable, it seems to fit that bill. It's perhaps more useful on longer methods, though.

function os::log::stacktrace::install() {
# setting 'errtrace' propagates our ERR handler to functions, expansions and subshells
set -o errtrace

# OS_USE_STACKTRACE is read by os::util::trap at runtime to request a stacktrace
export OS_USE_STACKTRACE=true

os::util::trap::init_err
}
readonly -f os::log::stacktrace::install

# os::log::stacktrace::print prints the stacktrace and exits with the return code from the script that
# called for a stack trace. This function will always return 0 if it is not handling the signal, and if it
# is handling the signal, this function will always `exit`, not return, the return code it receives as
# its first argument.
#
# Globals:
# - BASH_SOURCE
# - BASH_LINENO
# - FUNCNAME
# Arguments:
# - 1: the return code of the command in the script that generated the ERR signal
# - 2: the last command that ran before handlers were invoked
# - 3: whether or not `set -o errexit` was set in the script that generated the ERR signal
# Returns:
# None
function os::log::stacktrace::print() {
local return_code=$1
local last_command=$2
local errexit_set=${3:-}
set --

# evaulate all of the variables in the last command literal so we have a useful stacktrace
# this will *not* be able to capture positional variables ($1, $@, $*) or variables declared
# in this scope or in the trap handler ($return_code, $last_command, $errexit_set)
# Furthermore, since it is possible that the failure in the last command itself was an unset
# variable, we need to turn off that check to ensure we're not re-triggering it here.
set +o nounset
local -r last_command_with_vars="$( eval "echo \"${last_command}\"" )"
set -o nounset

if [[ "${return_code}" = "0" ]]; then
# we're not supposed to respond when no error has occurred
return 0
fi

if [[ -z "${errexit_set}" ]]; then
# if errexit wasn't set in the shell when the ERR signal was issued, then we can ignore the signal
# as this is not cause for failure
return 0
fi

# iterate backwards through the stack until we leave library files, so we can be sure we start logging
# actual script code and not this handler's call
local stack_begin_index
for (( stack_begin_index = 0; stack_begin_index < ${#BASH_SOURCE[@]}; stack_begin_index++ )); do
if [[ ! "${BASH_SOURCE[${stack_begin_index}]}" =~ hack/lib/(log/stacktrace|util/trap)\.sh ]]; then
break
fi
done

local preamble_finished
local stack_index=1
local i
for (( i = stack_begin_index; i < ${#BASH_SOURCE[@]}; i++ )); do
local bash_source
bash_source="$( os::util::repository_relative_path "${BASH_SOURCE[$i]}" )"
if [[ -z "${preamble_finished:-}" ]]; then
preamble_finished=true
os::log::error "PID ${BASHPID:-$$}: ${bash_source}:${BASH_LINENO[$i-1]}: \`${last_command}\` exited with status ${return_code}." >&2
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Miciah I understand ${BASHPID:-$$} isn't technically a loss-less default, but I'm not particularly interested in making this nice for people using 12-year-old-Bash, so instead of a clear indication that each stacktrace is for a failed process, they'll just always see the parent PID.

os::log::error "Command with variables substituted is: "
os::log::error $'\t'"${last_command_with_vars}"
os::log::info $'\t\t'"Stack Trace: " >&2
os::log::info $'\t\t'" ${stack_index}: ${bash_source}:${BASH_LINENO[$i-1]}: \`${last_command}\`" >&2
else
os::log::info $'\t\t'" ${stack_index}: ${bash_source}:${BASH_LINENO[$i-1]}: ${FUNCNAME[$i-1]}" >&2
fi
stack_index=$(( stack_index + 1 ))
done

# we know we're the privileged handler in this chain, so we can safely exit the shell without
# starving another handler of the privilege of reacting to this signal
os::log::info " Exiting with code ${return_code}." >&2
exit "${return_code}"
}
readonly -f os::log::stacktrace::print
96 changes: 96 additions & 0 deletions hack/lib/util/misc.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
#!/bin/bash
#
# This library holds miscellaneous utility functions. If there begin to be groups of functions in this
# file that share intent or are thematically similar, they should be split into their own files.

# os::util::describe_return_code describes an exit code
#
# Globals:
# - OS_SCRIPT_START_TIME
# Arguments:
# - 1: exit code to describe
# Returns:
# None
function os::util::describe_return_code() {
local return_code=$1

if [[ "${return_code}" = "0" ]]; then
echo -n "[INFO] $0 succeeded "
else
echo -n "[ERROR] $0 failed "
fi

if [[ -n "${OS_SCRIPT_START_TIME:-}" ]]; then
local end_time
end_time="$(date +%s)"
local elapsed_time
elapsed_time="$(( end_time - OS_SCRIPT_START_TIME ))"
local formatted_time
formatted_time="$( os::util::format_seconds "${elapsed_time}" )"
echo "after ${formatted_time}"
else
echo
fi
}
readonly -f os::util::describe_return_code

# os::util::install_describe_return_code installs the return code describer for the EXIT trap
# If the EXIT trap is not initialized, installing this plugin will initialize it.
#
# Globals:
# None
# Arguments:
# None
# Returns:
# - export OS_DESCRIBE_RETURN_CODE
# - export OS_SCRIPT_START_TIME
function os::util::install_describe_return_code() {
export OS_DESCRIBE_RETURN_CODE="true"
OS_SCRIPT_START_TIME="$( date +%s )"; export OS_SCRIPT_START_TIME
os::util::trap::init_exit
}
readonly -f os::util::install_describe_return_code

# os::util::repository_relative_path returns the relative path from the $OS_ROOT directory to the
# given file, if the file is inside of the $OS_ROOT directory. If the file is outside of $OS_ROOT,
# this function will return the absolute path to the file
#
# Globals:
# - OS_ROOT
# Arguments:
# - 1: the path to relativize
# Returns:
# None
function os::util::repository_relative_path() {
local filename=$1

if which realpath >/dev/null 2>&1; then
local trim_path
trim_path="$( realpath "${OS_ROOT}" )/"
filename="$( realpath "${filename}" )"
filename="${filename##*${trim_path}}"
fi

echo "${filename}"
}
readonly -f os::util::repository_relative_path

# os::util::format_seconds formats a duration of time in seconds to print in HHh MMm SSs
#
# Globals:
# None
# Arguments:
# - 1: time in seconds to format
# Return:
# None
function os::util::format_seconds() {
local raw_seconds=$1

local hours minutes seconds
(( hours=raw_seconds/3600 ))
(( minutes=(raw_seconds%3600)/60 ))
(( seconds=raw_seconds%60 ))

printf '%02dh %02dm %02ds' "${hours}" "${minutes}" "${seconds}"
}
readonly -f os::util::format_seconds
Loading