Skip to content
This repository has been archived by the owner on Jun 6, 2024. It is now read-only.

Commit

Permalink
runtime fix
Browse files Browse the repository at this point in the history
  • Loading branch information
Binyang2014 committed Sep 6, 2019
1 parent 83c344d commit 2061dd7
Show file tree
Hide file tree
Showing 4 changed files with 15 additions and 14 deletions.
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package main

import (
"fmt"
"os"
"strconv"

Expand All @@ -16,12 +17,13 @@ func init() {
log = logger.NewLogger()
}

// This function will extract error summary to the specific file and print the exit code
func main() {
argsWithoutProg := os.Args[1:]

if len(argsWithoutProg) < 5 {
log.Error("args is not valid")
os.Exit(255)
os.Exit(defaultExitCode)
}

userExitCode, err := strconv.Atoi(argsWithoutProg[0])
Expand Down Expand Up @@ -75,5 +77,5 @@ DUMP_RESULT:
os.Exit(defaultExitCode)
}
log.Info("finish generating the exit summary")
os.Exit(exitInfo.Exitcode)
fmt.Print(exitInfo.Exitcode)
}
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ func TestGenerateExitInfoWithAndLogic(t *testing.T) {
assert.Equal(t, exitInfo.Exitcode, 15)
assert.Equal(t, exitInfo.OriginUserExitCode, 1)
assert.Equal(t, *exitInfo.Reason, "Tensorboad not startup correctly")
// assert.Equal(t, *exitInfo.Solution, "Please try again or connect admin")
assert.Equal(t, exitInfo.Solution[0], "Please try again or connect admin")
assert.Equal(t, *exitInfo.MatchedPlatformLogString, "Failed to start tensorboard")
assert.Equal(t, *exitInfo.MatchedUserLogString, "connect tensorboard failed")
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
"gopkg.in/yaml.v2"
)

// User zero to present undefined exitCode
// Use zero to present undefined exitCode in error spec
const undefinedExitCode = 0

type runtimeErrorSpec struct {
Expand Down Expand Up @@ -152,8 +152,8 @@ func (a *ErrorAggregator) GenerateExitInfo(userExitCode int) (*RuntimeExitInfo,
exitInfo.Exitcode = a.defaulExitCode
exitInfo.OriginUserExitCode = userExitCode
exitInfo.ErrorLogs = new(ErrorLogs)
exitInfo.ErrorLogs.Platform = ptrString(strings.Join(a.extractNlineTailLog(platformLog, a.maxRuntimeLogLines), "\n"))
exitInfo.ErrorLogs.User = ptrString(strings.Join(a.extractNlineTailLog(userLog, a.maxUserLogLines), "\n"))
exitInfo.ErrorLogs.Platform = ptrString(strings.Join(a.extractNlinesTailLog(platformLog, a.maxRuntimeLogLines), "\n"))
exitInfo.ErrorLogs.User = ptrString(strings.Join(a.extractNlinesTailLog(userLog, a.maxUserLogLines), "\n"))
}

return &exitInfo, nil
Expand Down Expand Up @@ -204,7 +204,7 @@ func (a *ErrorAggregator) mergeLogs(lhs []string, rhs []string, matchString stri
return res
}

func (a *ErrorAggregator) extractNlineTailLog(conent []byte, maxLogLines int) []string {
func (a *ErrorAggregator) extractNlinesTailLog(conent []byte, maxLogLines int) []string {
var start int
if logLen := len(conent); logLen > a.maxAggregateLogSize {
start = logLen - a.maxAggregateLogSize
Expand All @@ -222,7 +222,7 @@ func (a *ErrorAggregator) extractMatchLog(loc []int, content []byte, maxLogLines
// use simple rules. will extract 2 lines above the match pattern and other lines below the match pattern
if loc == nil {
// fallback to extract tail logs
return a.extractNlineTailLog(content, maxLogLines), nil
return a.extractNlinesTailLog(content, maxLogLines), nil
}

if len(loc) < 2 {
Expand Down
11 changes: 5 additions & 6 deletions src/kube-runtime/src/runtime
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,11 @@ TERMINATE_MESSAGE_PATH=/tmp/pai-termination-log
function exit_handler()
{
USER_EXIT_CODE=$?
# genergate aggregate errors
${PAI_RUNTIME_DIR}/exithandler ${USER_EXIT_CODE} \
${PAI_LOG_DIR}/user.pai.all \
${PAI_LOG_DIR}/runtime.pai.stderr \
${TERMINATE_MESSAGE_PATH} ${PATTERN_FILE} >> ${PAI_LOG_DIR}/runtime.pai.stderr 2>&1
CONTAINER_EXIT_CODE=$?
# genergate aggregated exit info
CONTAINER_EXIT_CODE=$(${PAI_RUNTIME_DIR}/exithandler ${USER_EXIT_CODE} \
${PAI_LOG_DIR}/user.pai.all \
${PAI_LOG_DIR}/runtime.pai.stderr \
${TERMINATE_MESSAGE_PATH} ${PATTERN_FILE} 2> ${PAI_LOG_DIR}/runtime.pai.stderr)
cp ${TERMINATE_MESSAGE_PATH} ${PAI_LOG_DIR}

exit ${CONTAINER_EXIT_CODE}
Expand Down

0 comments on commit 2061dd7

Please sign in to comment.