diff --git a/container/containerd/client.go b/container/containerd/client.go index 02fd8ce782..c4921a5b7a 100644 --- a/container/containerd/client.go +++ b/container/containerd/client.go @@ -16,6 +16,7 @@ package containerd import ( "context" + "errors" "fmt" "net" "sync" @@ -24,6 +25,7 @@ import ( containersapi "github.com/containerd/containerd/api/services/containers/v1" tasksapi "github.com/containerd/containerd/api/services/tasks/v1" versionapi "github.com/containerd/containerd/api/services/version/v1" + tasktypes "github.com/containerd/containerd/api/types/task" ptypes "github.com/gogo/protobuf/types" "github.com/google/cadvisor/container/containerd/containers" "github.com/google/cadvisor/container/containerd/errdefs" @@ -44,6 +46,10 @@ type ContainerdClient interface { Version(ctx context.Context) (string, error) } +var ( + ErrTaskIsInUnknownState = errors.New("containerd task is in unknown state") // used when process reported in containerd task is in Unknown State +) + var once sync.Once var ctrdClient ContainerdClient = nil @@ -114,6 +120,9 @@ func (c *client) TaskPid(ctx context.Context, id string) (uint32, error) { if err != nil { return 0, errdefs.FromGRPC(err) } + if response.Process.Status == tasktypes.StatusUnknown { + return 0, ErrTaskIsInUnknownState + } return response.Process.Pid, nil } diff --git a/container/containerd/handler.go b/container/containerd/handler.go index 0fc8803542..848550ec6e 100644 --- a/container/containerd/handler.go +++ b/container/containerd/handler.go @@ -17,6 +17,7 @@ package containerd import ( "encoding/json" + "errors" "fmt" "strings" "time" @@ -101,10 +102,14 @@ func newContainerdContainerHandler( if err == nil { break } - retry-- - if !errdefs.IsNotFound(err) || retry == 0 { + + // Retry when task is not created yet or task is in unknown state (likely in process of initializing) + isRetriableError := errdefs.IsNotFound(err) || errors.Is(err, ErrTaskIsInUnknownState) + if !isRetriableError || retry == 0 { return nil, err } + + retry-- time.Sleep(backoff) backoff *= 2 }