Merge pull request #3105 from bobbypage/retry-containerd-pid

Retry containerd task api request when task is in unknown state

container/containerd/client.go

@@ -16,6 +16,7 @@ package containerd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"sync"
@@ -24,6 +25,7 @@ import (
 	containersapi "github.com/containerd/containerd/api/services/containers/v1"
 	tasksapi "github.com/containerd/containerd/api/services/tasks/v1"
 	versionapi "github.com/containerd/containerd/api/services/version/v1"
+	tasktypes "github.com/containerd/containerd/api/types/task"
 	ptypes "github.com/gogo/protobuf/types"
 	"github.com/google/cadvisor/container/containerd/containers"
 	"github.com/google/cadvisor/container/containerd/errdefs"
@@ -44,6 +46,10 @@ type ContainerdClient interface {
 	Version(ctx context.Context) (string, error)
 }
 
+var (
+	ErrTaskIsInUnknownState = errors.New("containerd task is in unknown state") // used when process reported in containerd task is in Unknown State
+)
+
 var once sync.Once
 var ctrdClient ContainerdClient = nil
 
@@ -114,6 +120,9 @@ func (c *client) TaskPid(ctx context.Context, id string) (uint32, error) {
 	if err != nil {
 		return 0, errdefs.FromGRPC(err)
 	}
+	if response.Process.Status == tasktypes.StatusUnknown {
+		return 0, ErrTaskIsInUnknownState
+	}
 	return response.Process.Pid, nil
 }
 

container/containerd/handler.go

@@ -17,6 +17,7 @@ package containerd
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -101,10 +102,14 @@ func newContainerdContainerHandler(
 		if err == nil {
 			break
 		}
-		retry--
-		if !errdefs.IsNotFound(err) || retry == 0 {
+
+		// Retry when task is not created yet or task is in unknown state (likely in process of initializing)
+		isRetriableError := errdefs.IsNotFound(err) || errors.Is(err, ErrTaskIsInUnknownState)
+		if !isRetriableError || retry == 0 {
 			return nil, err
 		}
+
+		retry--
 		time.Sleep(backoff)
 		backoff *= 2
 	}