[CONTINT-3554] Send the Datadog-Entity-ID header, containing either the container-id or the cgroup inode if available

ddtrace/tracer/transport.go

@@ -99,6 +99,9 @@ func newHTTPTransport(url string, client *http.Client) *httpTransport {
 	if cid := internal.ContainerID(); cid != "" {
 		defaultHeaders["Datadog-Container-ID"] = cid
 	}
+	if eid := internal.EntityID(); eid != "" {
+		defaultHeaders["Datadog-Entity-ID"] = eid
+	}
 	return &httpTransport{
 		traceURL: fmt.Sprintf("%s/v0.4/traces", url),
 		statsURL: fmt.Sprintf("%s/v0.6/stats", url),

internal/container.go

@@ -10,12 +10,18 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"regexp"
+	"strings"
+	"syscall"
 )
 
 const (
 	// cgroupPath is the path to the cgroup file where we can find the container id if one exists.
 	cgroupPath = "/proc/self/cgroup"
+
+	// mountsPath is the path to the mounts file where we can find the cgroup v2 mount point.
+	mountsPath = "/proc/mounts"
 )
 
 const (
@@ -33,10 +39,17 @@ var (
 
 	// containerID is the containerID read at init from /proc/self/cgroup
 	containerID string
+
+	// entityID is the entityID to use for the container. It is the `cid-<containerID>` if the container id available,
+	// otherwise the cgroup v2 node inode prefixed with `in-` or an empty string on cgroup v1 or incompatible OS.
+	// It is retrieved by finding cgroup2 mounts in /proc/mounts, finding the cgroup v2 node path in /proc/self/cgroup and
+	// calling stat on mountPath+nodePath to get the inode.
+	entityID string
 )
 
 func init() {
 	containerID = readContainerID(cgroupPath)
+	entityID = readEntityID()
 }
 
 // parseContainerID finds the first container ID reading from r and returns it.
@@ -69,3 +82,85 @@ func readContainerID(fpath string) string {
 func ContainerID() string {
 	return containerID
 }
+
+// parseCgroupV2MountPath parses the cgroup mount path from /proc/mounts
+// It returns an empty string if cgroup v2 is not used
+func parseCgroupV2MountPath(r io.Reader) string {
+	scn := bufio.NewScanner(r)
+	for scn.Scan() {
+		line := scn.Text()
+		// a correct line line should be formatted as `cgroup2 <path> cgroup2 rw,nosuid,nodev,noexec,relatime,nsdelegate 0 0`
+		tokens := strings.Fields(line)
+		if len(tokens) >= 3 {
+			fsType := tokens[2]
+			if fsType == "cgroup2" {
+				return tokens[1]
+			}
+		}
+	}
+	return ""
+}
+
+// parseCgroupV2NodePath parses the cgroup node path from /proc/self/cgroup
+// It returns an empty string if cgroup v2 is not used
+// With respect to https://man7.org/linux/man-pages/man7/cgroups.7.html#top_of_page, in cgroupv2, only 0::<path> should exist
+func parseCgroupV2NodePath(r io.Reader) string {
+	scn := bufio.NewScanner(r)
+	for scn.Scan() {
+		line := scn.Text()
+		// The cgroup node path is the last element of the line starting with "0::"
+		if strings.HasPrefix(line, "0::") {
+			return line[3:]
+		}
+	}
+	return ""
+}
+
+// getCgroupV2Inode returns the cgroup v2 node inode if it exists otherwise an empty string.
+// The inode is prefixed by "in-" and is used by the agent to retrieve the container ID.
+func getCgroupV2Inode(mountsPath, cgroupPath string) string {
+	// Retrieve a cgroup mount point from /proc/mounts
+	f, err := os.Open(mountsPath)
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+	cgroupMountPath := parseCgroupV2MountPath(f)
+	if cgroupMountPath == "" {
+		return ""
+	}
+	// Parse /proc/self/cgroup to retrieve the cgroup node path
+	f, err = os.Open(cgroupPath)
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+	cgroupNodePath := parseCgroupV2NodePath(f)
+	if cgroupNodePath == "" {
+		return ""
+	}
+	// Retrieve the cgroup inode from the cgroup mount and cgroup node path
+	fi, err := os.Stat(path.Clean(cgroupMountPath + cgroupNodePath))
+	if err != nil {
+		return ""
+	}
+	stats, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return ""
+	}
+	return fmt.Sprintf("in-%d", stats.Ino)
+}
+
+// readEntityID attempts to return the cgroup v2 node inode or empty on failure.
+func readEntityID() string {
+	if containerID != "" {
+		return "cid-" + containerID
+	}
+	return getCgroupV2Inode(mountsPath, cgroupPath)
+}
+
+// EntityID attempts to return the container ID or the cgroup v2 node inode if the container ID is not available.
+// The cid is prefixed with `cid-` and the inode with `in-`.
+func EntityID() string {
+	return entityID
+}

internal/container_test.go

@@ -6,12 +6,16 @@
 package internal
 
 import (
+	"fmt"
 	"io"
 	"os"
+	"path"
 	"strings"
+	"syscall"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestReadContainerID(t *testing.T) {
@@ -79,3 +83,194 @@ func TestReadContainerIDFromCgroup(t *testing.T) {
 	actualCID := readContainerID(tmpFile.Name())
 	assert.Equal(t, cid, actualCID)
 }
+
+func TestPrioritizeContainerID(t *testing.T) {
+	// reset cid after test
+	defer func(cid string) { containerID = cid }(containerID)
+
+	containerID = "fakeContainerID"
+	eid := readEntityID()
+	assert.Equal(t, "cid-fakeContainerID", eid)
+}
+
+func TestParseCgroupMountPath(t *testing.T) {
+	// Test cases
+	cases := []struct {
+		name     string
+		content  string
+		expected string
+	}{
+		{
+			name: "cgroup2 and cgroup mounts found",
+			content: `
+none /proc proc rw,nosuid,nodev,noexec,relatime 0 0
+sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0
+cgroup2 /sys/fs/cgroup/cgroup2 cgroup2 rw,nosuid,nodev,noexec,relatime,cpuacct,cpu 0 0
+tmpfs /sys/fs/cgroup tmpfs ro,nosuid,nodev,noexec,mode=755 0 0
+cgroup /sys/fs/cgroup/cpu,cpuacct cgroup rw,nosuid,nodev,noexec,relatime,cpuacct,cpu 0 0
+`,
+			expected: "/sys/fs/cgroup/cgroup2",
+		},
+		{
+			name: "only cgroup found",
+			content: `
+none /proc proc rw,nosuid,nodev,noexec,relatime 0 0
+sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0
+tmpfs /sys/fs/cgroup tmpfs ro,nosuid,nodev,noexec,mode=755 0 0
+cgroup /sys/fs/cgroup/cpu,cpuacct cgroup rw,nosuid,nodev,noexec,relatime,cpuacct,cpu 0 0
+`,
+			expected: "",
+		},
+		{
+			name: "cgroup mount not found",
+			content: `
+none /proc proc rw,nosuid,nodev,noexec,relatime 0 0
+sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0
+tmpfs /dev tmpfs rw,nosuid,size=65536k,mode=755 0 0
+`,
+			expected: "",
+		},
+	}
+
+	// Run test cases
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			reader := strings.NewReader(c.content)
+			result := parseCgroupV2MountPath(reader)
+			require.Equal(t, c.expected, result)
+		})
+	}
+}
+
+func TestParseCgroupNodePath(t *testing.T) {
+	// Test cases
+	cases := []struct {
+		name     string
+		content  string
+		expected string
+	}{
+		{
+			name:     "cgroup2 normal case",
+			content:  `0::/`,
+			expected: "/",
+		},
+		{
+			name: "cgroup node path found",
+			content: `other_line
+12:pids:/docker/abc123
+11:hugetlb:/docker/abc123
+10:net_cls,net_prio:/docker/abc123
+0::/docker/abc123
+`,
+			expected: "/docker/abc123",
+		},
+		{
+			name: "cgroup node path not found",
+			content: `12:pids:/docker/abc123
+11:hugetlb:/docker/abc123
+10:net_cls,net_prio:/docker/abc123
+`,
+			expected: "",
+		},
+	}
+
+	// Run test cases
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			reader := strings.NewReader(c.content)
+			result := parseCgroupV2NodePath(reader)
+			require.Equal(t, c.expected, result)
+		})
+	}
+}
+
+func TestGetCgroupInode(t *testing.T) {
+	tests := []struct {
+		description           string
+		procMountsContent     string
+		cgroupNodeDir         string
+		procSelfCgroupContent string
+		expectedResult        string
+	}{
+		{
+			description:           "default case - matching entry in /proc/self/cgroup and /proc/mounts",
+			procMountsContent:     "cgroup2 %s cgroup2 rw,nosuid,nodev,noexec,relatime,cpu,cpuacct 0 0\n",
+			cgroupNodeDir:         "system.slice/docker-abcdef0123456789abcdef0123456789.scope",
+			procSelfCgroupContent: "0::/system.slice/docker-abcdef0123456789abcdef0123456789.scope\n",
+			expectedResult:        "in-%d", // Will be formatted with inode number
+		},
+		{
+			description:           "should not match cgroup v1",
+			procMountsContent:     "cgroup %s cgroup rw,nosuid,nodev,noexec,relatime,cpu,cpuacct 0 0\n",
+			cgroupNodeDir:         "system.slice/docker-abcdef0123456789abcdef0123456789.scope",
+			procSelfCgroupContent: "0::/system.slice/docker-abcdef0123456789abcdef0123456789.scope\n",
+			expectedResult:        "",
+		},
+		{
+			description: "hybrid cgroup - should match only cgroup2",
+			procMountsContent: `other_line
+cgroup /sys/fs/cgroup/memory cgroup foo,bar 0 0
+cgroup2 %s cgroup2 rw,nosuid,nodev,noexec,relatime,cpu,cpuacct 0 0
+`,
+			cgroupNodeDir:         "system.slice/docker-abcdef0123456789abcdef0123456789.scope",
+			procSelfCgroupContent: "0::/system.slice/docker-abcdef0123456789abcdef0123456789.scope\n",
+			expectedResult:        "in-%d", // Will be formatted with inode number
+		},
+		{
+			description:           "Non-matching entry in /proc/self/cgroup",
+			procMountsContent:     "cgroup2 %s cgroup2 rw,nosuid,nodev,noexec,relatime,cpu,cpuacct 0 0\n",
+			cgroupNodeDir:         "system.slice/nonmatching-scope.scope",
+			procSelfCgroupContent: "0::/system.slice/docker-abcdef0123456789abcdef0123456789.scope\n",
+			expectedResult:        "",
+		},
+		{
+			description:           "No cgroup2 entry in /proc/mounts",
+			procMountsContent:     "tmpfs %s tmpfs rw,nosuid,nodev,noexec,relatime 0 0\n",
+			cgroupNodeDir:         "system.slice/docker-abcdef0123456789abcdef0123456789.scope",
+			procSelfCgroupContent: "0::/system.slice/docker-abcdef0123456789abcdef0123456789.scope\n",
+			expectedResult:        "",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.description, func(t *testing.T) {
+			// Setup
+			cgroupMountPath, err := os.MkdirTemp(os.TempDir(), "sysfscgroup")
+			require.NoError(t, err)
+			defer os.RemoveAll(cgroupMountPath)
+
+			cgroupNodePath := path.Join(cgroupMountPath, tc.cgroupNodeDir)
+			err = os.MkdirAll(cgroupNodePath, 0755)
+			require.NoError(t, err)
+			defer os.RemoveAll(cgroupNodePath)
+
+			stat, err := os.Stat(cgroupNodePath)
+			require.NoError(t, err)
+
+			expectedInode := ""
+			if tc.expectedResult != "" {
+				expectedInode = fmt.Sprintf(tc.expectedResult, stat.Sys().(*syscall.Stat_t).Ino)
+			}
+
+			procMounts, err := os.CreateTemp("", "procmounts")
+			require.NoError(t, err)
+			defer os.Remove(procMounts.Name())
+			_, err = procMounts.WriteString(fmt.Sprintf(tc.procMountsContent, cgroupMountPath))
+			require.NoError(t, err)
+			err = procMounts.Close()
+			require.NoError(t, err)
+
+			procSelfCgroup, err := os.CreateTemp("", "procselfcgroup")
+			require.NoError(t, err)
+			defer os.Remove(procSelfCgroup.Name())
+			_, err = procSelfCgroup.WriteString(tc.procSelfCgroupContent)
+			require.NoError(t, err)
+			err = procSelfCgroup.Close()
+			require.NoError(t, err)
+
+			// Test
+			result := getCgroupV2Inode(procMounts.Name(), procSelfCgroup.Name())
+			require.Equal(t, expectedInode, result)
+		})
+	}
+}

internal/datastreams/transport.go

@@ -68,6 +68,9 @@ func newHTTPTransport(agentURL *url.URL, client *http.Client) *httpTransport {
 	if cid := internal.ContainerID(); cid != "" {
 		defaultHeaders["Datadog-Container-ID"] = cid
 	}
+	if entityID := internal.ContainerID(); entityID != "" {
+		defaultHeaders["Datadog-Entity-ID"] = entityID
+	}
 	url := fmt.Sprintf("%s/v0.1/pipeline_stats", agentURL.String())
 	return &httpTransport{
 		url:     url,

internal/telemetry/client.go

@@ -472,6 +472,7 @@ func (c *client) newRequest(t RequestType) *Request {
 		"DD-Agent-Env":               {c.Env},
 		"DD-Agent-Hostname":          {hostname},
 		"Datadog-Container-ID":       {internal.ContainerID()},
+		"Datadog-Entity-ID":          {internal.EntityID()},
 	}
 	if c.URL == getAgentlessURL() {
 		header.Set("DD-API-KEY", c.APIKey)

profiler/profiler.go

@@ -30,6 +30,7 @@ var (
 	mu             sync.Mutex
 	activeProfiler *profiler
 	containerID    = internal.ContainerID() // replaced in tests
+	entityID       = internal.EntityID()    // replaced in tests
 )
 
 // Start starts the profiler. If the profiler is already running, it will be

profiler/upload.go

@@ -110,6 +110,9 @@ func (p *profiler) doRequest(bat batch) error {
 	if containerID != "" {
 		req.Header.Set("Datadog-Container-ID", containerID)
 	}
+	if entityID != "" {
+		req.Header.Set("Datadog-Entity-ID", entityID)
+	}
 	req.Header.Set("Content-Type", contentType)
 
 	resp, err := p.cfg.httpClient.Do(req)