[chore] Add extension/cgroupruntime integration tests

.chloggen/cgroup_integration.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: 'enhancement'
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: 'cgroupruntimeextension'
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add cgroup integration tests
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [36545]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

Makefile.Common

@@ -30,8 +30,10 @@ GO_BUILD_LDFLAGS="-s -w"
 GOTEST_TIMEOUT?= 600s
 GOTEST_OPT?= -race -timeout $(GOTEST_TIMEOUT) -parallel 4 --tags=$(GO_BUILD_TAGS)
 GOTEST_INTEGRATION_OPT?= -race -timeout 360s -parallel 4
+GOTEST_INTEGRATION_OPT_SUDO= $(GOTEST_INTEGRATION_OPT) -exec sudo
 GOTEST_OPT_WITH_COVERAGE = $(GOTEST_OPT) -coverprofile=coverage.txt -covermode=atomic
 GOTEST_OPT_WITH_INTEGRATION=$(GOTEST_INTEGRATION_OPT) -tags=integration,$(GO_BUILD_TAGS)
+GOTEST_OPT_WITH_INTEGRATION_SUDO=$(GOTEST_INTEGRATION_OPT_SUDO) -tags=integration,sudo,$(GO_BUILD_TAGS)
 GOTEST_OPT_WITH_INTEGRATION_COVERAGE=$(GOTEST_OPT_WITH_INTEGRATION) -coverprofile=integration-coverage.txt -covermode=atomic
 GOCMD?= go
 GOOS=$(shell $(GOCMD) env GOOS)
@@ -152,12 +154,13 @@ endif
 runbuilttest: $(GOTESTSUM)
 ifneq (,$(wildcard ./builtunitetest.test))
 	$(GOTESTSUM) --raw-command -- $(GOCMD) tool test2json -p "./..." -t ./builtunitetest.test -test.v -test.failfast -test.timeout $(GOTEST_TIMEOUT)
-endif	
+endif
 
 .PHONY: mod-integration-test
 mod-integration-test: $(GOTESTSUM)
 	@echo "running $(GOCMD) integration test ./... in `pwd`"
 	$(GOTESTSUM) $(GOTESTSUM_OPT) --packages="./..." -- $(GOTEST_OPT_WITH_INTEGRATION)
+	$(GOTESTSUM) $(GOTESTSUM_OPT) --packages="./..." -- $(GOTEST_OPT_WITH_INTEGRATION_ROOT)
 	@if [ -e integration-coverage.txt ]; then \
 		$(GOCMD) tool cover -html=integration-coverage.txt -o integration-coverage.html; \
 	fi

extension/cgroupruntimeextension/go.mod

@@ -4,6 +4,7 @@ go 1.22.0
 
 require (
 	github.com/KimMachineGun/automemlimit v0.6.1
+	github.com/containerd/cgroups/v3 v3.0.2
 	github.com/stretchr/testify v1.10.0
 	go.opentelemetry.io/collector/component v0.114.1-0.20241202231142-b9ff1bc54c99
 	go.opentelemetry.io/collector/component/componenttest v0.114.1-0.20241202231142-b9ff1bc54c99
@@ -17,25 +18,24 @@ require (
 
 require (
 	github.com/cilium/ebpf v0.9.1 // indirect
-	github.com/containerd/cgroups/v3 v3.0.1 // indirect
-	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/docker/go-units v0.4.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
-	github.com/godbus/dbus/v5 v5.0.4 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
 	github.com/knadh/koanf/providers/confmap v0.1.0 // indirect
 	github.com/knadh/koanf/v2 v2.1.2 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
-	github.com/opencontainers/runtime-spec v1.0.2 // indirect
+	github.com/opencontainers/runtime-spec v1.1.0 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
 	go.opentelemetry.io/collector/config/configtelemetry v0.114.1-0.20241202231142-b9ff1bc54c99 // indirect
 	go.opentelemetry.io/collector/pdata v1.20.1-0.20241202231142-b9ff1bc54c99 // indirect
 	go.opentelemetry.io/otel v1.32.0 // indirect

extension/cgroupruntimeextension/integration_test.go

@@ -0,0 +1,219 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+//go:build integration && sudo
+// +build integration,sudo
+
+// Priviledged access is required to set cgroup's memory and cpu max values
+
+package cgroupruntimeextension // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/cgroupruntimeextension"
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"os"
+	"path"
+	"path/filepath"
+	"runtime"
+	"runtime/debug"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/containerd/cgroups/v3/cgroup2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/component/componenttest"
+	"go.opentelemetry.io/collector/extension/extensiontest"
+)
+
+const (
+	defaultCgroup2Path = "/sys/fs/cgroup"
+)
+
+// cgroupMaxCpu returns the CPU max definition for a given cgroup slice path
+// File format: cpu_quote cpu_period
+func cgroupMaxCpu(filename string) (quota int64, period uint64, err error) {
+	out, err := os.ReadFile(filepath.Join(defaultCgroup2Path, filename, "cpu.max"))
+	if err != nil {
+		return 0, 0, err
+	}
+	values := strings.Split(strings.TrimSpace(string(out)), " ")
+	if values[0] == "max" {
+		quota = math.MaxInt64
+	} else {
+		quota, _ = strconv.ParseInt(values[0], 10, 64)
+	}
+	period, _ = strconv.ParseUint(values[1], 10, 64)
+	return quota, period, err
+}
+
+func TestCgroupV2Integration(t *testing.T) {
+	pointerInt64 := func(int int64) *int64 {
+		return &int
+	}
+	pointerUint64 := func(int uint64) *uint64 {
+		return &int
+	}
+
+	tests := []struct {
+		name string
+		// nil CPU quota == "max" cgroup string value
+		cgroupCpuQuota     *int64
+		cgroupCpuPeriod    uint64
+		cgroupMaxMemory    int64
+		config             *Config
+		expectedGoMaxProcs int
+		expectedGoMemLimit int64
+	}{
+		{
+			name:            "90% the max cgroup memory and 12 GOMAXPROCS",
+			cgroupCpuQuota:  pointerInt64(100000),
+			cgroupCpuPeriod: 8000,
+			// 128 Mb
+			cgroupMaxMemory: 134217728,
+			config: &Config{
+				GoMaxProcs: GoMaxProcsConfig{
+					Enabled: true,
+				},
+				GoMemLimit: GoMemLimitConfig{
+					Enabled: true,
+					Ratio:   0.9,
+				},
+			},
+			// 100000 / 8000
+			expectedGoMaxProcs: 12,
+			// 134217728 * 0.9
+			expectedGoMemLimit: 120795955,
+		},
+		{
+			name:            "50% of the max cgroup memory and 1 GOMAXPROCS",
+			cgroupCpuQuota:  pointerInt64(100000),
+			cgroupCpuPeriod: 100000,
+			// 128 Mb
+			cgroupMaxMemory: 134217728,
+			config: &Config{
+				GoMaxProcs: GoMaxProcsConfig{
+					Enabled: true,
+				},
+				GoMemLimit: GoMemLimitConfig{
+					Enabled: true,
+					Ratio:   0.5,
+				},
+			},
+			// 100000 / 100000
+			expectedGoMaxProcs: 1,
+			// 134217728 * 0.5
+			expectedGoMemLimit: 67108864,
+		},
+		{
+			name:            "10% of the max cgroup memory, max cpu, default GOMAXPROCS",
+			cgroupCpuQuota:  nil,
+			cgroupCpuPeriod: 100000,
+			// 128 Mb
+			cgroupMaxMemory: 134217728,
+			config: &Config{
+				GoMaxProcs: GoMaxProcsConfig{
+					Enabled: true,
+				},
+				GoMemLimit: GoMemLimitConfig{
+					Enabled: true,
+					Ratio:   0.1,
+				},
+			},
+			// GOMAXPROCS is set to the value of  `cpu.max / cpu.period`
+			// If cpu.max is set to max, GOMAXPROCS should not be
+			// modified
+			expectedGoMaxProcs: runtime.GOMAXPROCS(-1),
+			// 134217728 * 0.1
+			expectedGoMemLimit: 13421772,
+		},
+	}
+
+	cgroupPath, err := cgroup2.PidGroupPath(os.Getpid())
+	assert.NoError(t, err)
+	manager, err := cgroup2.Load(cgroupPath)
+	assert.NoError(t, err)
+
+	stats, err := manager.Stat()
+	require.NoError(t, err)
+
+	// Startup resource values
+	initialMaxMemory := stats.GetMemory().GetUsageLimit()
+	memoryCgroupCleanUp := func() {
+		err = manager.Update(&cgroup2.Resources{
+			Memory: &cgroup2.Memory{
+				Max: pointerInt64(int64(initialMaxMemory)),
+			},
+		})
+		assert.NoError(t, err)
+	}
+
+	if initialMaxMemory == math.MaxUint64 {
+		// fallback solution to set cgroup's max memory to "max"
+		memoryCgroupCleanUp = func() {
+			err = os.WriteFile(path.Join(defaultCgroup2Path, cgroupPath, "memory.max"), []byte("max"), 0o644)
+			assert.NoError(t, err)
+		}
+	}
+
+	initialCpuQuota, initialCpuPeriod, err := cgroupMaxCpu(cgroupPath)
+	require.NoError(t, err)
+	cpuCgroupCleanUp := func() {
+		fmt.Println(initialCpuQuota)
+		err = manager.Update(&cgroup2.Resources{
+			CPU: &cgroup2.CPU{
+				Max: cgroup2.NewCPUMax(pointerInt64(initialCpuQuota), pointerUint64(initialCpuPeriod)),
+			},
+		})
+		assert.NoError(t, err)
+	}
+
+	if initialCpuQuota == math.MaxInt64 {
+		// fallback solution to set cgroup's max cpu to "max"
+		cpuCgroupCleanUp = func() {
+			err = os.WriteFile(path.Join(defaultCgroup2Path, cgroupPath, "cpu.max"), []byte("max"), 0o644)
+			assert.NoError(t, err)
+		}
+	}
+
+	initialGoMem := debug.SetMemoryLimit(-1)
+	initialGoProcs := runtime.GOMAXPROCS(-1)
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			// restore startup cgroup initial resource values
+			t.Cleanup(func() {
+				debug.SetMemoryLimit(initialGoMem)
+				runtime.GOMAXPROCS(initialGoProcs)
+				memoryCgroupCleanUp()
+				cpuCgroupCleanUp()
+			})
+
+			err = manager.Update(&cgroup2.Resources{
+				Memory: &cgroup2.Memory{
+					// Default max memory must be
+					// overwritten
+					// to automemlimit change the GOMEMLIMIT
+					// value
+					Max: pointerInt64(test.cgroupMaxMemory),
+				},
+				CPU: &cgroup2.CPU{
+					Max: cgroup2.NewCPUMax(test.cgroupCpuQuota, pointerUint64(test.cgroupCpuPeriod)),
+				},
+			})
+			require.NoError(t, err)
+
+			factory := NewFactory()
+			ctx := context.Background()
+			extension, err := factory.Create(ctx, extensiontest.NewNopSettings(), test.config)
+			require.NoError(t, err)
+
+			err = extension.Start(ctx, componenttest.NewNopHost())
+			require.NoError(t, err)
+
+			assert.Equal(t, test.expectedGoMaxProcs, runtime.GOMAXPROCS(-1))
+			assert.Equal(t, test.expectedGoMemLimit, debug.SetMemoryLimit(-1))
+		})
+	}
+}