Skip to content

Commit

Permalink
fix: run memfill using cgexec instead of runc
Browse files Browse the repository at this point in the history 
With runc we cannot join exisitng cgroups, but using a child cgroup for
the memfill attack when cgroup v1 is active yields undesired results as
the desired usage/limit of the parent cgroup is not visible.
  • Loading branch information
@joshiste
joshiste committed Aug 21, 2024
1 parent b125f64 commit 2c9e307
Show file tree
Hide file tree
Showing 4 changed files with 42 additions and 103 deletions.
2 changes: 1 addition & 1 deletion go/action_kit_commons/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ require (
golang.org/x/mod v0.18.0 // indirect
golang.org/x/net v0.26.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/sys v0.24.0 // indirect
golang.org/x/telemetry v0.0.0-20240701175443-4e29c7872ac1 // indirect
golang.org/x/term v0.21.0 // indirect
golang.org/x/text v0.16.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go/action_kit_commons/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/telemetry v0.0.0-20240701175443-4e29c7872ac1 h1:jveUVYFLPlIma1aZBg9rrUN+Dqk4e6QbVSGiZGwA/2Y=
golang.org/x/telemetry v0.0.0-20240701175443-4e29c7872ac1/go.mod h1:n38mvGdgc4dA684EC4NwQwoPKSw4jyKw8/DgZHDA1Dk=
golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
Expand Down
117 changes: 24 additions & 93 deletions go/action_kit_commons/memfill/memfill.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,18 @@ package memfill

import (
"context"
"errors"
"fmt"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/steadybit/action-kit/go/action_kit_commons/runc"
"syscall"
"os/exec"
"strconv"
"time"
)

type MemFill struct {
bundle runc.ContainerBundle
runc runc.Runc

cmd *exec.Cmd
state *runc.BackgroundState
args []string
Noop bool
}

type Mode string
Expand Down Expand Up @@ -50,17 +45,19 @@ func (o Opts) processArgs() []string {
return args
}

func New(ctx context.Context, r runc.Runc, sidecar SidecarOpts, opts Opts) (*MemFill, error) {
bundle, err := createBundle(ctx, r, sidecar, opts.processArgs()...)
if err != nil {
log.Error().Err(err).Msg("failed to create start bundle")
return nil, err
}
func New(targeProcess runc.LinuxProcessInfo, opts Opts) (*MemFill, error) {
args := append([]string{
"nsenter", "-t", "1", "-C", "--",
//when util-linux package >= 2.39 is broadly available we could also the cgroup change using nsenter,
"cgexec", "-g", fmt.Sprintf("memory:%s", targeProcess.CGroupPath),
"nsenter", "-t", strconv.Itoa(targeProcess.Pid), "-p", "-F", "--",
},
opts.processArgs()...,
)

cmd := runc.RootCommandContext(context.Background(), args[0], args[1:]...)

return &MemFill{
bundle: bundle,
runc: r,
}, nil
return &MemFill{cmd: cmd, args: opts.processArgs()}, nil
}

func (mf *MemFill) Exited() (bool, error) {
Expand All @@ -69,105 +66,39 @@ func (mf *MemFill) Exited() (bool, error) {

func (mf *MemFill) Start() error {
log.Info().
Str("containerId", mf.bundle.ContainerId()).
Strs("args", mf.args).
Msg("Starting memfill")

if state, err := runc.RunBundleInBackground(context.Background(), mf.runc, mf.bundle); err != nil {
return fmt.Errorf("failed to start memfill: %w", err)
if state, err := runc.RunCommandInBackground(mf.cmd, log.With().Str("id", "memfill").Logger()); err != nil {
return fmt.Errorf("failed to start: %w", err)
} else {
mf.state = state
}

return nil
}

func (mf *MemFill) Stop() error {
log.Info().
Str("containerId", mf.bundle.ContainerId()).
Msg("stopping memfill")
ctx := context.Background()

if err := mf.runc.Kill(ctx, mf.bundle.ContainerId(), syscall.SIGINT); err != nil {
log.Warn().Str("id", mf.bundle.ContainerId()).Err(err).Msg("failed to send SIGINT to container")
//as the process is running with a different user, we also need to do so, for sending signals
ctx := context.Background()
if err := runc.RootCommandContext(ctx, "kill", "-s", "SIGINT", strconv.Itoa(mf.cmd.Process.Pid)).Run(); err != nil {
log.Warn().Err(err).Msg("failed to send SIGINT to memfill")
}

timerStart := time.AfterFunc(10*time.Second, func() {
if err := mf.runc.Kill(ctx, mf.bundle.ContainerId(), syscall.SIGTERM); err != nil {
log.Warn().Str("id", mf.bundle.ContainerId()).Err(err).Msg("failed to send SIGTERM to container")
if err := runc.RootCommandContext(ctx, "kill", "-s", "SIGTERM", strconv.Itoa(mf.cmd.Process.Pid)).Run(); err != nil {
log.Warn().Err(err).Msg("failed to send SIGTERM to memfill")
}
})

mf.state.Wait()
timerStart.Stop()

if err := mf.runc.Delete(ctx, mf.bundle.ContainerId(), false); err != nil {
level := zerolog.WarnLevel
if errors.Is(err, runc.ErrContainerNotFound) {
level = zerolog.DebugLevel
}
log.WithLevel(level).Str("id", mf.bundle.ContainerId()).Err(err).Msg("failed to delete container")
}

if err := mf.bundle.Remove(); err != nil {
log.Warn().Str("id", mf.bundle.ContainerId()).Err(err).Msg("failed to remove bundle")
}
return nil
}

func (mf *MemFill) Args() []string {
return mf.args
}

type SidecarOpts struct {
TargetProcess runc.LinuxProcessInfo
IdSuffix string
ImagePath string
}

func createBundle(ctx context.Context, r runc.Runc, sidecar SidecarOpts, processArgs ...string) (runc.ContainerBundle, error) {
containerId := getNextContainerId(sidecar.IdSuffix)
bundle, err := r.Create(ctx, sidecar.ImagePath, containerId)
if err != nil {
return nil, fmt.Errorf("failed to prepare bundle: %w", err)
}

success := false
defer func() {
if success {
return
}
if err := bundle.Remove(); err != nil {
log.Warn().Str("id", containerId).Err(err).Msg("failed to remove bundle")
}
}()

runc.RefreshNamespaces(ctx, sidecar.TargetProcess.Namespaces, specs.PIDNamespace, specs.CgroupNamespace)

if err := bundle.EditSpec(
runc.WithHostname(containerId),
runc.WithAnnotations(map[string]string{
"com.steadybit.sidecar": "true",
}),
runc.WithProcessArgs(processArgs...),
runc.WithProcessCwd("/tmp"),
runc.WithCgroupPath(sidecar.TargetProcess.CGroupPath, containerId),
runc.WithDisableOOMKiller(),
runc.WithNamespaces(runc.FilterNamespaces(sidecar.TargetProcess.Namespaces, specs.PIDNamespace, specs.CgroupNamespace)),
runc.WithCapabilities("CAP_SYS_RESOURCE"),
runc.WithMountIfNotPresent(specs.Mount{
Destination: "/tmp",
Type: "tmpfs",
Options: []string{"noexec", "nosuid", "nodev", "rprivate"},
}),
); err != nil {
return nil, err
}

success = true

return bundle, nil
}

func getNextContainerId(suffix string) string {
return fmt.Sprintf("sb-memfill-%d-%s", time.Now().UnixMilli(), suffix)
}
24 changes: 15 additions & 9 deletions go/action_kit_commons/runc/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"errors"
"fmt"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"io"
"os"
Expand All @@ -35,27 +36,32 @@ func RunBundleInBackground(ctx context.Context, runc Runc, bundle ContainerBundl
return nil, fmt.Errorf("failed to run %s: %w", bundle.ContainerId(), err)
}

var outb bytes.Buffer
logger := log.With().Str("id", bundle.ContainerId()).Logger()

return RunCommandInBackground(cmd, logger)
}

func RunCommandInBackground(cmd *exec.Cmd, logger zerolog.Logger) (*BackgroundState, error) {
var outb_stderr bytes.Buffer
pr, pw := io.Pipe()
writer := io.MultiWriter(&outb, pw)
cmd.Stdout = writer
cmd.Stderr = writer
cmd.Stdout = pw
cmd.Stderr = io.MultiWriter(&outb_stderr, pw)

go func() {
defer func() { _ = pr.Close() }()
bufReader := bufio.NewReader(pr)

for {
if line, err := bufReader.ReadString('\n'); err == nil {
log.Debug().Str("id", bundle.ContainerId()).Msg(line)
logger.Debug().Msg(line)
} else {
break
}
}
}()

if err = cmd.Start(); err != nil {
return nil, fmt.Errorf("failed to start %s: %w", bundle.ContainerId(), err)
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("failed to start: %w", err)
}

result := &BackgroundState{
Expand All @@ -67,15 +73,15 @@ func RunBundleInBackground(ctx context.Context, runc Runc, bundle ContainerBundl
go func(r *BackgroundState) {
defer func() { _ = pw.Close() }()
err := cmd.Wait()
log.Trace().Str("id", bundle.ContainerId()).Int("exitCode", cmd.ProcessState.ExitCode()).Msgf("%s exited", bundle.ContainerId())
logger.Trace().Int("exitCode", cmd.ProcessState.ExitCode()).Msg("exited.")

r.cond.L.Lock()
defer r.cond.L.Unlock()

r.exited = true
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
exitErr.Stderr = outb.Bytes()
exitErr.Stderr = outb_stderr.Bytes()
r.err = exitErr
} else {
r.err = err
Expand Down

0 comments on commit 2c9e307

Please sign in to comment.