Skip to content
This repository has been archived by the owner on Oct 24, 2023. It is now read-only.

chore(CIS): add protect-kernel-defaults #999

Merged
merged 7 commits into from
Apr 25, 2019
Merged

chore(CIS): add protect-kernel-defaults #999

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
060add2
fix: add protect-kernel-defaults for CIS
andyzhangx Apr 10, 2019
8b7cb13
set up kernel tunable flags
andyzhangx Apr 22, 2019
45302f3
test: preferred way of applying and validating changes
jackfrancis Apr 22, 2019
f5a3008
test: add unit test, make default not static
jackfrancis Apr 22, 2019
88c5805
chore: only add --protect-kernel-defaults setting for ubuntu distro
jackfrancis Apr 25, 2019
5a3277b
test: update unit tests
jackfrancis Apr 25, 2019
d6f4929
chore: lint
jackfrancis Apr 25, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion parts/k8s/cloud-init/artifacts/sysctl-d-60-CIS.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,8 @@ net.ipv6.conf.all.accept_ra = 0
net.ipv6.conf.default.accept_ra = 0
# 3.3.2 Ensure IPv6 redirects are not accepted
net.ipv6.conf.all.accept_redirects = 0
net.ipv6.conf.default.accept_redirects = 0
net.ipv6.conf.default.accept_redirects = 0
# refer to https://github.com/kubernetes/kubernetes/blob/75d45bdfc9eeda15fb550e00da662c12d7d37985/pkg/kubelet/cm/container_manager_linux.go#L359-L397
vm.overcommit_memory = 1
Copy link
Contributor Author

@andyzhangx andyzhangx Apr 25, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jackfrancis I found this code change will fail all e2e test, and actully the kernel flags are not set correctly after this PR:

Failed to start ContainerManager [Invalid kernel flag: kernel/panic, expected value: 10, actual value: 0, Invalid kernel flag: kernel/panic_on_oops, expected value: 1, actual value: 0, Invalid kernel flag: vm/overcommit_memory, expected value: 1, actual value: 0]

There is also no /etc/sysctl.d/60-CIS.conf file under /etc/sysctl.d/

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Addressed by putting the whole thing behind "if ubuntu distro" only logic. Once the changes in this file are baked into the VHD then we can turn the functionality on, and run tests, by default

kernel.panic = 10
kernel.panic_on_oops = 1
2 changes: 1 addition & 1 deletion pkg/api/const.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ const (
Linux OSType = "Linux"
)

// the LinuxDistros supported by vlabs
// Distro string consts
const (
Ubuntu Distro = "ubuntu"
Ubuntu1804 Distro = "ubuntu-18.04"
Expand Down
6 changes: 6 additions & 0 deletions pkg/api/defaults-kubelet.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,12 @@ func (cs *ContainerService) setKubeletConfig() {
"--streaming-connection-idle-timeout": "5m",
}

// "--protect-kernel-defaults" is true is currently only valid using base Ubuntu OS image
// until the changes are baked into a VHD
if cs.Properties.IsUbuntuDistroForAllNodes() {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like it will always return false now @jackfrancis

// IsUbuntuDistroForAllNodes returns true if all of the agent pools plus masters are running the base Ubuntu image
func (p *Properties) IsUbuntuDistroForAllNodes() bool {
        if len(p.AgentPoolProfiles) > 0 {
                for _, ap := range p.AgentPoolProfiles {
                        if ap.Distro != Ubuntu && ap.Distro != Ubuntu1804 {
                                return false
                        }
                }
        }
        if p.MasterProfile != nil {
                return p.MasterProfile.Distro == Ubuntu || p.MasterProfile.Distro == Ubuntu1804
        }
        return true
}

I think I have already tried the latest Ubuntu version:

    "agentpoolosImageSKU": {
      "value": "aks-ubuntu-1604-201904"
    },
    "agentpoolosImageVersion": {
      "value": "2019.04.24"
    },

defaultKubeletConfig["--protect-kernel-defaults"] = "true"
}

// Set --non-masquerade-cidr if ip-masq-agent is disabled on AKS
if !cs.Properties.IsIPMasqAgentEnabled() {
defaultKubeletConfig["--non-masquerade-cidr"] = cs.Properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet
Expand Down
62 changes: 62 additions & 0 deletions pkg/api/defaults-kubelet_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -361,6 +361,68 @@ func TestEnforceNodeAllocatable(t *testing.T) {
}
}

func TestProtectKernelDefaults(t *testing.T) {
// Validate default
cs := CreateMockContainerService("testcluster", "1.12.7", 3, 2, false)
cs.setKubeletConfig()
k := cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
if k["--protect-kernel-defaults"] != "" {
t.Fatalf("got unexpected '--protect-kernel-defaults' kubelet config value %s, the expected value is %s",
k["--protect-kernel-defaults"], "pods")
}

// Validate that --protect-kernel-defaults is "true" by default for Ubuntu distros
for _, distro := range DistroValues {
switch distro {
case Ubuntu, Ubuntu1804:
cs = CreateMockContainerService("testcluster", "1.10.13", 3, 2, false)
cs.Properties.MasterProfile.Distro = distro
cs.Properties.AgentPoolProfiles[0].Distro = distro
cs.setKubeletConfig()
k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
if k["--protect-kernel-defaults"] != "true" {
t.Fatalf("got unexpected '--protect-kernel-defaults' kubelet config value %s, the expected value is %s",
k["--protect-kernel-defaults"], "true")
}
}
}

// Validate that --protect-kernel-defaults is overridable
cs = CreateMockContainerService("testcluster", "1.10.13", 3, 2, false)
cs.Properties.OrchestratorProfile.KubernetesConfig = &KubernetesConfig{
KubeletConfig: map[string]string{
"--protect-kernel-defaults": "false",
},
}
cs.setKubeletConfig()
k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
if k["--protect-kernel-defaults"] != "false" {
t.Fatalf("got unexpected '--protect-kernel-defaults' kubelet config value %s, the expected value is %s",
k["--protect-kernel-defaults"], "false")
}

// Validate that --protect-kernel-defaults is overridable for Ubuntu distros
for _, distro := range DistroValues {
switch distro {
case Ubuntu, Ubuntu1804:
cs = CreateMockContainerService("testcluster", "1.10.13", 3, 2, false)
cs.Properties.MasterProfile.Distro = "ubuntu"
cs.Properties.AgentPoolProfiles[0].Distro = "ubuntu"
cs.Properties.OrchestratorProfile.KubernetesConfig = &KubernetesConfig{
KubeletConfig: map[string]string{
"--protect-kernel-defaults": "false",
},
}
cs.setKubeletConfig()
k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
if k["--protect-kernel-defaults"] != "false" {
t.Fatalf("got unexpected '--protect-kernel-defaults' kubelet config value %s, the expected value is %s",
k["--protect-kernel-defaults"], "false")
}
}
}
}

func TestStaticWindowsConfig(t *testing.T) {
cs := CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 1, false)
p := GetK8sDefaultProperties(true)
Expand Down
3 changes: 3 additions & 0 deletions pkg/api/defaults.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ import (
"github.com/pkg/errors"
)

// DistroValues is a list of currently supported distros
var DistroValues = []Distro{"", Ubuntu, Ubuntu1804, RHEL, CoreOS, AKS, AKS1804, ACC1604}

// SetPropertiesDefaults for the container Properties, returns true if certs are generated
func (cs *ContainerService) SetPropertiesDefaults(isUpgrade, isScale bool) (bool, error) {
properties := cs.Properties
Expand Down
6 changes: 6 additions & 0 deletions test/e2e/kubernetes/scripts/net-config-validate.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ IPV4_LOG_MARTIANS_VALUE=1
IPV4_TCP_RETRIES2_VALUE=8
IPV6_ACCEPT_RA_VALUE=0
IPV6_ACCEPT_REDIRECTS_VALUE=0
KERNEL_PANIC_VALUE=10
KERNEL_PANIC_ON_OOPS_VALUE=1
VM_OVERCOMMIT_MEMORY_VALUE=1

set -x
cat /proc/sys/net/ipv4/conf/all/send_redirects | grep $IPV4_SEND_REDIRECTS_VALUE || exit 1
Expand All @@ -26,3 +29,6 @@ cat /proc/sys/net/ipv6/conf/default/accept_redirects | grep $IPV6_ACCEPT_REDIREC

# validate net config workaround from kubelet.service
cat /proc/sys/net/ipv4/tcp_retries2 | grep $IPV4_TCP_RETRIES2_VALUE || exit 1
cat /proc/sys/kernel/panic | grep $KERNEL_PANIC_VALUE || exit 1
cat /proc/sys/kernel/panic_on_oops | grep $KERNEL_PANIC_ON_OOPS_VALUE || exit 1
cat /proc/sys/vm/overcommit_memory | grep $VM_OVERCOMMIT_MEMORY_VALUE || exit 1