From aebe4f7ad2fd3ae67e84f2f3a8ef9a1e97a478eb Mon Sep 17 00:00:00 2001 From: Paul Greenberg Date: Sun, 15 Mar 2020 22:38:35 -0400 Subject: [PATCH] firewall: add nftables backend Resolves: #461 Signed-off-by: Paul Greenberg --- plugins/meta/firewall/README.md | 96 +++++ plugins/meta/firewall/firewall.go | 2 + .../meta/firewall/firewall_nftables_test.go | 162 ++++++++ plugins/meta/firewall/nftables.go | 347 ++++++++++++++++++ 4 files changed, 607 insertions(+) create mode 100644 plugins/meta/firewall/firewall_nftables_test.go create mode 100644 plugins/meta/firewall/nftables.go diff --git a/plugins/meta/firewall/README.md b/plugins/meta/firewall/README.md index f73a41213..b5cd1831b 100644 --- a/plugins/meta/firewall/README.md +++ b/plugins/meta/firewall/README.md @@ -133,3 +133,99 @@ of the container as shown: - `-s 10.88.0.2 -m conntrack --ctstate RELATED,ESTABLISHED -j CNI-FORWARD` - `-d 10.88.0.2 -j CNI-FORWARD` +## nftables backend rule structure + +The prerequisite for the backend is the existence of `filter` table and +the existence of `FORWARD` chain in the table. + +A sample standalone config list (with the file extension `.conflist`) using +`nftables` backend might look like: + +```json +{ + "cniVersion": "0.4.0", + "name": "podman", + "plugins": [ + { + "type": "bridge", + "bridge": "cni-podman0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "ranges": [ + [ + { + "subnet": "192.168.124.0/24", + "gateway": "192.168.124.1" + } + ] + ] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + }, + { + "type": "firewall", + "backend": "nftables" + } + ] +} +``` + +Prior to the invocation of CNI `firewall` plugin, the `FORWARD` chain in `filter` table is: + +``` +$ nft list chain ip filter FORWARD -a +table ip filter { + chain FORWARD { # handle 2 + type filter hook forward priority 0; policy drop; + oifname "virbr0" ip daddr 192.168.122.0/24 ct state established,related counter packets 0 bytes 0 accept # handle 51 + iifname "virbr0" ip saddr 192.168.122.0/24 counter packets 0 bytes 0 accept # handle 52 + iifname "virbr0" oifname "virbr0" counter packets 0 bytes 0 accept # handle 53 + log prefix "IPv4 FORWARD drop: " flags all # handle 54 + counter packets 10 bytes 630 drop # handle 55 + } +} +``` + +After starting a container, the plugin executes the following commands based +on the configuration above. Please note that `position 51` refers to the handle +at the top of the chain. + +``` +nft insert rule filter FORWARD position 51 oifname "cni-podman0" ip daddr 192.168.124.0/24 ct state established,related counter packets 0 bytes 0 accept +nft insert rule filter FORWARD position 51 iifname "cni-podman0" ip saddr 192.168.124.0/24 counter packets 0 bytes 0 accept +nft insert rule filter FORWARD position 51 iifname "cni-podman0" oifname "cni-podman0" counter packets 0 bytes 0 accept +``` + +After the plugin's execution, the chain looks like this: + +``` +$ nft list chain ip filter FORWARD -a +table ip filter { + chain FORWARD { # handle 2 + type filter hook forward priority 0; policy drop; + oifname "cni-podman0" ip daddr 192.168.124.0/24 ct state established,related counter packets 100 bytes 113413 accept # handle 71 + iifname "cni-podman0" ip saddr 192.168.124.0/24 counter packets 124 bytes 12996 accept # handle 72 + iifname "cni-podman0" oifname "cni-podman0" counter packets 0 bytes 0 accept # handle 73 + oifname "virbr0" ip daddr 192.168.122.0/24 ct state established,related counter packets 0 bytes 0 accept # handle 51 + iifname "virbr0" ip saddr 192.168.122.0/24 counter packets 0 bytes 0 accept # handle 52 + iifname "virbr0" oifname "virbr0" counter packets 0 bytes 0 accept # handle 53 + log prefix "IPv4 FORWARD drop: " flags all # handle 54 + counter packets 10 bytes 630 drop # handle 55 + } +} +``` + +Subsequent executions of the plugin do not create additional rules in the chain, unless +the CNI network configuration changes. diff --git a/plugins/meta/firewall/firewall.go b/plugins/meta/firewall/firewall.go index 875943beb..f33ac54dd 100644 --- a/plugins/meta/firewall/firewall.go +++ b/plugins/meta/firewall/firewall.go @@ -97,6 +97,8 @@ func getBackend(conf *FirewallNetConf) (FirewallBackend, error) { switch conf.Backend { case "iptables": return newIptablesBackend(conf) + case "nftables": + return newNftablesBackend(conf) case "firewalld": return newFirewalldBackend(conf) } diff --git a/plugins/meta/firewall/firewall_nftables_test.go b/plugins/meta/firewall/firewall_nftables_test.go new file mode 100644 index 000000000..0a5cc56cc --- /dev/null +++ b/plugins/meta/firewall/firewall_nftables_test.go @@ -0,0 +1,162 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types/current" + "github.com/containernetworking/plugins/pkg/ns" + "github.com/containernetworking/plugins/pkg/testutils" + + "github.com/vishvananda/netlink" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func validateNftRules(bytes []byte) { + prevResult := getPrevResult(bytes) + + for _, ip := range prevResult.IPs { + Expect(ip).To(Equal(true)) + /* + ipt, err := iptables.NewWithProtocol(protoForIP(ip.Address)) + Expect(err).NotTo(HaveOccurred()) + + // Ensure chains + chains, err := ipt.ListChains("filter") + Expect(err).NotTo(HaveOccurred()) + foundAdmin, foundPriv := findChains(chains) + Expect(foundAdmin).To(Equal(true)) + Expect(foundPriv).To(Equal(true)) + + // Look for the FORWARD chain jump rules to our custom chains + rules, err := ipt.List("filter", "FORWARD") + Expect(err).NotTo(HaveOccurred()) + Expect(len(rules)).Should(BeNumerically(">", 1)) + _, foundPriv = findForwardJumpRules(rules) + Expect(foundPriv).To(Equal(true)) + + // Look for the allow rules in our custom FORWARD chain + rules, err = ipt.List("filter", "CNI-FORWARD") + Expect(err).NotTo(HaveOccurred()) + Expect(len(rules)).Should(BeNumerically(">", 1)) + foundAdmin, _ = findForwardJumpRules(rules) + Expect(foundAdmin).To(Equal(true)) + + // Look for the IP allow rules + foundOne, foundTwo := findForwardAllowRules(rules, ipString(ip.Address)) + Expect(foundOne).To(Equal(true)) + Expect(foundTwo).To(Equal(true)) + */ + } +} + +var _ = Describe("firewall plugin nftables backend v0.4.x", func() { + var originalNS, targetNS ns.NetNS + const IFNAME string = "dummy0" + + fullConf := []byte(`{ + "name": "test", + "type": "firewall", + "backend": "nftables", + "ifName": "dummy0", + "cniVersion": "0.4.0", + "prevResult": { + "interfaces": [ + {"name": "dummy0"} + ], + "ips": [ + { + "version": "4", + "address": "192.168.200.10/24", + "interface": 0 + }, + { + "version": "6", + "address": "2001:db8:1:2::1/64", + "interface": 0 + } + ] + } + }`) + + BeforeEach(func() { + // Create a new NetNS so we don't modify the host + var err error + originalNS, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + + err = originalNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + + err = netlink.LinkAdd(&netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: IFNAME, + }, + }) + Expect(err).NotTo(HaveOccurred()) + _, err = netlink.LinkByName(IFNAME) + Expect(err).NotTo(HaveOccurred()) + return nil + }) + Expect(err).NotTo(HaveOccurred()) + + targetNS, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + Expect(originalNS.Close()).To(Succeed()) + Expect(targetNS.Close()).To(Succeed()) + }) + + It("installs nftables rules, Check rules then cleans up on delete using v4.0.x", func() { + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: targetNS.Path(), + IfName: IFNAME, + StdinData: fullConf, + } + + err := originalNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + + r, _, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) + Expect(err).NotTo(HaveOccurred()) + + _, err = current.GetResult(r) + Expect(err).NotTo(HaveOccurred()) + + err = testutils.CmdCheckWithArgs(args, func() error { + return cmdCheck(args) + }) + Expect(err).NotTo(HaveOccurred()) + validateNftRules(fullConf) + + /* + err = testutils.CmdDelWithArgs(args, func() error { + return cmdDel(args) + }) + Expect(err).NotTo(HaveOccurred()) + validateCleanedUp(fullConf) + */ + return nil + }) + Expect(err).NotTo(HaveOccurred()) + }) +}) diff --git a/plugins/meta/firewall/nftables.go b/plugins/meta/firewall/nftables.go new file mode 100644 index 000000000..877a316ff --- /dev/null +++ b/plugins/meta/firewall/nftables.go @@ -0,0 +1,347 @@ +// Copyright 2018 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "fmt" + "github.com/containernetworking/cni/pkg/types/current" + "net" + "os/exec" + "strconv" + "strings" +) + +type nftBackend struct { + targetTable string + targetChain string + targetHandle uint64 + targetInterface string + targetAddresses []*nftAddress + rules []*nftRule +} + +type nftAddress struct { + ifname string + ipaddr *net.IPNet + netaddr *net.IPNet + handle uint64 + inRuleExists bool + outRuleExists bool + inRuleParts []string + outRuleParts []string +} + +func newNftAddress(ifname string, addr *net.IPNet) (*nftAddress, error) { + _, netAddr, err := net.ParseCIDR(addr.String()) + if err != nil { + return nil, err + } + ifaddr := &nftAddress{ + ifname: ifname, + ipaddr: addr, + netaddr: netAddr, + } + ifaddr.inRuleParts = []string{ + "oifname", fmt.Sprintf("\"%s\"", ifname), + "ip", "daddr", netAddr.String(), + "ct", "state", "established,related", + } + ifaddr.outRuleParts = []string{ + "iifname", fmt.Sprintf("\"%s\"", ifname), + "ip", "saddr", netAddr.String(), + } + return ifaddr, nil +} + +func (a *nftAddress) getInboundRulePattern() string { + return strings.Join(a.inRuleParts, " ") +} + +func (a *nftAddress) getOutboundRulePattern() string { + return strings.Join(a.outRuleParts, " ") +} + +func (a *nftAddress) getInboundRule(table string, chain string, handle uint64) []string { + rule := []string{} + if handle == 0 { + rule = append(rule, []string{"add", "rule", table, chain}...) + } else { + rule = append(rule, []string{"insert", "rule", table, chain, "position", fmt.Sprintf("%d", handle)}...) + } + rule = append(rule, a.inRuleParts...) + rule = append(rule, []string{"counter", "packets", "0", "bytes", "0", "accept"}...) + return rule + +} + +func (a *nftAddress) getOutboundRule(table string, chain string, handle uint64) []string { + rule := []string{} + if handle == 0 { + rule = append(rule, []string{"add", "rule", table, chain}...) + } else { + rule = append(rule, []string{"insert", "rule", table, chain, "position", fmt.Sprintf("%d", handle)}...) + } + rule = append(rule, a.outRuleParts...) + rule = append(rule, []string{"counter", "packets", "0", "bytes", "0", "accept"}...) + return rule +} + +type nftRule struct { + text string + handle uint64 + verdict string +} + +func newNftRule(s string) (*nftRule, error) { + r := &nftRule{ + text: s, + verdict: "unknown", + } + if err := r.parseText(); err != nil { + return nil, err + } + if strings.HasSuffix(r.text, " accept") { + r.verdict = "accept" + } + if strings.HasSuffix(r.text, " drop") { + r.verdict = "drop" + } + return r, nil +} + +func (r *nftRule) parseText() error { + offset := strings.LastIndex(r.text, "# handle ") + if offset > 0 { + // The rule handle was found + handle := strings.TrimLeft(r.text[offset:], "# handle ") + r.text = strings.TrimSpace(r.text[:offset-1]) + handleID, err := strconv.ParseUint(handle, 0, 64) + if err != nil { + return err + } + r.handle = handleID + } + + return nil +} + +// nftBackend implements the FirewallBackend interface +var _ FirewallBackend = &nftBackend{} + +func newNftablesBackend(conf *FirewallNetConf) (FirewallBackend, error) { + backend := &nftBackend{ + targetTable: "filter", + targetChain: "FORWARD", + targetHandle: 0, + targetAddresses: []*nftAddress{}, + rules: []*nftRule{}, + } + return backend, nil +} + +func (nb *nftBackend) execCommand(args []string) ([]string, []string, error) { + var stdout, stderr bytes.Buffer + cmd := exec.Command("nft", args...) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return []string{}, []string{}, fmt.Errorf("Error executing %s: %s", args, err) + } + + stdoutString := stdout.String() + stderrString := stderr.String() + stdoutLines := strings.Split(stdoutString, "\n") + stderrLines := strings.Split(stderrString, "\n") + return stdoutLines, stderrLines, nil +} + +func (nb *nftBackend) getRules() error { + cmdArgs := []string{"list", "chain", "ip", nb.targetTable, nb.targetChain, "-a"} + stdoutLines, _, err := nb.execCommand(cmdArgs) + if err != nil { + return err + } + for _, line := range stdoutLines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if strings.HasPrefix(line, "table") { + continue + } + if strings.HasPrefix(line, "type") { + continue + } + if strings.HasPrefix(line, "chain") { + continue + } + if strings.HasPrefix(line, "}") { + continue + } + rule, err := newNftRule(line) + if err != nil { + return err + } + if nb.targetHandle == 0 { + nb.targetHandle = rule.handle + } + nb.rules = append(nb.rules, rule) + } + return nil +} + +func (nb *nftBackend) addRules() error { + var isIntraInterfaceRuleExists bool + + intraInterfaceRule := fmt.Sprintf( + "iifname \"%s\" oifname \"%s\"", + nb.targetInterface, + nb.targetInterface, + ) + + for _, rule := range nb.rules { + if rule.verdict != "accept" { + continue + } + + if strings.HasPrefix(rule.text, intraInterfaceRule) { + isIntraInterfaceRuleExists = true + continue + } + + for _, addr := range nb.targetAddresses { + if strings.HasPrefix(rule.text, addr.getInboundRulePattern()) { + addr.inRuleExists = true + } + if strings.HasPrefix(rule.text, addr.getOutboundRulePattern()) { + addr.outRuleExists = true + } + } + } + + if !isIntraInterfaceRuleExists { + if len(nb.rules) == 0 { + if err := nb.addRule([]string{ + "add", "rule", nb.targetTable, nb.targetChain, + "iifname", fmt.Sprintf("\"%s\"", nb.targetInterface), + "oifname", fmt.Sprintf("\"%s\"", nb.targetInterface), + "counter", "packets", "0", "bytes", "0", "accept", + }); err != nil { + return err + } + } else { + if err := nb.addRule([]string{ + "insert", "rule", nb.targetTable, nb.targetChain, + "position", fmt.Sprintf("%d", nb.targetHandle), + "iifname", fmt.Sprintf("\"%s\"", nb.targetInterface), + "oifname", fmt.Sprintf("\"%s\"", nb.targetInterface), + "counter", "packets", "0", "bytes", "0", "accept", + }); err != nil { + return err + } + } + } + + for _, addr := range nb.targetAddresses { + if addr.ipaddr.IP.To4() == nil { + continue + } + if !addr.inRuleExists { + rule := addr.getInboundRule(nb.targetTable, nb.targetChain, nb.targetHandle) + if err := nb.addRule(rule); err != nil { + return err + } + } + if !addr.outRuleExists { + rule := addr.getOutboundRule(nb.targetTable, nb.targetChain, nb.targetHandle) + if err := nb.addRule(rule); err != nil { + return err + } + } + } + + return nil +} + +func (nb *nftBackend) addRule(cmdArgs []string) error { + stdoutLines, stderrLines, err := nb.execCommand(cmdArgs) + if err != nil { + return fmt.Errorf( + "encountered error %s: %s (stdout: %s, stderr: %s)", + cmdArgs, err, stdoutLines, stderrLines, + ) + } + return nil +} + +func (nb *nftBackend) isValidInput(result *current.Result) error { + if len(result.Interfaces) == 0 { + return fmt.Errorf("the data passed to firewall plugin did not contain network interfaces") + } + + if result.Interfaces[0].Name == "" { + return fmt.Errorf("the data passed to firewall plugin has no bridge name, e.g. cnibr0") + } + + nb.targetInterface = result.Interfaces[0].Name + + if len(result.IPs) == 0 { + return fmt.Errorf("the data passed to firewall plugin has no IP addresses") + } + + for _, entry := range result.IPs { + if entry.Address.String() == "" { + return fmt.Errorf("the data passed to firewall plugin has empty IP address") + } + + addr, err := newNftAddress(nb.targetInterface, &entry.Address) + if err != nil { + return fmt.Errorf( + "the data passed to firewall plugin triggered error %s %s", + entry.Address.String(), + err, + ) + } + + nb.targetAddresses = append(nb.targetAddresses, addr) + } + + return nil +} + +func (nb *nftBackend) Add(conf *FirewallNetConf, result *current.Result) error { + if err := nb.isValidInput(result); err != nil { + return fmt.Errorf("nftBackend.Add() %s", err) + } + + if err := nb.getRules(); err != nil { + return fmt.Errorf("nftBackend.Add() %s", err) + } + + if err := nb.addRules(); err != nil { + return fmt.Errorf("nftBackend.Add() %s", err) + } + + return nil +} + +func (nb *nftBackend) Del(conf *FirewallNetConf, result *current.Result) error { + return nil +} + +func (nb *nftBackend) Check(conf *FirewallNetConf, result *current.Result) error { + return nil +}